gzdoom/wadsrc/static/shaders/glsl/main.fp
Rachael Alexanderson 7652fd7ee0 Revert "- model normals fix"
This worked in some instances but it also broke attenuated lights in other instances, and also caused the shaders to fail to compile on Mac.

This reverts commit 679518f8e5.
2022-05-08 07:07:54 -04:00

800 lines
21 KiB
GLSL

layout(location = 0) in vec4 vTexCoord;
layout(location = 1) in vec4 vColor;
layout(location = 2) in vec4 pixelpos;
layout(location = 3) in vec3 glowdist;
layout(location = 4) in vec3 gradientdist;
layout(location = 5) in vec4 vWorldNormal;
layout(location = 6) in vec4 vEyeNormal;
layout(location = 9) in vec3 vLightmap;
#ifdef NO_CLIPDISTANCE_SUPPORT
layout(location = 7) in vec4 ClipDistanceA;
layout(location = 8) in vec4 ClipDistanceB;
#endif
layout(location=0) out vec4 FragColor;
#ifdef GBUFFER_PASS
layout(location=1) out vec4 FragFog;
layout(location=2) out vec4 FragNormal;
#endif
struct Material
{
vec4 Base;
vec4 Bright;
vec4 Glow;
vec3 Normal;
vec3 Specular;
float Glossiness;
float SpecularLevel;
float Metallic;
float Roughness;
float AO;
};
vec4 Process(vec4 color);
vec4 ProcessTexel();
Material ProcessMaterial(); // note that this is deprecated. Use SetupMaterial!
void SetupMaterial(inout Material mat);
vec4 ProcessLight(Material mat, vec4 color);
vec3 ProcessMaterialLight(Material material, vec3 color);
vec2 GetTexCoord();
// These get Or'ed into uTextureMode because it only uses its 3 lowermost bits.
const int TEXF_Brightmap = 0x10000;
const int TEXF_Detailmap = 0x20000;
const int TEXF_Glowmap = 0x40000;
const int TEXF_ClampY = 0x80000;
//===========================================================================
//
// Color to grayscale
//
//===========================================================================
float grayscale(vec4 color)
{
return dot(color.rgb, vec3(0.3, 0.56, 0.14));
}
//===========================================================================
//
// Desaturate a color
//
//===========================================================================
vec4 dodesaturate(vec4 texel, float factor)
{
if (factor != 0.0)
{
float gray = grayscale(texel);
return mix (texel, vec4(gray,gray,gray,texel.a), factor);
}
else
{
return texel;
}
}
//===========================================================================
//
// Desaturate a color
//
//===========================================================================
vec4 desaturate(vec4 texel)
{
return dodesaturate(texel, uDesaturationFactor);
}
//===========================================================================
//
// Texture tinting code originally from JFDuke but with a few more options
//
//===========================================================================
const int Tex_Blend_Alpha = 1;
const int Tex_Blend_Screen = 2;
const int Tex_Blend_Overlay = 3;
const int Tex_Blend_Hardlight = 4;
vec4 ApplyTextureManipulation(vec4 texel, int blendflags)
{
// Step 1: desaturate according to the material's desaturation factor.
texel = dodesaturate(texel, uTextureModulateColor.a);
// Step 2: Invert if requested
if ((blendflags & 8) != 0)
{
texel.rgb = vec3(1.0 - texel.r, 1.0 - texel.g, 1.0 - texel.b);
}
// Step 3: Apply additive color
texel.rgb += uTextureAddColor.rgb;
// Step 4: Colorization, including gradient if set.
texel.rgb *= uTextureModulateColor.rgb;
// Before applying the blend the value needs to be clamped to [0..1] range.
texel.rgb = clamp(texel.rgb, 0.0, 1.0);
// Step 5: Apply a blend. This may just be a translucent overlay or one of the blend modes present in current Build engines.
if ((blendflags & 7) != 0)
{
vec3 tcol = texel.rgb * 255.0; // * 255.0 to make it easier to reuse the integer math.
vec4 tint = uTextureBlendColor * 255.0;
switch (blendflags & 7)
{
default:
tcol.b = tcol.b * (1.0 - uTextureBlendColor.a) + tint.b * uTextureBlendColor.a;
tcol.g = tcol.g * (1.0 - uTextureBlendColor.a) + tint.g * uTextureBlendColor.a;
tcol.r = tcol.r * (1.0 - uTextureBlendColor.a) + tint.r * uTextureBlendColor.a;
break;
// The following 3 are taken 1:1 from the Build engine
case Tex_Blend_Screen:
tcol.b = 255.0 - (((255.0 - tcol.b) * (255.0 - tint.r)) / 256.0);
tcol.g = 255.0 - (((255.0 - tcol.g) * (255.0 - tint.g)) / 256.0);
tcol.r = 255.0 - (((255.0 - tcol.r) * (255.0 - tint.b)) / 256.0);
break;
case Tex_Blend_Overlay:
tcol.b = tcol.b < 128.0? (tcol.b * tint.b) / 128.0 : 255.0 - (((255.0 - tcol.b) * (255.0 - tint.b)) / 128.0);
tcol.g = tcol.g < 128.0? (tcol.g * tint.g) / 128.0 : 255.0 - (((255.0 - tcol.g) * (255.0 - tint.g)) / 128.0);
tcol.r = tcol.r < 128.0? (tcol.r * tint.r) / 128.0 : 255.0 - (((255.0 - tcol.r) * (255.0 - tint.r)) / 128.0);
break;
case Tex_Blend_Hardlight:
tcol.b = tint.b < 128.0 ? (tcol.b * tint.b) / 128.0 : 255.0 - (((255.0 - tcol.b) * (255.0 - tint.b)) / 128.0);
tcol.g = tint.g < 128.0 ? (tcol.g * tint.g) / 128.0 : 255.0 - (((255.0 - tcol.g) * (255.0 - tint.g)) / 128.0);
tcol.r = tint.r < 128.0 ? (tcol.r * tint.r) / 128.0 : 255.0 - (((255.0 - tcol.r) * (255.0 - tint.r)) / 128.0);
break;
}
texel.rgb = tcol / 255.0;
}
return texel;
}
//===========================================================================
//
// This function is common for all (non-special-effect) fragment shaders
//
//===========================================================================
vec4 getTexel(vec2 st)
{
vec4 texel = texture(tex, st);
//
// Apply texture modes
//
switch (uTextureMode & 0xffff)
{
case 1: // TM_STENCIL
texel.rgb = vec3(1.0,1.0,1.0);
break;
case 2: // TM_OPAQUE
texel.a = 1.0;
break;
case 3: // TM_INVERSE
texel = vec4(1.0-texel.r, 1.0-texel.b, 1.0-texel.g, texel.a);
break;
case 4: // TM_ALPHATEXTURE
{
float gray = grayscale(texel);
texel = vec4(1.0, 1.0, 1.0, gray*texel.a);
break;
}
case 5: // TM_CLAMPY
if (st.t < 0.0 || st.t > 1.0)
{
texel.a = 0.0;
}
break;
case 6: // TM_OPAQUEINVERSE
texel = vec4(1.0-texel.r, 1.0-texel.b, 1.0-texel.g, 1.0);
break;
case 7: //TM_FOGLAYER
return texel;
}
if ((uTextureMode & TEXF_ClampY) != 0)
{
if (st.t < 0.0 || st.t > 1.0)
{
texel.a = 0.0;
}
}
// Apply the texture modification colors.
int blendflags = int(uTextureAddColor.a); // this alpha is unused otherwise
if (blendflags != 0)
{
// only apply the texture manipulation if it contains something.
texel = ApplyTextureManipulation(texel, blendflags);
}
// Apply the Doom64 style material colors on top of everything from the texture modification settings.
// This may be a bit redundant in terms of features but the data comes from different sources so this is unavoidable.
texel.rgb += uAddColor.rgb;
if (uObjectColor2.a == 0.0) texel *= uObjectColor;
else texel *= mix(uObjectColor, uObjectColor2, gradientdist.z);
// Last but not least apply the desaturation from the sector's light.
return desaturate(texel);
}
//===========================================================================
//
// Vanilla Doom wall colormap equation
//
//===========================================================================
float R_WallColormap(float lightnum, float z, vec3 normal)
{
// R_ScaleFromGlobalAngle calculation
float projection = 160.0; // projection depends on SCREENBLOCKS!! 160 is the fullscreen value
vec2 line_v1 = pixelpos.xz; // in vanilla this is the first curline vertex
vec2 line_normal = normal.xz;
float texscale = projection * clamp(dot(normalize(uCameraPos.xz - line_v1), line_normal), 0.0, 1.0) / z;
float lightz = clamp(16.0 * texscale, 0.0, 47.0);
// scalelight[lightnum][lightz] lookup
float startmap = (15.0 - lightnum) * 4.0;
return startmap - lightz * 0.5;
}
//===========================================================================
//
// Vanilla Doom plane colormap equation
//
//===========================================================================
float R_PlaneColormap(float lightnum, float z)
{
float lightz = clamp(z / 16.0f, 0.0, 127.0);
// zlight[lightnum][lightz] lookup
float startmap = (15.0 - lightnum) * 4.0;
float scale = 160.0 / (lightz + 1.0);
return startmap - scale * 0.5;
}
//===========================================================================
//
// zdoom colormap equation
//
//===========================================================================
float R_ZDoomColormap(float light, float z)
{
float L = light * 255.0;
float vis = min(uGlobVis / z, 24.0 / 32.0);
float shade = 2.0 - (L + 12.0) / 128.0;
float lightscale = shade - vis;
return lightscale * 31.0;
}
float R_DoomColormap(float light, float z)
{
if ((uPalLightLevels >> 16) == 16) // gl_lightmode 16
{
float lightnum = clamp(light * 15.0, 0.0, 15.0);
if (dot(vWorldNormal.xyz, vWorldNormal.xyz) > 0.5)
{
vec3 normal = normalize(vWorldNormal.xyz);
return mix(R_WallColormap(lightnum, z, normal), R_PlaneColormap(lightnum, z), abs(normal.y));
}
else // vWorldNormal is not set on sprites
{
return R_PlaneColormap(lightnum, z);
}
}
else
{
return R_ZDoomColormap(light, z);
}
}
//===========================================================================
//
// Doom software lighting equation
//
//===========================================================================
float R_DoomLightingEquation(float light)
{
// z is the depth in view space, positive going into the screen
float z;
if (((uPalLightLevels >> 8) & 0xff) == 2)
{
z = distance(pixelpos.xyz, uCameraPos.xyz);
}
else
{
z = pixelpos.w;
}
if ((uPalLightLevels >> 16) == 5) // gl_lightmode 5: Build software lighting emulation.
{
// This is a lot more primitive than Doom's lighting...
float numShades = float(uPalLightLevels & 255);
float curshade = (1.0 - light) * (numShades - 1.0);
float visibility = max(uGlobVis * uLightFactor * z, 0.0);
float shade = clamp((curshade + visibility), 0.0, numShades - 1.0);
return clamp(shade * uLightDist, 0.0, 1.0);
}
float colormap = R_DoomColormap(light, z);
if ((uPalLightLevels & 0xff) != 0)
colormap = floor(colormap) + 0.5;
// Result is the normalized colormap index (0 bright .. 1 dark)
return clamp(colormap, 0.0, 31.0) / 32.0;
}
//===========================================================================
//
// Check if light is in shadow according to its 1D shadow map
//
//===========================================================================
#ifdef SUPPORTS_SHADOWMAPS
float shadowDirToU(vec2 dir)
{
if (abs(dir.y) > abs(dir.x))
{
float x = dir.x / dir.y * 0.125;
if (dir.y >= 0.0)
return 0.125 + x;
else
return (0.50 + 0.125) + x;
}
else
{
float y = dir.y / dir.x * 0.125;
if (dir.x >= 0.0)
return (0.25 + 0.125) - y;
else
return (0.75 + 0.125) - y;
}
}
vec2 shadowUToDir(float u)
{
u *= 4.0;
vec2 raydir;
switch (int(u))
{
case 0: raydir = vec2(u * 2.0 - 1.0, 1.0); break;
case 1: raydir = vec2(1.0, 1.0 - (u - 1.0) * 2.0); break;
case 2: raydir = vec2(1.0 - (u - 2.0) * 2.0, -1.0); break;
case 3: raydir = vec2(-1.0, (u - 3.0) * 2.0 - 1.0); break;
}
return raydir;
}
float sampleShadowmap(vec3 planePoint, float v)
{
float bias = 1.0;
float negD = dot(vWorldNormal.xyz, planePoint);
vec3 ray = planePoint;
vec2 isize = textureSize(ShadowMap, 0);
float scale = float(isize.x) * 0.25;
// Snap to shadow map texel grid
if (abs(ray.z) > abs(ray.x))
{
ray.y = ray.y / abs(ray.z);
ray.x = ray.x / abs(ray.z);
ray.x = (floor((ray.x + 1.0) * 0.5 * scale) + 0.5) / scale * 2.0 - 1.0;
ray.z = sign(ray.z);
}
else
{
ray.y = ray.y / abs(ray.x);
ray.z = ray.z / abs(ray.x);
ray.z = (floor((ray.z + 1.0) * 0.5 * scale) + 0.5) / scale * 2.0 - 1.0;
ray.x = sign(ray.x);
}
float t = negD / dot(vWorldNormal.xyz, ray) - bias;
vec2 dir = ray.xz * t;
float u = shadowDirToU(dir);
float dist2 = dot(dir, dir);
return step(dist2, texture(ShadowMap, vec2(u, v)).x);
}
float sampleShadowmapPCF(vec3 planePoint, float v)
{
float bias = 1.0;
float negD = dot(vWorldNormal.xyz, planePoint);
vec3 ray = planePoint;
if (abs(ray.z) > abs(ray.x))
ray.y = ray.y / abs(ray.z);
else
ray.y = ray.y / abs(ray.x);
vec2 isize = textureSize(ShadowMap, 0);
float scale = float(isize.x);
float texelPos = floor(shadowDirToU(ray.xz) * scale);
float sum = 0.0;
float step_count = uShadowmapFilter;
texelPos -= step_count + 0.5;
for (float x = -step_count; x <= step_count; x++)
{
float u = fract(texelPos / scale);
vec2 dir = shadowUToDir(u);
ray.x = dir.x;
ray.z = dir.y;
float t = negD / dot(vWorldNormal.xyz, ray) - bias;
dir = ray.xz * t;
float dist2 = dot(dir, dir);
sum += step(dist2, texture(ShadowMap, vec2(u, v)).x);
texelPos++;
}
return sum / (uShadowmapFilter * 2.0 + 1.0);
}
float shadowmapAttenuation(vec4 lightpos, float shadowIndex)
{
if (shadowIndex >= 1024.0)
return 1.0; // No shadowmap available for this light
vec3 planePoint = pixelpos.xyz - lightpos.xyz;
planePoint += 0.01; // nudge light position slightly as Doom maps tend to have their lights perfectly aligned with planes
if (dot(planePoint.xz, planePoint.xz) < 1.0)
return 1.0; // Light is too close
float v = (shadowIndex + 0.5) / 1024.0;
if (uShadowmapFilter <= 0)
{
return sampleShadowmap(planePoint, v);
}
else
{
return sampleShadowmapPCF(planePoint, v);
}
}
float shadowAttenuation(vec4 lightpos, float lightcolorA)
{
float shadowIndex = abs(lightcolorA) - 1.0;
return shadowmapAttenuation(lightpos, shadowIndex);
}
#else
float shadowAttenuation(vec4 lightpos, float lightcolorA)
{
return 1.0;
}
#endif
float spotLightAttenuation(vec4 lightpos, vec3 spotdir, float lightCosInnerAngle, float lightCosOuterAngle)
{
vec3 lightDirection = normalize(lightpos.xyz - pixelpos.xyz);
float cosDir = dot(lightDirection, spotdir);
return smoothstep(lightCosOuterAngle, lightCosInnerAngle, cosDir);
}
//===========================================================================
//
// Adjust normal vector according to the normal map
//
//===========================================================================
#if defined(NORMALMAP)
mat3 cotangent_frame(vec3 n, vec3 p, vec2 uv)
{
// get edge vectors of the pixel triangle
vec3 dp1 = dFdx(p);
vec3 dp2 = dFdy(p);
vec2 duv1 = dFdx(uv);
vec2 duv2 = dFdy(uv);
// solve the linear system
vec3 dp2perp = cross(n, dp2); // cross(dp2, n);
vec3 dp1perp = cross(dp1, n); // cross(n, dp1);
vec3 t = dp2perp * duv1.x + dp1perp * duv2.x;
vec3 b = dp2perp * duv1.y + dp1perp * duv2.y;
// construct a scale-invariant frame
float invmax = inversesqrt(max(dot(t,t), dot(b,b)));
return mat3(t * invmax, b * invmax, n);
}
vec3 ApplyNormalMap(vec2 texcoord)
{
#define WITH_NORMALMAP_UNSIGNED
#define WITH_NORMALMAP_GREEN_UP
//#define WITH_NORMALMAP_2CHANNEL
vec3 interpolatedNormal = normalize(vWorldNormal.xyz);
vec3 map = texture(normaltexture, texcoord).xyz;
#if defined(WITH_NORMALMAP_UNSIGNED)
map = map * 255./127. - 128./127.; // Math so "odd" because 0.5 cannot be precisely described in an unsigned format
#endif
#if defined(WITH_NORMALMAP_2CHANNEL)
map.z = sqrt(1 - dot(map.xy, map.xy));
#endif
#if defined(WITH_NORMALMAP_GREEN_UP)
map.y = -map.y;
#endif
mat3 tbn = cotangent_frame(interpolatedNormal, pixelpos.xyz, vTexCoord.st);
vec3 bumpedNormal = normalize(tbn * map);
return bumpedNormal;
}
#else
vec3 ApplyNormalMap(vec2 texcoord)
{
return normalize(vWorldNormal.xyz);
}
#endif
//===========================================================================
//
// Sets the common material properties.
//
//===========================================================================
void SetMaterialProps(inout Material material, vec2 texCoord)
{
#ifdef NPOT_EMULATION
if (uNpotEmulation.y != 0.0)
{
float period = floor(texCoord.t / uNpotEmulation.y);
texCoord.s += uNpotEmulation.x * floor(mod(texCoord.t, uNpotEmulation.y));
texCoord.t = period + mod(texCoord.t, uNpotEmulation.y);
}
#endif
material.Base = getTexel(texCoord.st);
material.Normal = ApplyNormalMap(texCoord.st);
// OpenGL doesn't care, but Vulkan pukes all over the place if these texture samplings are included in no-texture shaders, even though never called.
#ifndef NO_LAYERS
if ((uTextureMode & TEXF_Brightmap) != 0)
material.Bright = desaturate(texture(brighttexture, texCoord.st));
if ((uTextureMode & TEXF_Detailmap) != 0)
{
vec4 Detail = texture(detailtexture, texCoord.st * uDetailParms.xy) * uDetailParms.z;
material.Base.rgb *= Detail.rgb;
}
if ((uTextureMode & TEXF_Glowmap) != 0)
material.Glow = desaturate(texture(glowtexture, texCoord.st));
#endif
}
//===========================================================================
//
// Calculate light
//
// It is important to note that the light color is not desaturated
// due to ZDoom's implementation weirdness. Everything that's added
// on top of it, e.g. dynamic lights and glows are, though, because
// the objects emitting these lights are also.
//
// This is making this a bit more complicated than it needs to
// because we can't just desaturate the final fragment color.
//
//===========================================================================
vec4 getLightColor(Material material, float fogdist, float fogfactor)
{
vec4 color = vColor;
if (uLightLevel >= 0.0)
{
float newlightlevel = 1.0 - R_DoomLightingEquation(uLightLevel);
color.rgb *= newlightlevel;
}
else if (uFogEnabled > 0)
{
// brightening around the player for light mode 2
if (fogdist < uLightDist)
{
color.rgb *= uLightFactor - (fogdist / uLightDist) * (uLightFactor - 1.0);
}
//
// apply light diminishing through fog equation
//
color.rgb = mix(vec3(0.0, 0.0, 0.0), color.rgb, fogfactor);
}
//
// handle glowing walls
//
if (uGlowTopColor.a > 0.0 && glowdist.x < uGlowTopColor.a)
{
color.rgb += desaturate(uGlowTopColor * (1.0 - glowdist.x / uGlowTopColor.a)).rgb;
}
if (uGlowBottomColor.a > 0.0 && glowdist.y < uGlowBottomColor.a)
{
color.rgb += desaturate(uGlowBottomColor * (1.0 - glowdist.y / uGlowBottomColor.a)).rgb;
}
color = min(color, 1.0);
// these cannot be safely applied by the legacy format where the implementation cannot guarantee that the values are set.
#if !defined LEGACY_USER_SHADER && !defined NO_LAYERS
//
// apply glow
//
color.rgb = mix(color.rgb, material.Glow.rgb, material.Glow.a);
//
// apply brightmaps
//
color.rgb = min(color.rgb + material.Bright.rgb, 1.0);
#endif
//
// apply other light manipulation by custom shaders, default is a NOP.
//
color = ProcessLight(material, color);
//
// apply lightmaps
//
if (vLightmap.z >= 0.0)
{
color.rgb += texture(LightMap, vLightmap).rgb;
}
//
// apply dynamic lights
//
return vec4(ProcessMaterialLight(material, color.rgb), material.Base.a * vColor.a);
}
//===========================================================================
//
// Applies colored fog
//
//===========================================================================
vec4 applyFog(vec4 frag, float fogfactor)
{
return vec4(mix(uFogColor.rgb, frag.rgb, fogfactor), frag.a);
}
//===========================================================================
//
// The color of the fragment if it is fully occluded by ambient lighting
//
//===========================================================================
vec3 AmbientOcclusionColor()
{
float fogdist;
float fogfactor;
//
// calculate fog factor
//
if (uFogEnabled == -1)
{
fogdist = max(16.0, pixelpos.w);
}
else
{
fogdist = max(16.0, distance(pixelpos.xyz, uCameraPos.xyz));
}
fogfactor = exp2 (uFogDensity * fogdist);
return mix(uFogColor.rgb, vec3(0.0), fogfactor);
}
//===========================================================================
//
// Main shader routine
//
//===========================================================================
void main()
{
#ifdef NO_CLIPDISTANCE_SUPPORT
if (ClipDistanceA.x < 0 || ClipDistanceA.y < 0 || ClipDistanceA.z < 0 || ClipDistanceA.w < 0 || ClipDistanceB.x < 0) discard;
#endif
#ifndef LEGACY_USER_SHADER
Material material;
material.Base = vec4(0.0);
material.Bright = vec4(0.0);
material.Glow = vec4(0.0);
material.Normal = vec3(0.0);
material.Specular = vec3(0.0);
material.Glossiness = 0.0;
material.SpecularLevel = 0.0;
material.Metallic = 0.0;
material.Roughness = 0.0;
material.AO = 0.0;
SetupMaterial(material);
#else
Material material = ProcessMaterial();
#endif
vec4 frag = material.Base;
#ifndef NO_ALPHATEST
if (frag.a <= uAlphaThreshold) discard;
#endif
if (uFogEnabled != -3) // check for special 2D 'fog' mode.
{
float fogdist = 0.0;
float fogfactor = 0.0;
//
// calculate fog factor
//
if (uFogEnabled != 0)
{
if (uFogEnabled == 1 || uFogEnabled == -1)
{
fogdist = max(16.0, pixelpos.w);
}
else
{
fogdist = max(16.0, distance(pixelpos.xyz, uCameraPos.xyz));
}
fogfactor = exp2 (uFogDensity * fogdist);
}
if ((uTextureMode & 0xffff) != 7)
{
frag = getLightColor(material, fogdist, fogfactor);
//
// colored fog
//
if (uFogEnabled < 0)
{
frag = applyFog(frag, fogfactor);
}
}
else
{
frag = vec4(uFogColor.rgb, (1.0 - fogfactor) * frag.a * 0.75 * vColor.a);
}
}
else // simple 2D (uses the fog color to add a color overlay)
{
if ((uTextureMode & 0xffff) == 7)
{
float gray = grayscale(frag);
vec4 cm = (uObjectColor + gray * (uAddColor - uObjectColor)) * 2;
frag = vec4(clamp(cm.rgb, 0.0, 1.0), frag.a);
}
frag = frag * ProcessLight(material, vColor);
frag.rgb = frag.rgb + uFogColor.rgb;
}
FragColor = frag;
#ifdef GBUFFER_PASS
FragFog = vec4(AmbientOcclusionColor(), 1.0);
FragNormal = vec4(vEyeNormal.xyz * 0.5 + 0.5, 1.0);
#endif
}