- added a fog layer when drawing sprites with render style reverse subtract.

src/gl/renderer/gl_colormap.h

@@ -24,6 +24,7 @@ enum EColorManipulation
 	// These are not to be passed to the texture manager
 	CM_LITE	= 0x20000000,		// special values to handle these items without excessive hacking
 	CM_TORCH= 0x20000010,		// These are not real color manipulations
+	CM_FOGLAYER= 0x20000020,	// Sprite shaped fog layer - this is only used as a parameter to FMaterial::BindPatch
 };
 
 #define CM_MAXCOLORMAP int(CM_FIRSTSPECIALCOLORMAP + SpecialColormaps.Size())

src/gl/renderer/gl_renderstate.h

@@ -211,6 +211,11 @@ public:
 		mLightData = lightdata;	// caution: the data must be preserved by the caller until the 'apply' call!
 	}
 
+	void SetFixedColormap(int cm)
+	{
+		mColormapState = cm;
+	}
+
 	PalEntry GetFogColor() const
 	{
 		return mFogColor;

src/gl/scene/gl_sprite.cpp

@@ -101,6 +101,8 @@ void gl_SetRenderStyle(FRenderStyle style, bool drawopaque, bool allowcolorblend
 	gl_RenderState.SetTextureMode(tm);
 }
 
+CVAR(Bool, gl_nolayer, false, 0)
+
 //==========================================================================
 //
 // 
@@ -117,6 +119,7 @@ void GLSprite::Draw(int pass)
 
 
 	bool additivefog = false;
+	bool foglayer = false;
 	int rel = getExtraLight();
 
 	if (pass==GLPASS_TRANSLUCENT)
@@ -208,8 +211,36 @@ void GLSprite::Draw(int pass)
 		Colormap.FadeColor = Colormap.FadeColor.InverseColor();
 		additivefog=false;
 	}
+	if (RenderStyle.BlendOp == STYLEOP_RevSub || RenderStyle.BlendOp == STYLEOP_Sub)
+	{
+		if (!modelframe)
+		{
+			// non-black fog with subtractive style needs special treatment
+			if (!gl_isBlack(Colormap.FadeColor))
+			{
+				if (gl.shadermodel >= 4 && !gl_nolayer)
+				{
+					// fog layer only works on modern hardware. 
+					foglayer = true;
+					// Due to the two-layer approach we need to force an alpha test that lets everything pass
+					gl_RenderState.AlphaFunc(GL_GREATER, 0);
+				}
+				else
+				{
+					// this at least partially handles the fog issue
+					Colormap.FadeColor = Colormap.FadeColor.InverseColor();
+				}
+			}
+		}
+		else RenderStyle.BlendOp = STYLEOP_Fuzz;	// subtractive with models is not going to work.
+	}
 
-	gl_SetFog(foglevel, rel, &Colormap, additivefog);
+	if (!foglayer) gl_SetFog(foglevel, rel, &Colormap, additivefog);
+	else
+	{
+		gl_RenderState.EnableFog(false);
+		gl_RenderState.SetFog(0, 0);
+	}
 
 	if (gltexture) gltexture->BindPatch(Colormap.colormap, translation, OverrideShader);
 	else if (!modelframe) gl_RenderState.EnableTexture(false);
@@ -221,27 +252,67 @@ void GLSprite::Draw(int pass)
 		                                   //&& GLRenderer->mViewActor != NULL
 		                                   && (gl_billboard_mode == 1 || (actor && actor->renderflags & RF_FORCEXYBILLBOARD ))) );
 		gl_RenderState.Apply();
-		glBegin(GL_TRIANGLE_STRIP);
+
-		if ( drawWithXYBillboard )
+		Vector v1;
+		Vector v2;
+		Vector v3;
+		Vector v4;
+
+		if (drawWithXYBillboard)
 		{
 			// Rotate the sprite about the vector starting at the center of the sprite
 			// triangle strip and with direction orthogonal to where the player is looking
 			// in the x/y plane.
-			float xcenter = (x1+x2)*0.5;
+			float xcenter = (x1 + x2)*0.5;
-			float ycenter = (y1+y2)*0.5;
+			float ycenter = (y1 + y2)*0.5;
-			float zcenter = (z1+z2)*0.5;
+			float zcenter = (z1 + z2)*0.5;
 			float angleRad = DEG2RAD(270. - float(GLRenderer->mAngles.Yaw));
 
 			Matrix3x4 mat;
 			mat.MakeIdentity();
-			mat.Translate( xcenter, zcenter, ycenter);
+			mat.Translate(xcenter, zcenter, ycenter);
 			mat.Rotate(-sin(angleRad), 0, cos(angleRad), -GLRenderer->mAngles.Pitch);
-			mat.Translate( -xcenter, -zcenter, -ycenter);
+			mat.Translate(-xcenter, -zcenter, -ycenter);
-			Vector v1 = mat * Vector(x1,z1,y1);
+			v1 = mat * Vector(x1, z1, y1);
-			Vector v2 = mat * Vector(x2,z1,y2);
+			v2 = mat * Vector(x2, z1, y2);
-			Vector v3 = mat * Vector(x1,z2,y1);
+			v3 = mat * Vector(x1, z2, y1);
-			Vector v4 = mat * Vector(x2,z2,y2);
+			v4 = mat * Vector(x2, z2, y2);
+		}
+		else
+		{
+			v1 = Vector(x1, z1, y1);
+			v2 = Vector(x2, z1, y2);
+			v3 = Vector(x1, z2, y1);
+			v4 = Vector(x2, z2, y2);
+		}
 
+		glBegin(GL_TRIANGLE_STRIP);
+		if (gltexture)
+		{
+			glTexCoord2f(ul, vt); glVertex3fv(&v1[0]);
+			glTexCoord2f(ur, vt); glVertex3fv(&v2[0]);
+			glTexCoord2f(ul, vb); glVertex3fv(&v3[0]);
+			glTexCoord2f(ur, vb); glVertex3fv(&v4[0]);
+		}
+		else	// Particle
+		{
+			glVertex3fv(&v1[0]);
+			glVertex3fv(&v2[0]);
+			glVertex3fv(&v3[0]);
+			glVertex3fv(&v4[0]);
+		}
+
+		glEnd();
+
+		if (foglayer)
+		{
+			// If we get here we know that we have colored fog and no fixed colormap.
+			gl_SetFog(foglevel, rel, &Colormap, additivefog);
+			gl_RenderState.SetFixedColormap(CM_FOGLAYER);
+			gl_RenderState.BlendEquation(GL_FUNC_ADD);
+			gl_RenderState.Apply();
+
+			glBegin(GL_TRIANGLE_STRIP);
 			if (gltexture)
 			{
 				glTexCoord2f(ul, vt); glVertex3fv(&v1[0]);
@@ -256,26 +327,9 @@ void GLSprite::Draw(int pass)
 				glVertex3fv(&v3[0]);
 				glVertex3fv(&v4[0]);
 			}
+			glEnd();
 
 		}
-		else
-		{
-			if (gltexture)
-			{
-				glTexCoord2f(ul, vt); glVertex3f(x1, z1, y1);
-				glTexCoord2f(ur, vt); glVertex3f(x2, z1, y2);
-				glTexCoord2f(ul, vb); glVertex3f(x1, z2, y1);
-				glTexCoord2f(ur, vb); glVertex3f(x2, z2, y2);
-			}
-			else	// Particle
-			{
-				glVertex3f(x1, z1, y1);
-				glVertex3f(x2, z1, y2);
-				glVertex3f(x1, z2, y1);
-				glVertex3f(x2, z2, y2);
-			}
-		}
-		glEnd();
 	}
 	else
 	{

src/gl/shaders/gl_shader.cpp

@@ -155,7 +155,7 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 		glGetShaderInfoLog(hFragProg, 10000, NULL, buffer);
 		if (*buffer) 
 		{
-			error << "Vertex shader:\n" << buffer << "\n";
+			error << "Fragment shader:\n" << buffer << "\n";
 		}
 
 		glGetProgramInfoLog(hShader, 10000, NULL, buffer);
@@ -289,6 +289,23 @@ FShaderContainer::FShaderContainer(const char *ShaderName, const char *ShaderPat
 		I_Error("Unable to load shader %s:\n%s\n", name.GetChars(), err.GetMessage());
 	}
 
+	name << ShaderName << "::foglayer";
+
+	try
+	{
+		shader_fl = new FShader;
+		if (!shader_fl->Load(name, "shaders/glsl/main.vp", "shaders/glsl/main_foglayer.fp", ShaderPath, "#define NO_GLOW\n"))
+		{
+			delete shader_fl;
+			shader_fl = NULL;
+		}
+	}
+	catch (CRecoverableError &err)
+	{
+		shader_fl = NULL;
+		I_Error("Unable to load shader %s:\n%s\n", name.GetChars(), err.GetMessage());
+	}
+
 	if (gl.shadermodel > 2)
 	{
 		for(int i = 0;i < NUM_SHADERS; i++)
@@ -344,8 +361,9 @@ FShaderContainer::FShaderContainer(const char *ShaderName, const char *ShaderPat
 //==========================================================================
 FShaderContainer::~FShaderContainer()
 {
-	delete shader_cm;
+	if (shader_cm != NULL) delete shader_cm;
-	for(int i = 0;i < NUM_SHADERS; i++)
+	if (shader_fl != NULL) delete shader_fl;
+	for (int i = 0; i < NUM_SHADERS; i++)
 	{
 		if (shader[i] != NULL)
 		{
@@ -365,7 +383,15 @@ FShader *FShaderContainer::Bind(int cm, bool glowing, float Speed, bool lights)
 {
 	FShader *sh=NULL;
 
-	if (cm >= CM_FIRSTSPECIALCOLORMAP && cm < CM_MAXCOLORMAP)
+	if (cm == CM_FOGLAYER)
+	{
+		if (shader_fl)
+		{
+			shader_fl->Bind(Speed);
+		}
+		return shader_fl;
+	}
+	else if (cm >= CM_FIRSTSPECIALCOLORMAP && cm < CM_MAXCOLORMAP)
 	{
 		// these are never used with any kind of lighting or fog
 		sh = shader_cm;

src/gl/shaders/gl_shader.h

@@ -105,12 +105,12 @@ class FShaderContainer
 
 	FShader *shader[NUM_SHADERS];
 	FShader *shader_cm;	// the shader for fullscreen colormaps
+	FShader *shader_fl;	// the shader for the fog layer
 
 public:
 	FShaderContainer(const char *ShaderName, const char *ShaderPath);
 	~FShaderContainer();
 	FShader *Bind(int cm, bool glowing, float Speed, bool lights);
-	
 };
 
 

wadsrc/static/shaders/glsl/main_foglayer.fp

@@ -0,0 +1,59 @@
+uniform int fogenabled;
+uniform vec4 fogcolor;
+uniform vec3 camerapos;
+varying vec4 pixelpos;
+varying vec4 fogparm;
+
+uniform sampler2D tex;
+
+vec4 Process(vec4 color);
+
+//===========================================================================
+//
+// Gets a texel and performs common manipulations
+//
+//===========================================================================
+
+vec4 desaturate(vec4 texel)
+{
+	return texel;
+}
+
+vec4 getTexel(vec2 st)
+{
+	vec4 texel = texture2D(tex, st);
+	return texel;
+}
+
+//===========================================================================
+//
+// Main shader routine
+//
+//===========================================================================
+
+void main()
+{
+	float fogdist;
+	float fogfactor;
+	
+	//
+	// calculate fog factor
+	//
+	#ifndef NO_SM4
+		if (fogenabled == -1) 
+		{
+			fogdist = pixelpos.w;
+		}
+		else 
+		{
+			fogdist = max(16.0, distance(pixelpos.xyz, camerapos));
+		}
+	#else
+		fogdist = pixelpos.w;
+	#endif
+	fogfactor = exp2 (fogparm.z * fogdist);
+	
+	vec4 frag = Process(vec4(1.0,1.0,1.0,1.0));
+	gl_FragColor = vec4(fogcolor.rgb, (1.0 - fogfactor) * frag.a * 0.75);
+}
+