- implemented desaturation for the legacy GL renderer's 2D.

A lot of work that's only needed to be able to use the hardware renderer's 2D code with the software renderer.

src/gl/compatibility/gl_20.cpp

@@ -50,6 +50,7 @@
 #include "gl/scene/gl_drawinfo.h"
 #include "gl/scene/gl_scenedrawer.h"
 #include "gl/data/gl_vertexbuffer.h"
+#include "gl/textures/gl_translate.h"
 
 
 CVAR(Bool, gl_lights_additive, false, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
@@ -896,3 +897,77 @@ void GLSceneDrawer::RenderMultipassStuff()
 
 }
 
+
+//==========================================================================
+//
+// Draws a color overlay for Legacy OpenGL
+//
+//==========================================================================
+
+void LegacyColorOverlay(F2DDrawer *drawer, F2DDrawer::RenderCommand & cmd)
+{
+	if (cmd.mDrawMode == F2DDrawer::DTM_Opaque || cmd.mDrawMode == F2DDrawer::DTM_InvertOpaque)
+	{
+		gl_RenderState.EnableTexture(false);
+	}
+	else
+	{
+		gl_RenderState.SetTextureMode(TM_MASK);
+	}
+	// Draw this the old fashioned way, there really is no point setting up a buffer for it.
+	glBegin(GL_TRIANGLES);
+	for (int i = 0; i < cmd.mIndexCount; i++)
+	{
+		auto &vertex = drawer->mVertices[drawer->mIndices[i + cmd.mIndexIndex]];
+		glColor4ub(cmd.mColor1.r, cmd.mColor1.g, cmd.mColor1.b, cmd.mColor1.a);
+		glTexCoord2f(vertex.u, vertex.v);
+		glVertex3f(vertex.x, vertex.y, vertex.z);
+	}
+	glEnd();
+}
+
+//==========================================================================
+//
+// Desaturation with translations.
+// Let's keep this fallback crap strictly out of the main code, 
+// including the data it creates!
+//
+//==========================================================================
+
+struct DesaturatedTranslations
+{
+	FRemapTable *tables[32] = { nullptr };
+};
+
+static TMap<FRemapTable *, DesaturatedTranslations> DesaturatedTranslationTable;
+static TDeletingArray<FRemapTable *> DesaturatedRemaps;	// This is only here to delete the remap tables without infesting other code.
+
+
+int LegacyDesaturation(F2DDrawer::RenderCommand &cmd)
+{
+	int desat = cmd.mDesaturate / 8;
+	if (desat <= 0 || desat >= 32) return -1;
+	if(cmd.mTranslation == nullptr) return desat - 1 + STRange_Desaturate;
+	// Now it gets nasty. We got a combination of translation and desaturation.
+
+	// The easy case: It was already done.
+	auto find = DesaturatedTranslationTable.CheckKey(cmd.mTranslation);
+	if (find != nullptr && find->tables[desat] != nullptr) return GLTranslationPalette::GetInternalTranslation(find->tables[desat]);
+
+	// To handle this case for the legacy renderer a desaturated variant of the translation needs to be built.
+	auto newremap = new FRemapTable(*cmd.mTranslation);
+	DesaturatedRemaps.Push(newremap);
+	for (int i = 0; i < newremap->NumEntries; i++)
+	{
+		// This is used for true color texture creation, so the remap table can be left alone.
+		auto &p = newremap->Palette[i];
+		auto gray = p.Luminance();
+
+		p.r = (p.r * (31 - desat) + gray * (1 + desat)) / 32;
+		p.g = (p.g * (31 - desat) + gray * (1 + desat)) / 32;
+		p.b = (p.b * (31 - desat) + gray * (1 + desat)) / 32;
+	}
+	auto &tbl = DesaturatedTranslationTable[cmd.mTranslation];
+	tbl.tables[desat] = newremap;
+	return GLTranslationPalette::GetInternalTranslation(newremap);
+}

src/gl/renderer/gl_renderer.cpp

@@ -484,8 +484,13 @@ public:
 //
 //===========================================================================
 
+void LegacyColorOverlay(F2DDrawer *drawer, F2DDrawer::RenderCommand & cmd);
+int LegacyDesaturation(F2DDrawer::RenderCommand &cmd);
+
 void FGLRenderer::Draw2D(F2DDrawer *drawer)
 {
+
+
 	auto &vertices = drawer->mVertices;
 	auto &indices = drawer->mIndices;
 	auto &commands = drawer->mData;
@@ -500,10 +505,12 @@ void FGLRenderer::Draw2D(F2DDrawer *drawer)
 	auto vb = new F2DVertexBuffer;
 	vb->UploadData(&vertices[0], vertices.Size(), &indices[0], indices.Size());
 	gl_RenderState.SetVertexBuffer(vb);
-
+	gl_RenderState.SetFixedColormap(CM_DEFAULT);
 
 	for(auto &cmd : commands)
 	{
+
+		int gltrans = -1;
 		int tm, sb, db, be;
 		// The texture mode being returned here cannot be used, because the higher level code 
 		// already manipulated the data so that some cases will not be handled correctly.
@@ -533,65 +540,90 @@ void FGLRenderer::Draw2D(F2DDrawer *drawer)
 		}
 		else glDisable(GL_SCISSOR_TEST);
 
-		// Legacy mode cannot replicate the more complex effects
-		if (gl.legacyMode)
+		if (cmd.mSpecialColormap != nullptr)
 		{
-			gl_RenderState.SetFixedColormap(CM_DEFAULT);
-		}
-		else
-		{
-			gl_RenderState.Set2DColors(cmd.mColor1, cmd.mColor2);
-			if (cmd.mFlags & F2DDrawer::DTF_SpecialColormap)
-			{
-				gl_RenderState.SetFixedColormap(CM_SPECIAL2D);
+			auto index = cmd.mSpecialColormap - &SpecialColormaps[0];
+			if (index < 0 || (unsigned)index >= SpecialColormaps.Size()) index = 0;	// if it isn't in the table FBitmap cannot use it. Shouldn't happen anyway.
+			if (!gl.legacyMode)
+			{ 
+				gl_RenderState.SetFixedColormap(CM_FIRSTSPECIALCOLORMAP + int(index));
 			}
 			else
 			{
-				gl_RenderState.SetFixedColormap(CM_PLAIN2D);
+				// map the special colormap to a translation for the legacy renderer.
+				// This only gets used on the software renderer's weapon sprite.
+				gltrans = STRange_Specialcolormap + index;
 			}
 		}
+		else
+		{
+			if (!gl.legacyMode)
+			{
+				gl_RenderState.Set2DOverlayColor(cmd.mColor1);
+				gl_RenderState.SetFixedColormap(CM_PLAIN2D);
+			}
+			else if (cmd.mDesaturate > 0)
+			{
+				gltrans = LegacyDesaturation(cmd);
+			}
+		}
+
 		gl_RenderState.SetColor(1, 1, 1, 1, cmd.mDesaturate); 
 
 		gl_RenderState.AlphaFunc(GL_GEQUAL, 0.f);
 
+		if (cmd.mTexture != nullptr)
+		{
+			auto mat = FMaterial::ValidateTexture(cmd.mTexture, false);
+			if (mat == nullptr) continue;
+
+			if (gltrans == -1) gltrans = GLTranslationPalette::GetInternalTranslation(cmd.mTranslation);
+			gl_RenderState.SetMaterial(mat, cmd.mFlags & F2DDrawer::DTF_Wrap ? CLAMP_NONE : CLAMP_XY_NOMIP, -gltrans, -1, cmd.mDrawMode == F2DDrawer::DTM_AlphaTexture);
+			gl_RenderState.EnableTexture(true);
+
+			// Canvas textures are stored upside down
+			if (cmd.mTexture->bHasCanvas)
+			{
+				gl_RenderState.mTextureMatrix.loadIdentity();
+				gl_RenderState.mTextureMatrix.scale(1.f, -1.f, 1.f);
+				gl_RenderState.mTextureMatrix.translate(0.f, 1.f, 0.0f);
+				gl_RenderState.EnableTextureMatrix(true);
+			}
+		}
+		else
+		{
+			gl_RenderState.EnableTexture(false);
+		}
+		gl_RenderState.Apply();
+
 		switch (cmd.mType)
 		{
 		case F2DDrawer::DrawTypeTriangles:
-			if (cmd.mTexture != nullptr)
-			{
-				auto mat = FMaterial::ValidateTexture(cmd.mTexture, false);
-				if (mat == nullptr) continue;
-				int gltrans = GLTranslationPalette::GetInternalTranslation(cmd.mTranslation);
-				gl_RenderState.SetMaterial(mat, cmd.mFlags & F2DDrawer::DTF_Wrap ? CLAMP_NONE : CLAMP_XY_NOMIP, -gltrans, -1, false);
-				gl_RenderState.EnableTexture(true);
-			}
-			else
-			{
-				gl_RenderState.EnableTexture(false);
-			}
-			gl_RenderState.Apply();
 			glDrawElements(GL_TRIANGLES, cmd.mIndexCount, GL_UNSIGNED_INT, (const void *)(cmd.mIndexIndex * sizeof(unsigned int)));
+			if (gl.legacyMode && cmd.mColor1 != 0)
+			{
+				// Draw the overlay as a separate operation.
+				LegacyColorOverlay(drawer, cmd);
+			}
 			break;
 
 		case F2DDrawer::DrawTypeLines:
-			gl_RenderState.EnableTexture(false);
-			gl_RenderState.Apply();
 			glDrawArrays(GL_LINES, cmd.mVertIndex, cmd.mVertCount);
 			break;
 
 		case F2DDrawer::DrawTypePoints:
-			gl_RenderState.EnableTexture(false);
-			gl_RenderState.Apply();
 			glDrawArrays(GL_POINTS, cmd.mVertIndex, cmd.mVertCount);
 			break;
 
 		}
+		gl_RenderState.EnableTextureMatrix(false);
 	}
 	glDisable(GL_SCISSOR_TEST);
 
 	gl_RenderState.SetVertexBuffer(GLRenderer->mVBO);
 	gl_RenderState.EnableTexture(true);
 	gl_RenderState.SetTextureMode(TM_MODULATE);
+	gl_RenderState.SetFixedColormap(CM_DEFAULT);
 	gl_RenderState.ResetColor();
 	gl_RenderState.Apply();
 	delete vb;

src/gl/renderer/gl_renderstate.h

@@ -145,7 +145,7 @@ public:
 
 	void SetMaterial(FMaterial *mat, int clampmode, int translation, int overrideshader, bool alphatexture)
 	{
-		// alpha textures need special treatment in the legacy renderer because without shaders they need a different texture.
+		// alpha textures need special treatment in the legacy renderer because without shaders they need a different texture. This will also override all other translations.
 		if (alphatexture &&  gl.legacyMode) translation = INT_MAX;
 		
 		if (mat->tex->bHasCanvas)
@@ -393,13 +393,11 @@ public:
 		mObjectColor2 = pe;
 	}
 
-	void Set2DColors(PalEntry pe, PalEntry pe2)
+	void Set2DOverlayColor(PalEntry pe)
 	{
 		m2DColors[0] = pe;
-		m2DColors[1] = pe2;
 	}
 
-
 	void SetSpecular(float glossiness, float specularLevel)
 	{
 		mGlossiness = glossiness;

src/gl/system/gl_swframebuffer.cpp

@@ -1169,6 +1169,7 @@ void OpenGLSWFrameBuffer::Update()
 		if (InScene)
 		{
 			DrawRateStuff();
+			Draw2D();
 			EndBatch();		// Make sure all batched primitives are drawn.
 			Flip();
 		}

src/gl/system/gl_wipe.cpp

@@ -169,9 +169,6 @@ bool OpenGLFrameBuffer::WipeStartScreen(int type)
 
 void OpenGLFrameBuffer::WipeEndScreen()
 {
-	screen->Draw2D();
-	screen->Clear2D();
-
 	const auto &viewport = GLRenderer->mScreenViewport;
 	wipeendscreen = new FHardwareTexture(viewport.width, viewport.height, true);
 	wipeendscreen->CreateTexture(NULL, viewport.width, viewport.height, 0, false, 0, "WipeEndScreen");

src/gl/textures/gl_material.cpp

@@ -213,9 +213,20 @@ unsigned char * FGLTexture::CreateTexBuffer(int translation, int & w, int & h, F
 
 	FBitmap bmp(buffer, W*4, W, H);
 
-	if (translation <= 0 || alphatrans)
+	if (translation <= 0 || alphatrans || translation >= STRange_Min)
 	{
-		int trans = tex->CopyTrueColorPixels(&bmp, exx, exx);
+		// Allow creation of desaturated or special-colormapped textures for the legacy renderer.
+		FCopyInfo inf = { OP_COPY, BLEND_NONE, {0}, 0, 0 };
+		if (translation >= STRange_Desaturate && translation < STRange_Desaturate+31)	// there are 31 ranges of desaturations available
+		{
+			inf.blend = (EBlend)(BLEND_DESATURATE1 + translation - STRange_Desaturate);
+		}
+		else if (translation >= STRange_Specialcolormap && translation < STRange_Specialcolormap + SpecialColormaps.Size())
+		{
+			inf.blend = (EBlend)(BLEND_SPECIALCOLORMAP1 + translation - STRange_Specialcolormap);
+		}
+
+		int trans = tex->CopyTrueColorPixels(&bmp, exx, exx, 0, translation >= STRange_Min? &inf : nullptr);
 		tex->CheckTrans(buffer, W*H, trans);
 		isTransparent = tex->gl_info.mIsTransparent;
 		if (bIsTransparent == -1) bIsTransparent = isTransparent;

src/gl/textures/gl_material.h

@@ -50,6 +50,12 @@ struct FTexCoordInfo
 //===========================================================================
 class FMaterial;
 
+enum ESpecialTranslations : uint32_t
+{
+	STRange_Min = 0x10000000,
+	STRange_Desaturate = 0x10000000,
+	STRange_Specialcolormap = 0x20000000,
+};
 
 class FGLTexture
 {

src/v_2ddrawer.cpp

@@ -180,22 +180,8 @@ bool F2DDrawer::SetStyle(FTexture *tex, DrawParms &parms, PalEntry &vertexcolor,
 
 		if (parms.specialcolormap != nullptr)
 		{ // Emulate an invulnerability or similar colormap.
-			float *start, *end;
-			start = parms.specialcolormap->ColorizeStart;
-			end = parms.specialcolormap->ColorizeEnd;
-			if (quad.mDrawMode == DTM_Invert)
-			{
-				quad.mDrawMode = DTM_Normal;
-				std::swap(start, end);
-			}
-			quad.mFlags |= DTF_SpecialColormap;
-			// SpecialColormap uses the two color uniforms to set its ramp.
-			quad.mColor1.r = (uint8_t)(start[0] * (255 / 2));
-			quad.mColor1.g = (uint8_t)(start[1] * (255 / 2));
-			quad.mColor1.b = (uint8_t)(start[2] * (255 / 2));
-			quad.mColor2.r = (uint8_t)(end[0] * (255 / 2));
-			quad.mColor2.g = (uint8_t)(end[1] * (255 / 2));
-			quad.mColor2.b = (uint8_t)(end[2] * (255 / 2));
+			quad.mSpecialColormap = parms.specialcolormap;
+			quad.mColor1 = 0;	// this disables the color overlay.
 		}
 		quad.mDesaturate = parms.desaturate;
 	}

src/v_2ddrawer.h

@@ -34,7 +34,7 @@ public:
 	{
 		DTF_Wrap = 1,
 		DTF_Scissor = 2,
-		DTF_SpecialColormap = 4,
+		//DTF_SpecialColormap = 4,
 	};
 
 
@@ -77,10 +77,11 @@ public:
 
 		FTexture *mTexture;
 		FRemapTable *mTranslation;
+		FSpecialColormap *mSpecialColormap;
 		int mScissor[4];
 		int mDesaturate;
 		FRenderStyle mRenderStyle;
-		PalEntry mColor1, mColor2;	// Can either be the overlay color or the special colormap ramp. Both features can not be combined.
+		PalEntry mColor1;	// Overlay color
 		ETextureDrawMode mDrawMode;
 		uint8_t mFlags;
 
@@ -95,13 +96,13 @@ public:
 			return mTexture == other.mTexture &&
 				mType == other.mType &&
 				mTranslation == other.mTranslation &&
+				mSpecialColormap == other.mSpecialColormap &&
 				!memcmp(mScissor, other.mScissor, sizeof(mScissor)) &&
 				mDesaturate == other.mDesaturate &&
 				mRenderStyle == other.mRenderStyle &&
 				mDrawMode == other.mDrawMode &&
 				mFlags == other.mFlags &&
-				mColor1 == other.mColor1 &&
-				mColor2 == other.mColor2;
+				mColor1 == other.mColor1;
 
 		}
 	};

src/win32/fb_d3d9.cpp

@@ -905,13 +905,14 @@ bool D3DFB::IsFullscreen ()
 //
 //==========================================================================
 
-void D3DFB::Update ()
+void D3DFB::Update()
 {
 	if (In2D == 3)
 	{
 		if (InScene)
 		{
 			DrawRateStuff();
+			Draw2D();
 			EndQuadBatch();		// Make sure all batched primitives are drawn.
 			In2D = 0;
 			Flip();
@@ -1873,11 +1874,11 @@ void D3DFB::Draw2D()
 			auto textype = cmd.mTexture->GetFormat();	// This never returns TEX_Gray.
 			if (cmd.mTranslation) textype = TEX_Pal;	// Translation requires a paletted texture, regardless of the source format.
 
-			if (cmd.mFlags & F2DDrawer::DTF_SpecialColormap)
+			if (cmd.mSpecialColormap != nullptr)
 			{
 				index = textype == TEX_Pal ? SHADER_SpecialColormapPal : SHADER_SpecialColormap;
-				SetConstant(PSCONST_Color1, cmd.mColor1.r / 510.f, cmd.mColor1.g / 510.f, cmd.mColor1.b / 510.f, 0);
-				SetConstant(PSCONST_Color2, cmd.mColor1.r / 510.f, cmd.mColor1.g / 510.f, cmd.mColor1.b / 510.f, 0);
+				SetConstant(PSCONST_Color1, cmd.mSpecialColormap->ColorizeStart[0] / 2.f, cmd.mSpecialColormap->ColorizeStart[1] / 2.f, cmd.mSpecialColormap->ColorizeStart[2] / 2.f, 0);
+				SetConstant(PSCONST_Color2, cmd.mSpecialColormap->ColorizeEnd[0] / 2.f, cmd.mSpecialColormap->ColorizeEnd[1] / 2.f, cmd.mSpecialColormap->ColorizeEnd[2] / 2.f, 0);
 			}
 			else
 			{
@@ -1950,7 +1951,7 @@ void D3DFB::Draw2D()
 		switch (cmd.mType)
 		{
 		case F2DDrawer::DrawTypeTriangles:
-			D3DDevice->DrawPrimitive(D3DPT_TRIANGLELIST, cmd.mVertIndex, cmd.mVertCount);
+			D3DDevice->DrawIndexedPrimitive(D3DPT_TRIANGLELIST, 0, cmd.mVertIndex, cmd.mVertCount, cmd.mIndexIndex, cmd.mIndexCount / 3);
 			break;
 
 		case F2DDrawer::DrawTypeLines:

wadsrc/static/shaders/glsl/main.fp

@@ -25,7 +25,7 @@ vec3 ProcessMaterial(vec3 material, vec3 color);
 
 float grayscale(vec4 color)
 {
-	return dot(color.rgb, vec3(0.4, 0.56, 0.14);
+	return dot(color.rgb, vec3(0.4, 0.56, 0.14));
 }
 
 //===========================================================================