- Removed a couple warnings about implicit vector truncation reported by fxc. (Apparently, it

also decided to compile some other shaders slightly differently, too.)
- Fixed: The InGameColormap had been designed without taking alpha into consideration.
  As the least likely parameter to be used, desaturation has been moved into a constant
  register to make room for the alpha parameter to live in the vertex's color value.

SVN r3208 (trunk)

src/win32/fb_d3d9.cpp

@@ -3293,8 +3293,8 @@ void D3DFB::FillSimplePoly(FTexture *texture, FVector2 *points, int npoints,
 				quad->Flags |= BQF_Desaturated;
 			}
 			quad->ShaderNum = BQS_InGameColormap;
-			color0 = D3DCOLOR_ARGB(colormap->Desaturate,
+			quad->Desat = colormap->Desaturate;
-				colormap->Color.r, colormap->Color.g, colormap->Color.b);
+			color0 = D3DCOLOR_ARGB(255, colormap->Color.r, colormap->Color.g, colormap->Color.b);
 			double fadelevel = clamp(shade / (NUMCOLORMAPS * 65536.0), 0.0, 1.0);
 			color1 = D3DCOLOR_ARGB(DWORD((1 - fadelevel) * 255),
 				DWORD(colormap->Fade.r * fadelevel),
@@ -3532,6 +3532,10 @@ void D3DFB::EndQuadBatch()
 			{
 				break;
 			}
+			if (quad->ShaderNum == BQS_InGameColormap && (quad->Flags & BQF_Desaturated) && quad->Desat != q2->Desat)
+			{
+				break;
+			}
 			indexpos += q2->NumTris * 3;
 			vertpos += q2->NumVerts;
 		}
@@ -3593,6 +3597,10 @@ void D3DFB::EndQuadBatch()
 			select = !!(quad->Flags & BQF_Desaturated);
 			select |= !!(quad->Flags & BQF_InvertSource) << 1;
 			select |= !!(quad->Flags & BQF_Paletted) << 2;
+			if (quad->Flags & BQF_Desaturated)
+			{
+				SetConstant(PSCONST_Desaturation, quad->Desat / 255.f, (255 - quad->Desat) / 255.f, 0, 0);
+			}
 			SetPixelShader(Shaders[SHADER_InGameColormap + select]);
 		}
 
@@ -3704,6 +3712,7 @@ bool D3DFB::SetStyle(D3DTex *tex, DrawParms &parms, D3DCOLOR &color0, D3DCOLOR &
 	stencilling = false;
 	quad.Palette = NULL;
 	quad.Flags = 0;
+	quad.Desat = 0;
 
 	switch (style.BlendOp)
 	{
@@ -3812,7 +3821,8 @@ bool D3DFB::SetStyle(D3DTex *tex, DrawParms &parms, D3DCOLOR &color0, D3DCOLOR &
 				quad.Flags |= BQF_Desaturated;
 			}
 			quad.ShaderNum = BQS_InGameColormap;
-			color0 = D3DCOLOR_ARGB(parms.colormapstyle->Desaturate,
+			quad.Desat = parms.colormapstyle->Desaturate;
+			color0 = D3DCOLOR_ARGB(color1 >> 24,
 				parms.colormapstyle->Color.r,
 				parms.colormapstyle->Color.g,
 				parms.colormapstyle->Color.b);

src/win32/win32iface.h

@@ -296,6 +296,7 @@ private:
 			};
 			DWORD Group1;
 		};
+		BYTE Desat;
 		D3DPal *Palette;
 		IDirect3DTexture9 *Texture;
 		WORD NumVerts;		// Number of _unique_ vertices used by this set.
@@ -304,6 +305,7 @@ private:
 
 	enum
 	{
+		PSCONST_Desaturation = 1,
 		PSCONST_PaletteMod = 2,
 		PSCONST_Weights = 6,
 		PSCONST_Gamma = 7,

wadsrc/static/shaders/d3d/shaders.ps

@@ -6,6 +6,7 @@ sampler1D Gamma2 : register(s3);
 sampler1D Gamma3 : register(s4);
 #endif
 
+float4 Desaturation : register(c1);	// { Desat, 1 - Desat }
 float4 PaletteMod : register(c2);
 float4 Weights : register(c6);		// RGB->Gray weighting { 77/256.0, 143/256.0, 37/256.0, 1 }
 float4 Gamma : register(c7);
@@ -70,15 +71,16 @@ float4 SpecialColormap(float2 tex_coord : TEXCOORD0, float4 start : COLOR0, floa
 	float4 range = end - start;
 	// We can't store values greater than 1.0 in a color register, so we multiply
 	// the final result by 2 and expect the caller to divide the start and end by 2.
-	color.rgb = 2 * (start + Grayscale(color) * range);
+	color.rgb = 2 * (start + Grayscale(color) * range).rgb;
 	// Duplicate alpha semantics of NormalColor.
 	color.a = start.a + color.a * end.a;
 	return color;
 }
 
 // In-game colormap effect: fade to a particular color and multiply by another, with 
-// optional desaturation of the original color. Desaturation is packed into color.a.
+// optional desaturation of the original color. Desaturation is stored in c1.
 // Fade level is packed int fade.a. Fade.rgb has been premultiplied by alpha.
+// Overall alpha is in color.a.
 float4 InGameColormap(float2 tex_coord : TEXCOORD0, float4 color : COLOR0, float4 fade : COLOR1) : COLOR
 {
 	float4 rgb = SampleTexture(tex_coord);
@@ -86,17 +88,16 @@ float4 InGameColormap(float2 tex_coord : TEXCOORD0, float4 color : COLOR0, float
 	// Desaturate
 #if DESAT
 	float3 intensity;
-	float invdesat;
+	intensity.rgb = Grayscale(rgb) * Desaturation.x;
-	intensity.rgb = Grayscale(rgb) * color.a;
+	rgb.rgb = intensity.rgb + rgb.rgb * Desaturation.y;
-	invdesat = Weights.w - color.a;
-	rgb.rgb = intensity + rgb * invdesat;
 #endif
 
 	// Fade
 	rgb.rgb = rgb.rgb * fade.aaa + fade.rgb;
 
-	// Shade
+	// Shade and Alpha
-	rgb.rgb = rgb.rgb * color.rgb;
+	rgb = rgb * color;
+
 	return rgb;
 }