Do not use the shader to handle STYLEF_RedIsAlpha.

Turns out that the name doesn't accurately describe what it does.
It is correct for images that come with their own palette or are true color.
But for images using the game palette it doesn't use the red channel to determine translucency but the palette index! Ugh...

This means it cannot be done with a simple operation in the shader because it won't get a proper source image. The only solution is to create a separate texture.

src/gl/renderer/gl_lightdata.cpp

@@ -144,11 +144,7 @@ void gl_GetRenderStyle(FRenderStyle style, bool drawopaque, bool allowcolorblend
 	int blendequation = renderops[style.BlendOp&15];
 	int texturemode = drawopaque? TM_OPAQUE : TM_MODULATE;
 
-	if (style.Flags & STYLEF_RedIsAlpha)
-	{
-		texturemode = TM_REDTOALPHA;
-	}
-	else if (style.Flags & STYLEF_ColorIsFixed)
+	if (style.Flags & STYLEF_ColorIsFixed)
 	{
 		texturemode = TM_MASK;
 	}

src/gl/renderer/gl_renderer.cpp

@@ -334,17 +334,14 @@ void FGLRenderer::DrawTexture(FTexture *img, DCanvas::DrawParms &parms)
 				if (pal) translation = -pal->GetIndex();
 			}
 		}
-		else 
-		{
-			// This is an alpha texture
-			gl_RenderState.SetTextureMode(TM_REDTOALPHA);
-		}
-		gltex->BindPatch(translation);
+		gl_SetRenderStyle(parms.style, !parms.masked, false);
+		gltex->BindPatch(translation, 0, !!(parms.style.Flags & STYLEF_RedIsAlpha));
 
 		u1 = gltex->GetUL();
 		v1 = gltex->GetVT();
 		u2 = gltex->GetUR();
 		v2 = gltex->GetVB();
+
 	}
 	else
 	{
@@ -391,12 +388,6 @@ void FGLRenderer::DrawTexture(FTexture *img, DCanvas::DrawParms &parms)
 	int space = (static_cast<OpenGLFrameBuffer*>(screen)->GetTrueHeight()-screen->GetHeight())/2;
 	glScissor(parms.lclip, btm - parms.dclip + space, parms.rclip - parms.lclip, parms.dclip - parms.uclip);
 	
-	gl_SetRenderStyle(parms.style, !parms.masked, false);
-	if (img->bHasCanvas)
-	{
-		gl_RenderState.SetTextureMode(TM_OPAQUE);
-	}
-
 	gl_RenderState.SetColor(color);
 	gl_RenderState.EnableAlphaTest(false);
 	gl_RenderState.Apply();

src/gl/renderer/gl_renderer.h

@@ -91,7 +91,7 @@ public:
 	void DrawScene(bool toscreen = false);
 	void DrawBlend(sector_t * viewsector);
 
-	void DrawPSprite (player_t * player,pspdef_t *psp,fixed_t sx, fixed_t sy, bool hudModelStep, int OverrideShader);
+	void DrawPSprite (player_t * player,pspdef_t *psp,fixed_t sx, fixed_t sy, bool hudModelStep, int OverrideShader, bool alphatexture);
 	void DrawPlayerSprites(sector_t * viewsector, bool hudModelStep);
 	void DrawTargeterSprites();
 

src/gl/renderer/gl_renderstate.h

@@ -178,7 +178,6 @@ public:
 	void SetTextureMode(int mode)
 	{
 		mTextureMode = mode;
-		gl.checkTextureMode(mode);
 	}
 
 	void EnableTexture(bool on)

src/gl/scene/gl_decal.cpp

@@ -328,7 +328,7 @@ void GLWall::DrawDecal(DBaseDecal *decal)
 
 
 	gl_SetRenderStyle(decal->RenderStyle, false, false);
-	tex->BindPatch(decal->Translation);
+	tex->BindPatch(decal->Translation, 0, !!(decal->RenderStyle.Flags & STYLEF_RedIsAlpha));
 
 
 	// If srcalpha is one it looks better with a higher alpha threshold

src/gl/scene/gl_sprite.cpp

@@ -221,7 +221,7 @@ void GLSprite::Draw(int pass)
 		gl_RenderState.SetFog(0, 0);
 	}
 
-	if (gltexture) gltexture->BindPatch(translation, OverrideShader);
+	if (gltexture) gltexture->BindPatch(translation, OverrideShader, !!(RenderStyle.Flags & STYLEF_RedIsAlpha));
 	else if (!modelframe) gl_RenderState.EnableTexture(false);
 
 	if (!modelframe)

src/gl/scene/gl_weapon.cpp

@@ -69,7 +69,7 @@ EXTERN_CVAR(Int, gl_fuzztype)
 //
 //==========================================================================
 
-void FGLRenderer::DrawPSprite (player_t * player,pspdef_t *psp,fixed_t sx, fixed_t sy, bool hudModelStep, int OverrideShader)
+void FGLRenderer::DrawPSprite (player_t * player,pspdef_t *psp,fixed_t sx, fixed_t sy, bool hudModelStep, int OverrideShader, bool alphatexture)
 {
 	float			fU1,fV1;
 	float			fU2,fV2;
@@ -95,7 +95,7 @@ void FGLRenderer::DrawPSprite (player_t * player,pspdef_t *psp,fixed_t sx, fixed
 	FMaterial * tex = FMaterial::ValidateTexture(lump, false);
 	if (!tex) return;
 
-	tex->BindPatch(0, OverrideShader);
+	tex->BindPatch(0, OverrideShader, alphatexture);
 
 	int vw = viewwidth;
 	int vh = viewheight;
@@ -386,7 +386,7 @@ void FGLRenderer::DrawPlayerSprites(sector_t * viewsector, bool hudModelStep)
 				gl_SetDynSpriteLight(playermo, NULL);
 				gl_SetColor(statebright[i] ? 255 : lightlevel, 0, &cmc, trans, true);
 			}
-			DrawPSprite (player,psp,psp->sx+ofsx, psp->sy+ofsy, hudModelStep, OverrideShader);
+			DrawPSprite(player, psp, psp->sx + ofsx, psp->sy + ofsy, hudModelStep, OverrideShader, !!(vis.RenderStyle.Flags & STYLEF_RedIsAlpha));
 		}
 	}
 	gl_RenderState.SetObjectColor(0xffffffff);
@@ -420,5 +420,5 @@ void FGLRenderer::DrawTargeterSprites()
 
 	// The Targeter's sprites are always drawn normally.
 	for (i=ps_targetcenter, psp = &player->psprites[ps_targetcenter]; i<NUMPSPRITES; i++,psp++)
-		if (psp->state) DrawPSprite (player,psp,psp->sx, psp->sy, false, 0);
+		if (psp->state) DrawPSprite (player,psp,psp->sx, psp->sy, false, 0, false);
 }

src/gl/system/gl_interface.h

@@ -20,9 +20,6 @@ enum TexMode
 	TM_MASK = 1,		// (1, 1, 1, a)
 	TM_OPAQUE = 2,		// (r, g, b, 1)
 	TM_INVERSE = 3,		// (1-r, 1-g, 1-b, a)
-	TM_REDTOALPHA = 4,	// (1, 1, 1, r)
-
-	// 4 cannot be done natively without shaders and requires special textures.
 };
 
 struct RenderContext
@@ -33,7 +30,6 @@ struct RenderContext
 	float glslversion;
 	int max_texturesize;
 	char * vendorstring;
-	bool needAlphaTexture;
 
 	int MaxLights() const
 	{
@@ -44,11 +40,6 @@ struct RenderContext
 	{
 		return glslversion >= 1.3f;
 	}
-
-	void checkTextureMode(int mode)
-	{
-		if (!hasGLSL()) needAlphaTexture = (mode == TM_REDTOALPHA);
-	}
 };
 
 extern RenderContext gl;

src/gl/textures/gl_material.cpp

@@ -883,7 +883,7 @@ void FMaterial::Bind(int clampmode, int translation, int overrideshader)
 //
 //===========================================================================
 
-void FMaterial::BindPatch(int translation, int overrideshader)
+void FMaterial::BindPatch(int translation, int overrideshader, bool alphatexture)
 {
 	int usebright = false;
 	int shaderindex = overrideshader > 0? overrideshader : mShaderIndex;
@@ -891,7 +891,7 @@ void FMaterial::BindPatch(int translation, int overrideshader)
 
 	int softwarewarp = gl_RenderState.SetupShader(tex->bHasCanvas, shaderindex, tex->gl_info.shaderspeed);
 
-	const FHardwareTexture *glpatch = mBaseLayer->BindPatch(0, translation, softwarewarp, gl.needAlphaTexture);
+	const FHardwareTexture *glpatch = mBaseLayer->BindPatch(0, translation, softwarewarp, alphatexture);
 	// The only multitexture effect usable on sprites is the brightmap.
 	if (glpatch != NULL && shaderindex == 3)
 	{

src/gl/textures/gl_material.h

@@ -135,7 +135,7 @@ public:
 	}
 
 	void Bind(int clamp = 0, int translation = 0, int overrideshader = 0);
-	void BindPatch(int translation = 0, int overrideshader = 0);
+	void BindPatch(int translation = 0, int overrideshader = 0, bool alphatexture = false);
 
 	unsigned char * CreateTexBuffer(int translation, int & w, int & h, bool expand = false, bool allowhires=true) const
 	{

wadsrc/static/shaders/glsl/main.fp

@@ -155,10 +155,6 @@ vec4 getTexel(vec2 st)
 		texel = vec4(1.0-texel.r, 1.0-texel.g, 1.0-texel.b, texel.a);
 		return texel;
 	}
-	else if (texturemode == 4)
-	{
-		texel = vec4(1.0, 1.0, 1.0, texel.r);
-	}
 	else if (texturemode == 2) 
 	{
 		texel.a = 1.0;

wadsrc/static/shaders/glsl/main_colormap.fp

@@ -26,10 +26,6 @@ vec4 getTexel(vec2 st)
 		texel = vec4(1.0-texel.r, 1.0-texel.g, 1.0-texel.b, texel.a);
 		return texel;
 	}
-	else if (texturemode == 4)
-	{
-		texel = vec4(1.0, 1.0, 1.0, texel.r);
-	}
 	else if (texturemode == 2) 
 	{
 		texel.a = 1.0;