Several changes:

Move HWR_SetShaderState calls from level rendering to HWR_Startup and shader cvar changes

Don't run hwr cvar onchange code when opengl is not loaded

Rename screen_palette_tex shader uniform to more generic palette_tex

Fix palette version of HWR_FadeScreenMenuBack not working outside levels

src/hardware/hw_main.c

@@ -6041,9 +6041,6 @@ void HWR_RenderSkyboxView(INT32 viewnumber, player_t *player)
 	//                     Actually it only works on Walls and Planes
 	HWD.pfnSetTransform(&atransform);
 
-	// Reset the shader state.
-	HWR_SetShaderState();
-
 	validcount++;
 
 	if (cv_glbatching.value)
@@ -6256,9 +6253,6 @@ void HWR_RenderPlayerView(INT32 viewnumber, player_t *player)
 	//                     Actually it only works on Walls and Planes
 	HWD.pfnSetTransform(&atransform);
 
-	// Reset the shader state.
-	HWR_SetShaderState();
-
 	ps_numbspcalls.value.i = 0;
 	ps_numpolyobjects.value.i = 0;
 	PS_START_TIMING(ps_bsptime);
@@ -6474,20 +6468,23 @@ static CV_PossibleValue_t glpalettedepth_cons_t[] = {{16, "16 bits"}, {24, "24 b
 consvar_t cv_glpaletterendering = CVAR_INIT ("gr_paletterendering", "Off", CV_SAVE|CV_CALL, CV_OnOff, CV_glpaletterendering_OnChange);
 consvar_t cv_glpalettedepth = CVAR_INIT ("gr_palettedepth", "16 bits", CV_SAVE|CV_CALL, glpalettedepth_cons_t, CV_glpalettedepth_OnChange);
 
+#define ONLY_IF_GL_LOADED if (vid.glstate != VID_GL_LIBRARY_LOADED) return;
+
 static void CV_glfiltermode_OnChange(void)
 {
-	if (rendermode == render_opengl)
+	ONLY_IF_GL_LOADED
 	HWD.pfnSetSpecialState(HWD_SET_TEXTUREFILTERMODE, cv_glfiltermode.value);
 }
 
 static void CV_glanisotropic_OnChange(void)
 {
-	if (rendermode == render_opengl)
+	ONLY_IF_GL_LOADED
 	HWD.pfnSetSpecialState(HWD_SET_TEXTUREANISOTROPICMODE, cv_glanisotropicmode.value);
 }
 
 static void CV_glmodellighting_OnChange(void)
 {
+	ONLY_IF_GL_LOADED
 	// if shaders have been compiled, then they now need to be recompiled.
 	if (gl_shadersavailable)
 		HWR_CompileShaders();
@@ -6495,6 +6492,7 @@ static void CV_glmodellighting_OnChange(void)
 
 static void CV_glpaletterendering_OnChange(void)
 {
+	ONLY_IF_GL_LOADED
 	if (gl_shadersavailable)
 	{
 		HWR_CompileShaders();
@@ -6504,6 +6502,7 @@ static void CV_glpaletterendering_OnChange(void)
 
 static void CV_glpalettedepth_OnChange(void)
 {
+	ONLY_IF_GL_LOADED
 	// refresh the screen palette
 	if (HWR_ShouldUsePaletteRendering())
 		HWR_SetPalette(pLocalPalette);
@@ -6511,6 +6510,8 @@ static void CV_glpalettedepth_OnChange(void)
 
 static void CV_glshaders_OnChange(void)
 {
+	ONLY_IF_GL_LOADED
+	HWR_SetShaderState();
 	if (cv_glpaletterendering.value)
 	{
 		// can't do palette rendering without shaders, so update the state if needed
@@ -6582,6 +6583,7 @@ void HWR_Startup(void)
 #endif
 
 		gl_shadersavailable = HWR_InitShaders();
+		HWR_SetShaderState();
 		HWR_LoadAllCustomShaders();
 		HWR_TogglePaletteRendering();
 	}
@@ -6870,9 +6872,6 @@ void HWR_DoWipe(UINT8 wipenum, UINT8 scrnnum)
 		// polycolor alpha communicates fadein / fadeout to the shader and the backend
 		surf.PolyColor.s.alpha = (wipestyleflags & WSF_FADEIN) ? 255 : 0;
 
-		// backend shader may not have been enabled yet so do it here
-		HWR_SetShaderState();
-
 		HWD.pfnSetShader(HWR_GetShaderFromTarget(SHADER_UI_TINTED_WIPE));
 		HWD.pfnDoScreenWipe(HWD_SCREENTEXTURE_WIPE_START, HWD_SCREENTEXTURE_WIPE_END,
 			&surf, polyflags);

src/hardware/hw_shaders.c

@@ -306,12 +306,12 @@
 #define GLSL_PALETTE_POSTPROCESS_SHADER \
 	"uniform sampler2D tex;\n" \
 	"uniform sampler3D palette_lookup_tex;\n" \
-	"uniform sampler1D screen_palette_tex;\n" \
+	"uniform sampler1D palette_tex;\n" \
 	"void main(void) {\n" \
 		"vec4 texel = texture2D(tex, gl_TexCoord[0].st);\n" \
 		"float tex_pal_idx = texture3D(palette_lookup_tex, vec3((texel * 63.0 + 0.5) / 64.0))[0] * 255.0;\n" \
 		"float palette_coord = (tex_pal_idx + 0.5) / 256.0;\n" \
-		"vec4 final_color = texture1D(screen_palette_tex, palette_coord);\n" \
+		"vec4 final_color = texture1D(palette_tex, palette_coord);\n" \
 		"gl_FragColor = final_color;\n" \
 	"}\0"
 

src/hardware/r_opengl/r_opengl.c

@@ -617,9 +617,9 @@ typedef enum
 	gluniform_fade_end,
 
 	// palette rendering
-	gluniform_screen_palette_tex,
+	gluniform_palette_tex, // 1d texture containing a palette
-	gluniform_palette_lookup_tex,
+	gluniform_palette_lookup_tex, // 3d texture containing the rgb->index lookup table
-	gluniform_lighttable_tex,
+	gluniform_lighttable_tex, // 2d texture containing a light table
 
 	// misc.
 	gluniform_leveltime,
@@ -1918,7 +1918,7 @@ static boolean Shader_CompileProgram(gl_shader_t *shader, GLint i)
 	shader->uniforms[gluniform_fade_end] = GETUNI("fade_end");
 
 	// palette rendering
-	shader->uniforms[gluniform_screen_palette_tex] = GETUNI("screen_palette_tex");
+	shader->uniforms[gluniform_palette_tex] = GETUNI("palette_tex");
 	shader->uniforms[gluniform_palette_lookup_tex] = GETUNI("palette_lookup_tex");
 	shader->uniforms[gluniform_lighttable_tex] = GETUNI("lighttable_tex");
 
@@ -1934,7 +1934,7 @@ static boolean Shader_CompileProgram(gl_shader_t *shader, GLint i)
 	pglUseProgram(shader->program);
 
 	// texture unit numbers for the samplers used for palette rendering
-	UNIFORM_1(shader->uniforms[gluniform_screen_palette_tex], 2, pglUniform1i);
+	UNIFORM_1(shader->uniforms[gluniform_palette_tex], 2, pglUniform1i);
 	UNIFORM_1(shader->uniforms[gluniform_palette_lookup_tex], 1, pglUniform1i);
 	UNIFORM_1(shader->uniforms[gluniform_lighttable_tex], 2, pglUniform1i);
 

src/sdl/i_video.c

@@ -1258,11 +1258,9 @@ void I_FinishUpdate(void)
 		if (HWR_ShouldUsePaletteRendering())
 		{
 			HWD.pfnMakeScreenTexture(HWD_SCREENTEXTURE_GENERIC2);
-			HWD.pfnSetSpecialState(HWD_SET_SHADERS, 1);
 			HWD.pfnSetShader(HWR_GetShaderFromTarget(SHADER_PALETTE_POSTPROCESS));
 			HWD.pfnDrawScreenTexture(HWD_SCREENTEXTURE_GENERIC2, NULL, 0);
 			HWD.pfnUnSetShader();
-			HWD.pfnSetSpecialState(HWD_SET_SHADERS, 0);
 		}
 		OglSdlFinishUpdate(cv_vidwait.value);
 	}