From d28d5a5e1fad863531b58cfbe3681c632bcb4365 Mon Sep 17 00:00:00 2001
From: Christoph Oelckers <coelckers@users.noreply.github.com>
Date: Sat, 27 Feb 2021 13:30:52 +0100
Subject: [PATCH] - use real fog, even in palette emulation mode.

Using the palette to apply fog is just far too broken and cannot be kept in check with all the hacks the Build engine allows.
This only works if all elements on screen use the identity translation lookup which is basically never the case.
Real fog, on the other hand, can easily be applied to everything.
---
 source/core/textures/hightile.cpp | 13 ++++++----
 source/games/duke/src/sbar_d.cpp  |  4 ++--
 source/glbackend/glbackend.cpp    | 40 +++++++++++++++----------------
 3 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/source/core/textures/hightile.cpp b/source/core/textures/hightile.cpp
index 766ebf8ff..f5d240233 100644
--- a/source/core/textures/hightile.cpp
+++ b/source/core/textures/hightile.cpp
@@ -311,13 +311,14 @@ int tileSetSkybox(int picnum, int palnum, const char **facenames, int flags )
 //
 //===========================================================================
 
-bool PickTexture(FGameTexture* tex, int paletteid, TexturePick& pick)
+bool PickTexture(FRenderState *state, FGameTexture* tex, int paletteid, TexturePick& pick)
 {
 	if (!tex->isValid() || tex->GetTexelWidth() <= 0 || tex->GetTexelHeight() <= 0) return false;
 
 	int usepalette = paletteid == 0? 0 : GetTranslationType(paletteid) - Translation_Remap;
 	int usepalswap = GetTranslationIndex(paletteid);
-	int TextureType = hw_int_useindexedcolortextures? TT_INDEXED : TT_TRUECOLOR;
+	bool foggy = state && (state->GetFogColor() & 0xffffff);
+	int TextureType = hw_int_useindexedcolortextures && !foggy? TT_INDEXED : TT_TRUECOLOR;
 
 	pick.translation = paletteid;
 	pick.basepalTint = 0xffffff;
@@ -383,12 +384,16 @@ bool PreBindTexture(FRenderState* state, FGameTexture*& tex, EUpscaleFlags& flag
 
 	if (tex->GetUseType() == ETextureType::Special) return true;
 
-	if (PickTexture(tex, translation, pick))
+	if (PickTexture(state, tex, translation, pick))
 	{
 		int TextureType = (pick.translation & 0x80000000) ? TT_INDEXED : TT_TRUECOLOR;
 		int lookuppal = pick.translation & 0x7fffffff;
 
-		if (pick.translation & 0x80000000) scaleflags |= CTF_Indexed;
+		if (pick.translation & 0x80000000)
+		{
+			scaleflags |= CTF_Indexed;
+			if (state) state->EnableFog(0);
+		}
 		tex = pick.texture;
 		translation = lookuppal;
 
diff --git a/source/games/duke/src/sbar_d.cpp b/source/games/duke/src/sbar_d.cpp
index 992d9f673..ac7088136 100644
--- a/source/games/duke/src/sbar_d.cpp
+++ b/source/games/duke/src/sbar_d.cpp
@@ -44,7 +44,7 @@ source as it is released.
 #include "texturemanager.h"
 #include "dukeactor.h"
 
-bool PickTexture(FGameTexture* tex, int paletteid, TexturePick& pick);
+bool PickTexture(FRenderState* state, FGameTexture* tex, int paletteid, TexturePick& pick);
 
 BEGIN_DUKE_NS
 
@@ -85,7 +85,7 @@ public:
 				int fh0 = zerotex->GetTexture()->CheckRealHeight();
 				int fh1 = fh0;
 				TexturePick pick;
-				if (PickTexture(zerotex, TRANSLATION(Translation_Remap, 0), pick))
+				if (PickTexture(nullptr, zerotex, TRANSLATION(Translation_Remap, 0), pick))
 				{
 					int oheight = zerotex->GetTexelHeight();
 					int dheight = pick.texture->GetTexelHeight();
diff --git a/source/glbackend/glbackend.cpp b/source/glbackend/glbackend.cpp
index 3ab31213c..6b3dad847 100644
--- a/source/glbackend/glbackend.cpp
+++ b/source/glbackend/glbackend.cpp
@@ -145,14 +145,29 @@ void GLInstance::SetPalswap(int index)
 
 void GLInstance::SetFade(int index)
 {
-	if (!hw_useindexedcolortextures)
-		renderState.FogColor = lookups.getFade(index);
-	else
-		renderState.FogColor = 0;
+	renderState.FogColor = lookups.getFade(index);
 }
 
 bool PolymostRenderState::Apply(FRenderState& state, GLState& oldState)
 {
+	// Fog must be done before the texture so that the texture selector can override it.
+	bool foggy = (GLInterface.useMapFog || (FogColor & 0xffffff));
+	// Disable brightmaps if non-black fog is used.
+	if (!(Flags & RF_FogDisabled) && ShadeDiv >= 1 / 1000.f && foggy)
+	{
+		state.EnableFog(1);
+		float density = GLInterface.useMapFog ? 350.f : 350.f - Scale(numshades - Shade, 150, numshades);
+		state.SetFog((GLInterface.useMapFog) ? PalEntry(0x999999) : FogColor, density);
+		state.SetSoftLightLevel(255);
+		state.SetLightParms(128.f, 1 / 1000.f);
+	}
+	else
+	{
+		state.EnableFog(0);
+		state.SetSoftLightLevel(ShadeDiv >= 1 / 1000.f ? 255 - Scale(Shade, 255, numshades) : 255);
+		state.SetLightParms(VisFactor, ShadeDiv / (numshades - 2));
+	}
+
 	if (Flags & RF_ColorOnly)
 	{
 		state.EnableTexture(false);
@@ -224,23 +239,6 @@ bool PolymostRenderState::Apply(FRenderState& state, GLState& oldState)
 		state.SetDepthFunc(DepthFunc);
 		oldState.DepthFunc = DepthFunc;
 	}
-	bool foggy = (GLInterface.useMapFog || (FogColor & 0xffffff));
-	// Disable brightmaps if non-black fog is used.
-	if (!(Flags & RF_FogDisabled) && ShadeDiv >= 1 / 1000.f && foggy)
-	{
-		state.EnableFog(1);
-		float density = GLInterface.useMapFog ? 350.f : 350.f - Scale(numshades - Shade, 150, numshades);
-		state.SetFog((GLInterface.useMapFog) ? PalEntry(0x999999) : FogColor, density);
-		state.SetSoftLightLevel(255);
-		state.SetLightParms(128.f, 1/1000.f);
-	}
-	else
-	{
-		state.EnableFog(0);
-		state.SetSoftLightLevel(ShadeDiv >= 1 / 1000.f ? 255 - Scale(Shade, 255, numshades) : 255);
-		state.SetLightParms(VisFactor, ShadeDiv / (numshades - 2));
-	}
-
 
 	state.SetTextureMode(TextureMode);