From e1df17964e4575bfdc6c4f4ca4f072c3ba600874 Mon Sep 17 00:00:00 2001
From: Christoph Oelckers <coelckers@users.noreply.github.com>
Date: Thu, 1 Apr 2021 19:09:08 +0200
Subject: [PATCH] - fixed handling of SW's shadows.

An out of range shade will only work without fog.
---
 source/core/rendering/scene/hw_sprites.cpp | 13 +++++++++++--
 source/glbackend/glbackend.cpp             |  3 ++-
 source/glbackend/glbackend.h               |  1 +
 source/glbackend/pm_renderstate.h          |  1 +
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/source/core/rendering/scene/hw_sprites.cpp b/source/core/rendering/scene/hw_sprites.cpp
index 390d7ccc1..a38e34c61 100644
--- a/source/core/rendering/scene/hw_sprites.cpp
+++ b/source/core/rendering/scene/hw_sprites.cpp
@@ -114,6 +114,15 @@ void HWSprite::DrawSprite(HWDrawInfo* di, FRenderState& state, bool translucent)
 	bool foggy = (GlobalMapFog || (fade & 0xffffff));
 	auto ShadeDiv = lookups.tables[palette].ShadeFactor;
 	// Disable brightmaps if non-black fog is used.
+	int shade = this->shade;
+	PalEntry color(255, globalr, globalg, globalb);
+	if (this->shade > numshades) // handling of SW's shadow hack using a shade of 127.
+	{
+		shade = sector[sprite->sectnum].floorshade;
+		color = 0xff000000;
+	}
+	shade = clamp(shade, 0, numshades - 1);
+
 	if (ShadeDiv >= 1 / 1000.f && foggy && !foglayer)
 	{
 		state.EnableFog(1);
@@ -131,7 +140,7 @@ void HWSprite::DrawSprite(HWDrawInfo* di, FRenderState& state, bool translucent)
 	}
 
 	// The shade rgb from the tint is ignored here.
-	state.SetColorAlpha(PalEntry(255, globalr, globalg, globalb), alpha);
+	state.SetColorAlpha(color, alpha);
 
 	state.SetMaterial(texture, UF_Texture, 0, CLAMP_XY, TRANSLATION(Translation_Remap + curbasepal, palette), -1);
 
@@ -320,7 +329,7 @@ void HWSprite::Process(HWDrawInfo* di, spritetype* spr, sectortype* sector, int
 
 	modelframe = 0;
 	dynlightindex = -1;
-	shade = clamp(spr->shade, 0, numshades-1);
+	shade = spr->shade;
 	palette = spr->pal;
 	fade = lookups.getFade(sector->floorpal);	// fog is per sector.
 	visibility = sectorVisibility(sector);
diff --git a/source/glbackend/glbackend.cpp b/source/glbackend/glbackend.cpp
index 9625f2364..23686eaa2 100644
--- a/source/glbackend/glbackend.cpp
+++ b/source/glbackend/glbackend.cpp
@@ -206,7 +206,8 @@ bool PolymostRenderState::Apply(FRenderState& state, GLState& oldState)
 		state.SetMaterial(mMaterial.mTexture, mMaterial.uFlags, mMaterial.mScaleFlags, mMaterial.mClampMode, mMaterial.mTranslation, mMaterial.mOverrideShader);
 	}
 
-	state.SetColor(Color[0], Color[1], Color[2], Color[3]);
+	if (!drawblack) state.SetColor(Color[0], Color[1], Color[2], Color[3]);
+	else state.SetColor(0, 0, 0, Color[3]);
 	if (StateFlags != oldState.Flags)
 	{
 		state.EnableDepthTest(StateFlags & STF_DEPTHTEST);
diff --git a/source/glbackend/glbackend.h b/source/glbackend/glbackend.h
index ced1e8885..a21ccb2a3 100644
--- a/source/glbackend/glbackend.h
+++ b/source/glbackend/glbackend.h
@@ -138,6 +138,7 @@ public:
 
 	void SetShade(int32_t shade, int numshades)
 	{
+		renderState.drawblack = shade > numshades;
 		renderState.Shade = std::min(shade, numshades-1);
 	}
 
diff --git a/source/glbackend/pm_renderstate.h b/source/glbackend/pm_renderstate.h
index 7127cbf77..c1d45580b 100644
--- a/source/glbackend/pm_renderstate.h
+++ b/source/glbackend/pm_renderstate.h
@@ -69,6 +69,7 @@ struct PolymostRenderState
 {
 	int vindex, vcount, primtype;
     int Shade;
+	int drawblack = false;
 	float ShadeDiv = 62.f;
 	float VisFactor = 128.f;
 	int Flags = 0;