- Add fog boundary support to softpoly

src/polyrenderer/drawers/poly_drawer32.h

@@ -42,7 +42,7 @@ namespace TriScreenDrawerModes
 	FORCEINLINE unsigned int Sample32(int32_t u, int32_t v, const uint32_t *texPixels, int texWidth, int texHeight, uint32_t oneU, uint32_t oneV, uint32_t color, const uint32_t *translation)
 	{
 		uint32_t texel;
-		if (SamplerT::Mode == (int)Samplers::Shaded || SamplerT::Mode == (int)Samplers::Stencil || SamplerT::Mode == (int)Samplers::Fill || SamplerT::Mode == (int)Samplers::Fuzz)
+		if (SamplerT::Mode == (int)Samplers::Shaded || SamplerT::Mode == (int)Samplers::Stencil || SamplerT::Mode == (int)Samplers::Fill || SamplerT::Mode == (int)Samplers::Fuzz || SamplerT::Mode == (int)Samplers::FogBoundary)
 		{
 			return color;
 		}
@@ -426,6 +426,7 @@ private:
 						bgcolor = 0;
 
 					// Sample fgcolor
+					if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[ix];
 					unsigned int ifgcolor = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					unsigned int ifgshade = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
@@ -508,6 +509,7 @@ private:
 						bgcolor = 0;
 
 					// Sample fgcolor
+					if (SamplerT::Mode == (int)Samplers::FogBoundary && (mask0 & (1 << 31))) color = dest[x];
 					unsigned int ifgcolor = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					unsigned int ifgshade = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
@@ -590,6 +592,7 @@ private:
 						bgcolor = 0;
 
 					// Sample fgcolor
+					if (SamplerT::Mode == (int)Samplers::FogBoundary && (mask1 & (1 << 31))) color = dest[x];
 					unsigned int ifgcolor = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					unsigned int ifgshade = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
@@ -765,6 +768,7 @@ private:
 					bgcolor = 0;
 
 				// Sample fgcolor
+				if (SamplerT::Mode == (int)Samplers::FogBoundary) color = *dest;
 				unsigned int ifgcolor = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 				unsigned int ifgshade = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 				posU += stepU;

src/polyrenderer/drawers/poly_drawer32_sse2.h

@@ -30,7 +30,7 @@ namespace TriScreenDrawerModes
 	FORCEINLINE unsigned int VECTORCALL Sample32(int32_t u, int32_t v, const uint32_t *texPixels, int texWidth, int texHeight, uint32_t oneU, uint32_t oneV, uint32_t color, const uint32_t *translation)
 	{
 		uint32_t texel;
-		if (SamplerT::Mode == (int)Samplers::Shaded || SamplerT::Mode == (int)Samplers::Stencil || SamplerT::Mode == (int)Samplers::Fill || SamplerT::Mode == (int)Samplers::Fuzz)
+		if (SamplerT::Mode == (int)Samplers::Shaded || SamplerT::Mode == (int)Samplers::Stencil || SamplerT::Mode == (int)Samplers::Fill || SamplerT::Mode == (int)Samplers::Fuzz || SamplerT::Mode == (int)Samplers::FogBoundary)
 		{
 			return color;
 		}
@@ -430,11 +430,13 @@ private:
 
 					// Sample fgcolor
 					unsigned int ifgcolor[2], ifgshade[2];
+					if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[ix * 2];
 					ifgcolor[0] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					ifgshade[0] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
 					posV += stepV;
 
+					if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[ix * 2 + 1];
 					ifgcolor[1] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					ifgshade[1] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
@@ -517,11 +519,13 @@ private:
 
 					// Sample fgcolor
 					unsigned int ifgcolor[2], ifgshade[2];
+					if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[x * 2];
 					ifgcolor[0] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					ifgshade[0] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
 					posV += stepV;
 
+					if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[x * 2 + 1];
 					ifgcolor[1] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					ifgshade[1] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
@@ -606,11 +610,13 @@ private:
 
 					// Sample fgcolor
 					unsigned int ifgcolor[2], ifgshade[2];
+					if (SamplerT::Mode == (int)Samplers::FogBoundary && (mask1 & (1 << 31))) color = dest[x * 2];
 					ifgcolor[0] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					ifgshade[0] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
 					posV += stepV;
 
+					if (SamplerT::Mode == (int)Samplers::FogBoundary && (mask1 & (1 << 30))) color = dest[x * 2 + 1];
 					ifgcolor[1] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 					ifgshade[1] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					posU += stepU;
@@ -780,10 +786,12 @@ private:
 
 				// Sample fgcolor
 				unsigned int ifgcolor[2], ifgshade[2];
+				if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[0];
 				ifgcolor[0] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 				ifgshade[0] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 				posU += stepU;
 
+				if (SamplerT::Mode == (int)Samplers::FogBoundary) color = dest[1];
 				ifgcolor[1] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 				ifgshade[1] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 				posU += stepU;
@@ -809,6 +817,7 @@ private:
 
 				// Sample fgcolor
 				unsigned int ifgcolor[2], ifgshade[2];
+				if (SamplerT::Mode == (int)Samplers::FogBoundary) color = *dest;
 				ifgcolor[0] = Sample32<SamplerT, FilterModeT>(posU, posV, texPixels, texWidth, texHeight, oneU, oneV, color, translation);
 				ifgshade[0] = SampleShade32<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 				ifgcolor[1] = 0;

src/polyrenderer/drawers/poly_drawer8.h

@@ -30,7 +30,7 @@ namespace TriScreenDrawerModes
 	FORCEINLINE unsigned int Sample8(int32_t u, int32_t v, const uint8_t *texPixels, int texWidth, int texHeight, uint32_t color, const uint8_t *translation)
 	{
 		uint8_t texel;
-		if (SamplerT::Mode == (int)Samplers::Shaded || SamplerT::Mode == (int)Samplers::Stencil || SamplerT::Mode == (int)Samplers::Fill || SamplerT::Mode == (int)Samplers::Fuzz)
+		if (SamplerT::Mode == (int)Samplers::Shaded || SamplerT::Mode == (int)Samplers::Stencil || SamplerT::Mode == (int)Samplers::Fill || SamplerT::Mode == (int)Samplers::Fuzz || SamplerT::Mode == (int)Samplers::FogBoundary)
 		{
 			return color;
 		}
@@ -290,6 +290,7 @@ public:
 				{
 					int lightshade = lightpos >> 8;
 					uint8_t bgcolor = dest[ix];
+					if (SamplerT::Mode == (int)Samplers::FogBoundary) color = bgcolor;
 					uint8_t fgcolor = Sample8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, color, translation);
 					uint32_t fgshade = SampleShade8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 					if (SamplerT::Mode == (int)Samplers::Fuzz) lightshade = 256;
@@ -339,6 +340,7 @@ public:
 					{
 						int lightshade = lightpos >> 8;
 						uint8_t bgcolor = dest[x];
+						if (SamplerT::Mode == (int)Samplers::FogBoundary) color = bgcolor;
 						uint8_t fgcolor = Sample8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, color, translation);
 						uint32_t fgshade = SampleShade8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 						if (SamplerT::Mode == (int)Samplers::Fuzz) lightshade = 256;
@@ -390,6 +392,7 @@ public:
 					{
 						int lightshade = lightpos >> 8;
 						uint8_t bgcolor = dest[x];
+						if (SamplerT::Mode == (int)Samplers::FogBoundary) color = bgcolor;
 						uint8_t fgcolor = Sample8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, color, translation);
 						uint32_t fgshade = SampleShade8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 						if (SamplerT::Mode == (int)Samplers::Fuzz) lightshade = 256;
@@ -473,6 +476,7 @@ public:
 			for (int i = 0; i < count; i++)
 			{
 				uint8_t bgcolor = *dest;
+				if (SamplerT::Mode == (int)Samplers::FogBoundary) color = bgcolor;
 				uint8_t fgcolor = Sample8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, color, translation);
 				uint32_t fgshade = SampleShade8<SamplerT>(posU, posV, texPixels, texWidth, texHeight, fuzzpos);
 				*dest = ShadeAndBlend8<BlendT>(fgcolor, bgcolor, fgshade, lightshade, colormaps, srcalpha, destalpha);

src/polyrenderer/drawers/screen_triangle.cpp

@@ -984,7 +984,8 @@ void(*ScreenTriangle::TriDrawers8[])(int, int, uint32_t, uint32_t, const TriDraw
 	&TriScreenDrawer8<TriScreenDrawerModes::RevSubClampBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillRevSub
 	&TriScreenDrawer8<TriScreenDrawerModes::AddSrcColorBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillAddSrcColor
 	&TriScreenDrawer8<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::SkycapSampler>::Execute,          // Skycap
-	&TriScreenDrawer8<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute             // Fuzz
+	&TriScreenDrawer8<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute,            // Fuzz
+	&TriScreenDrawer8<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::FogBoundarySampler>::Execute,     // FogBoundary
 };
 
 void(*ScreenTriangle::TriDrawers32[])(int, int, uint32_t, uint32_t, const TriDrawTriangleArgs *) =
@@ -1011,7 +1012,8 @@ void(*ScreenTriangle::TriDrawers32[])(int, int, uint32_t, uint32_t, const TriDra
 	&TriScreenDrawer32<TriScreenDrawerModes::RevSubClampBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillRevSub
 	&TriScreenDrawer32<TriScreenDrawerModes::AddSrcColorBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillAddSrcColor
 	&TriScreenDrawer32<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::SkycapSampler>::Execute,          // Skycap
-	&TriScreenDrawer32<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute             // Fuzz
+	&TriScreenDrawer32<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute,            // Fuzz
+	&TriScreenDrawer32<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::FogBoundarySampler>::Execute      // FogBoundary
 };
 
 void(*ScreenTriangle::RectDrawers8[])(const void *, int, int, int, const RectDrawArgs *, WorkerThreadData *) =
@@ -1038,7 +1040,8 @@ void(*ScreenTriangle::RectDrawers8[])(const void *, int, int, int, const RectDra
 	&RectScreenDrawer8<TriScreenDrawerModes::RevSubClampBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillRevSub
 	&RectScreenDrawer8<TriScreenDrawerModes::AddSrcColorBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillAddSrcColor
 	&RectScreenDrawer8<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::SkycapSampler>::Execute,          // Skycap
-	&RectScreenDrawer8<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute             // Fuzz
+	&RectScreenDrawer8<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute,            // Fuzz
+	&RectScreenDrawer8<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::FogBoundarySampler>::Execute      // FogBoundary
 };
 
 void(*ScreenTriangle::RectDrawers32[])(const void *, int, int, int, const RectDrawArgs *, WorkerThreadData *) =
@@ -1065,7 +1068,8 @@ void(*ScreenTriangle::RectDrawers32[])(const void *, int, int, int, const RectDr
 	&RectScreenDrawer32<TriScreenDrawerModes::RevSubClampBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillRevSub
 	&RectScreenDrawer32<TriScreenDrawerModes::AddSrcColorBlend, TriScreenDrawerModes::FillSampler>::Execute,       // FillAddSrcColor
 	&RectScreenDrawer32<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::SkycapSampler>::Execute,          // Skycap
-	&RectScreenDrawer32<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute             // Fuzz
+	&RectScreenDrawer32<TriScreenDrawerModes::ShadedBlend, TriScreenDrawerModes::FuzzSampler>::Execute,            // Fuzz
+	&RectScreenDrawer32<TriScreenDrawerModes::OpaqueBlend, TriScreenDrawerModes::FogBoundarySampler>::Execute,     // FogBoundary
 };
 
 int ScreenTriangle::FuzzStart = 0;

src/polyrenderer/drawers/screen_triangle.h

@@ -120,7 +120,8 @@ enum class TriBlendMode
 	FillRevSub,
 	FillAddSrcColor,
 	Skycap,
-	Fuzz
+	Fuzz,
+	FogBoundary
 };
 
 class ScreenTriangle
@@ -157,7 +158,7 @@ namespace TriScreenDrawerModes
 	struct SimpleShade { static const int Mode = (int)ShadeMode::Simple; };
 	struct AdvancedShade { static const int Mode = (int)ShadeMode::Advanced; };
 
-	enum class Samplers { Texture, Fill, Shaded, Stencil, Translated, Skycap, Fuzz };
+	enum class Samplers { Texture, Fill, Shaded, Stencil, Translated, Skycap, Fuzz, FogBoundary };
 	struct TextureSampler { static const int Mode = (int)Samplers::Texture; };
 	struct FillSampler { static const int Mode = (int)Samplers::Fill; };
 	struct ShadedSampler { static const int Mode = (int)Samplers::Shaded; };
@@ -165,6 +166,7 @@ namespace TriScreenDrawerModes
 	struct TranslatedSampler { static const int Mode = (int)Samplers::Translated; };
 	struct SkycapSampler { static const int Mode = (int)Samplers::Skycap; };
 	struct FuzzSampler { static const int Mode = (int)Samplers::Fuzz; };
+	struct FogBoundarySampler { static const int Mode = (int)Samplers::FogBoundary; };
 
 	static const int fuzzcolormap[FUZZTABLE] =
 	{

src/polyrenderer/scene/poly_wall.cpp

@@ -36,6 +36,7 @@
 #include "g_levellocals.h"
 
 EXTERN_CVAR(Bool, r_drawmirrors)
+EXTERN_CVAR(Bool, r_fogboundary)
 
 bool RenderPolyWall::RenderLine(const TriMatrix &worldToClip, const PolyClipPlane &clipPlane, PolyCull &cull, seg_t *line, sector_t *frontsector, uint32_t subsectorDepth, uint32_t stencilValue, std::vector<PolyTranslucentObject*> &translucentWallsOutput, std::vector<std::unique_ptr<PolyDrawLinePortal>> &linePortals, line_t *lastPortalLine)
 {
@@ -158,9 +159,10 @@ bool RenderPolyWall::RenderLine(const TriMatrix &worldToClip, const PolyClipPlan
 			wall.BottomTexZ = MIN(middlefloorz1, middlefloorz2);
 			wall.Texpart = side_t::mid;
 			wall.Masked = true;
+			wall.FogBoundary = IsFogBoundary(frontsector, backsector);
 
 			FTexture *midtex = TexMan(line->sidedef->GetTexture(side_t::mid), true);
-			if (midtex && midtex->UseType != FTexture::TEX_Null)
+			if (midtex && midtex->UseType != FTexture::TEX_Null || wall.FogBoundary)
 				translucentWallsOutput.push_back(PolyRenderer::Instance()->FrameMemory.NewObject<PolyTranslucentObject>(wall));
 
 			if (polyportal)
@@ -173,6 +175,13 @@ bool RenderPolyWall::RenderLine(const TriMatrix &worldToClip, const PolyClipPlan
 	return polyportal != nullptr;
 }
 
+bool RenderPolyWall::IsFogBoundary(sector_t *front, sector_t *back)
+{
+	return r_fogboundary && PolyCameraLight::Instance()->FixedColormap() == nullptr && front->Colormap.FadeColor &&
+		front->Colormap.FadeColor != back->Colormap.FadeColor &&
+		(front->GetTexture(sector_t::ceiling) != skyflatnum || back->GetTexture(sector_t::ceiling) != skyflatnum);
+}
+
 void RenderPolyWall::Render3DFloorLine(const TriMatrix &worldToClip, const PolyClipPlane &clipPlane, PolyCull &cull, seg_t *line, sector_t *frontsector, uint32_t subsectorDepth, uint32_t stencilValue, F3DFloor *fakeFloor, std::vector<PolyTranslucentObject*> &translucentWallsOutput)
 {
 	double frontceilz1 = fakeFloor->top.plane->ZatPoint(line->v1);
@@ -213,7 +222,7 @@ void RenderPolyWall::Render(const TriMatrix &worldToClip, const PolyClipPlane &c
 {
 	bool foggy = false;
 	FTexture *tex = GetTexture();
-	if (!tex && !Polyportal)
+	if (!tex && !Polyportal && !FogBoundary)
 		return;
 
 	TriVertex *vertices = PolyRenderer::Instance()->FrameMemory.AllocMemory<TriVertex>(4);
@@ -278,6 +287,18 @@ void RenderPolyWall::Render(const TriMatrix &worldToClip, const PolyClipPlane &c
 		args.SetTexture(tex);
 	args.SetClipPlane(clipPlane);
 
+	if (FogBoundary)
+	{
+		args.SetStyle(TriBlendMode::FogBoundary);
+		args.SetColor(0xffffffff, 254);
+		args.SetDepthTest(true);
+		args.SetWriteDepth(true);
+		args.SetWriteStencil(false);
+		args.DrawArray(vertices, 4, PolyDrawMode::TriangleFan);
+		if (!tex)
+			return;
+	}
+
 	if (Polyportal)
 	{
 		args.SetWriteStencil(true, Polyportal->StencilValue);

src/polyrenderer/scene/poly_wall.h

@@ -54,6 +54,7 @@ public:
 	double UnpeggedCeil2 = 0.0;
 	FSWColormap *Colormap = nullptr;
 	bool Masked = false;
+	bool FogBoundary = false;
 	uint32_t SubsectorDepth = 0;
 	uint32_t StencilValue = 0;
 	PolyDrawLinePortal *Polyportal = nullptr;
@@ -62,6 +63,8 @@ private:
 	void ClampHeight(TriVertex &v1, TriVertex &v2);
 	FTexture *GetTexture();
 	int GetLightLevel();
+
+	static bool IsFogBoundary(sector_t *front, sector_t *back);
 };
 
 class PolyWallTextureCoordsU