From 232b93534de2f0396bdf8e29bc0618196e17ad7e Mon Sep 17 00:00:00 2001
From: "Dileep V. Reddy" <dileepvr@gmail.com>
Date: Sun, 19 Jan 2025 17:19:54 -0700
Subject: [PATCH] Better flat visibility checks for Ortho projection.

---
 src/rendering/hwrenderer/scene/hw_flats.cpp | 12 ++++++------
 src/rendering/r_utility.cpp                 |  3 +++
 src/rendering/r_utility.h                   |  1 +
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/rendering/hwrenderer/scene/hw_flats.cpp b/src/rendering/hwrenderer/scene/hw_flats.cpp
index f423c73fce..f14757aded 100644
--- a/src/rendering/hwrenderer/scene/hw_flats.cpp
+++ b/src/rendering/hwrenderer/scene/hw_flats.cpp
@@ -518,7 +518,7 @@ void HWFlat::ProcessSector(HWDrawInfo *di, sector_t * frontsector, int which)
 	//
 	//
 	//
-	if (((which & SSRF_RENDERFLOOR) && frontsector->floorplane.ZatPoint(vp.Pos) <= vp.Pos.Z && (!section || !(section->flags & FSection::DONTRENDERFLOOR)))&& !(vp.IsOrtho() && (vp.PitchSin < 0.0)))
+	if ((which & SSRF_RENDERFLOOR) && (vp.IsOrtho() ? vp.ViewVector3D.dot(frontsector->floorplane.Normal()) < 0.0 : frontsector->floorplane.ZatPoint(vp.Pos) <= vp.Pos.Z) && (!section || !(section->flags & FSection::DONTRENDERFLOOR)))
 	{
 		// process the original floor first.
 
@@ -576,7 +576,7 @@ void HWFlat::ProcessSector(HWDrawInfo *di, sector_t * frontsector, int which)
 	//
 	// 
 	//
-	if (((which & SSRF_RENDERCEILING) && frontsector->ceilingplane.ZatPoint(vp.Pos) >= vp.Pos.Z && (!section || !(section->flags & FSection::DONTRENDERCEILING))) && !(vp.IsOrtho() && (vp.PitchSin > 0.0)))
+	if ((which & SSRF_RENDERCEILING) && (vp.IsOrtho() ? vp.ViewVector3D.dot(frontsector->ceilingplane.Normal()) < 0.0 : frontsector->ceilingplane.ZatPoint(vp.Pos) >= vp.Pos.Z) && (!section || !(section->flags & FSection::DONTRENDERCEILING)))
 	{
 		// process the original ceiling first.
 
@@ -661,7 +661,7 @@ void HWFlat::ProcessSector(HWDrawInfo *di, sector_t * frontsector, int which)
 					double ff_top = rover->top.plane->ZatPoint(sector->centerspot);
 					if (ff_top < lastceilingheight)
 					{
-						if (vp.Pos.Z <= rover->top.plane->ZatPoint(vp.Pos))
+						if ((vp.IsOrtho() ? vp.ViewVector3D.dot(rover->top.plane->Normal()) > 0.0 : vp.Pos.Z <= rover->top.plane->ZatPoint(vp.Pos)))
 						{
 							SetFrom3DFloor(rover, true, !!(rover->flags&FF_FOG));
 							Colormap.FadeColor = frontsector->Colormap.FadeColor;
@@ -675,7 +675,7 @@ void HWFlat::ProcessSector(HWDrawInfo *di, sector_t * frontsector, int which)
 					double ff_bottom = rover->bottom.plane->ZatPoint(sector->centerspot);
 					if (ff_bottom < lastceilingheight)
 					{
-						if (vp.Pos.Z <= rover->bottom.plane->ZatPoint(vp.Pos))
+						if ((vp.IsOrtho() ? vp.ViewVector3D.dot(rover->bottom.plane->Normal()) > 0.0 : vp.Pos.Z <= rover->bottom.plane->ZatPoint(vp.Pos)))
 						{
 							SetFrom3DFloor(rover, false, !(rover->flags&FF_FOG));
 							Colormap.FadeColor = frontsector->Colormap.FadeColor;
@@ -701,7 +701,7 @@ void HWFlat::ProcessSector(HWDrawInfo *di, sector_t * frontsector, int which)
 					double ff_bottom = rover->bottom.plane->ZatPoint(sector->centerspot);
 					if (ff_bottom > lastfloorheight || (rover->flags&FF_FIX))
 					{
-						if (vp.Pos.Z >= rover->bottom.plane->ZatPoint(vp.Pos))
+						if ((vp.IsOrtho() ? vp.ViewVector3D.dot(rover->bottom.plane->Normal()) > 0.0 : vp.Pos.Z >= rover->bottom.plane->ZatPoint(vp.Pos)))
 						{
 							SetFrom3DFloor(rover, false, !(rover->flags&FF_FOG));
 							Colormap.FadeColor = frontsector->Colormap.FadeColor;
@@ -722,7 +722,7 @@ void HWFlat::ProcessSector(HWDrawInfo *di, sector_t * frontsector, int which)
 					double ff_top = rover->top.plane->ZatPoint(sector->centerspot);
 					if (ff_top > lastfloorheight)
 					{
-						if (vp.Pos.Z >= rover->top.plane->ZatPoint(vp.Pos))
+						if ((vp.IsOrtho() ? vp.ViewVector3D.dot(rover->top.plane->Normal()) > 0.0 : vp.Pos.Z >= rover->top.plane->ZatPoint(vp.Pos)))
 						{
 							SetFrom3DFloor(rover, true, !!(rover->flags&FF_FOG));
 							Colormap.FadeColor = frontsector->Colormap.FadeColor;
diff --git a/src/rendering/r_utility.cpp b/src/rendering/r_utility.cpp
index 3a3fdd70e7..cd0c3d0c28 100644
--- a/src/rendering/r_utility.cpp
+++ b/src/rendering/r_utility.cpp
@@ -704,6 +704,9 @@ void FRenderViewpoint::SetViewAngle(const FViewWindow& viewWindow)
 	ViewVector.X = v.X;
 	ViewVector.Y = v.Y;
 	HWAngles.Yaw = FAngle::fromDeg(270.0 - Angles.Yaw.Degrees());
+	ViewVector3D.X = v.X * PitchCos;
+	ViewVector3D.Y = v.Y * PitchCos;
+	ViewVector3D.Z = -PitchSin;
 
 	if (IsOrtho() && (camera->ViewPos->Offset.XY().Length() > 0.0))
 	{
diff --git a/src/rendering/r_utility.h b/src/rendering/r_utility.h
index f7e4a1e1d3..d82b03e682 100644
--- a/src/rendering/r_utility.h
+++ b/src/rendering/r_utility.h
@@ -25,6 +25,7 @@ struct FRenderViewpoint
 	DRotator		Angles;			// Camera angles
 	FRotator		HWAngles;		// Actual rotation angles for the hardware renderer
 	DVector2		ViewVector;		// HWR only: direction the camera is facing.
+	DVector3		ViewVector3D;	// 3D direction the camera is facing.
 	AActor			*ViewActor;		// either the same as camera or nullptr
 	FLevelLocals	*ViewLevel;		// The level this viewpoint is on.