From 159409d5d2ac6a63a0c8670b66bfe30d770200de Mon Sep 17 00:00:00 2001
From: Christoph Oelckers <coelckers@users.noreply.github.com>
Date: Thu, 6 Jan 2022 00:12:19 +0100
Subject: [PATCH] - allow a bit of tolerance when sorting walls, so that
 microscopic intersections can be ignored.

---
 source/core/gamefuncs.h                       | 13 ++++
 .../core/rendering/scene/hw_bunchdrawer.cpp   | 59 ++++++++++++++-----
 2 files changed, 56 insertions(+), 16 deletions(-)

diff --git a/source/core/gamefuncs.h b/source/core/gamefuncs.h
index 18051977c..2b46eddda 100644
--- a/source/core/gamefuncs.h
+++ b/source/core/gamefuncs.h
@@ -376,6 +376,19 @@ inline double SquareDistToWall(double px, double py, const walltype* wal, DVecto
 	return SquareDist(px, py, xx, yy);
 }
 
+inline double SquareDistToLine(double px, double py, double lx1, double ly1, double lx2, double ly2)
+{
+	double wall_length = SquareDist(lx1, ly1, lx2, ly2);
+
+	if (wall_length == 0) return SquareDist(px, py, lx1, ly1);
+
+	double t = ((px - lx1) * (lx2 - lx1) + (py - ly1) * (ly2 - ly1)) / wall_length;
+	t = clamp(t, 0., 1.);
+	double xx = lx1 + t * (lx2 - lx1);
+	double yy = ly1 + t * (ly2 - ly1);
+	return SquareDist(px, py, xx, yy);
+}
+
 inline void alignceilslope(sectortype* sect, int x, int y, int z)
 {
 	sect->setceilingslope(getslopeval(sect, x, y, z, sect->ceilingz));
diff --git a/source/core/rendering/scene/hw_bunchdrawer.cpp b/source/core/rendering/scene/hw_bunchdrawer.cpp
index 6034113d1..c1df5b81d 100644
--- a/source/core/rendering/scene/hw_bunchdrawer.cpp
+++ b/source/core/rendering/scene/hw_bunchdrawer.cpp
@@ -389,11 +389,11 @@ int BunchDrawer::WallInFront(int line1, int line2)
 	double x2e = WallStartX(wall2e);
 	double y2e = WallStartY(wall2e);
 
-	double dx = x1e - x1s;
-	double dy = y1e - y1s;
+	double dx1 = x1e - x1s;
+	double dy1 = y1e - y1s;
 
-	double t1 = PointOnLineSide(x2s, y2s, x1s, y1s, dx, dy);
-	double t2 = PointOnLineSide(x2e, y2e, x1s, y1s, dx, dy);
+	double t1 = PointOnLineSide(x2s, y2s, x1s, y1s, dx1, dy1);
+	double t2 = PointOnLineSide(x2e, y2e, x1s, y1s, dx1, dy1);
 	if (t1 == 0)
 	{
 		if (t2 == 0) return(-1);
@@ -403,25 +403,52 @@ int BunchDrawer::WallInFront(int line1, int line2)
 
 	if ((t1 * t2) >= 0)
 	{
-		t2 = PointOnLineSide(viewx, viewy, x1s, y1s, dx, dy);
+		t2 = PointOnLineSide(viewx, viewy, x1s, y1s, dx1, dy1);
 		return((t2 * t1) <= 0);
 	}
 
-	dx = x2e - x2s;
-	dy = y2e - y2s;
-	t1 = PointOnLineSide(x1s, y1s, x2s, y2s, dx, dy);
-	t2 = PointOnLineSide(x1e, y1e, x2s, y2s, dx, dy);
-	if (t1 == 0)
+	double dx2 = x2e - x2s;
+	double dy2 = y2e - y2s;
+	double t3 = PointOnLineSide(x1s, y1s, x2s, y2s, dx2, dy2);
+	double t4 = PointOnLineSide(x1e, y1e, x2s, y2s, dx2, dy2);
+	if (t3 == 0)
 	{
-		if (t2 == 0) return(-1);
-		t1 = t2;
+		if (t4 == 0) return(-1);
+		t3 = t4;
 	}
-	if (t2 == 0) t2 = t1;
-	if ((t1 * t2) >= 0)
+	if (t4 == 0) t4 = t3;
+	if ((t3 * t4) >= 0)
 	{
-		t2 = PointOnLineSide(viewx, viewy, x2s, y2s, dx, dy);
-		return((t2 * t1) > 0);
+		t4 = PointOnLineSide(viewx, viewy, x2s, y2s, dx2, dy2);
+		return((t4 * t3) > 0);
 	}
+
+	// If we got here the walls intersect. Most of the time this is just a tiny sliver intruding into the other wall.
+	// If that is the case we can ignore that sliver and pretend it is completely on the other side.
+
+	const double max_dist = 3;
+	const double side_threshold = (max_dist * max_dist) / (16. * 16.);	// we are operating in render coordinate space but want 3 map units tolerance.
+
+	double d1 = SquareDistToLine(x2s, y2s, x1s, y1s, x1e, y1e);
+	if (d1 < side_threshold) t1 = t2; 
+	double d2 = SquareDistToLine(x2e, y2e, x1s, y1s, x1e, y1e);
+	if (d2 < side_threshold) t2 = t1;
+	if ((fabs(d1) < side_threshold) ^ (fabs(d2) < side_threshold)) // only acceptable if only one end of the wall got adjusted.
+	{
+		t2 = PointOnLineSide(viewx, viewy, x1s, y1s, dx1, dy1);
+		return((t2 * t1) <= 0);
+	}
+
+	double d3 = SquareDistToLine(x1s, y1s, x2s, y2s, x2e, y2e);
+	if (d3 < side_threshold) t1 = t2;
+	double d4 = SquareDistToLine(x1e, y1e, x2s, y2s, x2e, y2e);
+	if (d4 < side_threshold) t2 = t1;
+	if ((fabs(d3) < side_threshold) ^ (fabs(d4) < side_threshold)) // only acceptable if only one end of the wall got adjusted.
+	{
+		t2 = PointOnLineSide(viewx, viewy, x2s, y2s, dx2, dy2);
+		return((t2 * t1) <= 0);
+	}
+
 	return(-2);
 }