From 114b5c68d25104a249e74f6f759791d1b16a3010 Mon Sep 17 00:00:00 2001
From: RaveYard <MrRaveYard@gmail.com>
Date: Wed, 22 Jun 2022 13:21:06 +0200
Subject: [PATCH] Fix certain broken normals

---
 src/lightmap/levelmesh.cpp |  8 ++++----
 src/math/mathlib.h         |  3 ++-
 src/math/plane.cpp         | 31 +++++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/lightmap/levelmesh.cpp b/src/lightmap/levelmesh.cpp
index 3bb6ca1..2107317 100644
--- a/src/lightmap/levelmesh.cpp
+++ b/src/lightmap/levelmesh.cpp
@@ -539,7 +539,7 @@ void LevelMesh::CreateSideSurfaces(FLevel &doomMap, IntSideDef *side)
 			surf->verts[1].z = xfloor->floorplane.zAt(v1.x, v1.y);
 			surf->verts[2].z = xfloor->ceilingplane.zAt(v2.x, v2.y);
 			surf->verts[3].z = xfloor->ceilingplane.zAt(v1.x, v1.y);
-			surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2]);
+			surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2], surf->verts[3]);
 			surf->plane.SetDistance(surf->verts[0]);
 
 			float texZ = surf->verts[0].z;
@@ -589,7 +589,7 @@ void LevelMesh::CreateSideSurfaces(FLevel &doomMap, IntSideDef *side)
 				surf->verts[2].z = v1BottomBack;
 				surf->verts[3].z = v2BottomBack;
 
-				surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2]);
+				surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2], surf->verts[3]);
 				surf->plane.SetDistance(surf->verts[0]);
 				surf->type = ST_LOWERSIDE;
 				surf->typeIndex = typeIndex;
@@ -647,7 +647,7 @@ void LevelMesh::CreateSideSurfaces(FLevel &doomMap, IntSideDef *side)
 				surf->verts[2].z = v1Top;
 				surf->verts[3].z = v2Top;
 
-				surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2]);
+				surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2], surf->verts[3]);
 				surf->plane.SetDistance(surf->verts[0]);
 				surf->type = ST_UPPERSIDE;
 				surf->typeIndex = typeIndex;
@@ -695,7 +695,7 @@ void LevelMesh::CreateSideSurfaces(FLevel &doomMap, IntSideDef *side)
 		surf->verts[2].z = v1Top;
 		surf->verts[3].z = v2Top;
 
-		surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2]);
+		surf->plane.SetNormal(surf->verts[0], surf->verts[1], surf->verts[2], surf->verts[3]);
 		surf->plane.SetDistance(surf->verts[0]);
 		surf->type = ST_MIDDLESIDE;
 		surf->typeIndex = typeIndex;
diff --git a/src/math/mathlib.h b/src/math/mathlib.h
index 6182934..d38b978 100644
--- a/src/math/mathlib.h
+++ b/src/math/mathlib.h
@@ -98,7 +98,8 @@ public:
 	const vec3 &Normal() const;
 	vec3 &Normal();
 	Plane &SetNormal(const vec3 &normal);
-	Plane &SetNormal(const vec3 &pt1, const vec3 &pt2, const vec3 &pt3);
+	Plane& SetNormal(const vec3& pt1, const vec3& pt2, const vec3& pt3);
+	Plane& SetNormal(const vec3& v1, const vec3& v2, const vec3& v3, const vec3& v4);
 	float Distance(const vec3 &point);
 	Plane &SetDistance(const vec3 &point);
 	bool IsFacing(const float yaw);
diff --git a/src/math/plane.cpp b/src/math/plane.cpp
index ee4c483..528cd8e 100644
--- a/src/math/plane.cpp
+++ b/src/math/plane.cpp
@@ -86,6 +86,37 @@ Plane &Plane::SetNormal(const vec3 &pt1, const vec3 &pt2, const vec3 &pt3)
 	return *this;
 }
 
+Plane& Plane::SetNormal(const vec3& v1, const vec3& v2, const vec3& v3, const vec3& v4)
+{
+	// Handle cases where two vertices share the same position
+
+	// TODO fix this hack
+	// Certain vertex positions are close enough to cause NaN, but aren't equal
+	auto iv1 = ivec3(v1);
+	auto iv2 = ivec3(v2);
+	auto iv3 = ivec3(v3);
+	auto iv4 = ivec3(v4);
+
+	if (iv1 == iv3)
+	{
+		return SetNormal(v1, v2, v4);
+	}
+	else if (iv2 == iv4)
+	{
+		return SetNormal(v1, v2, v3);
+	}
+	else if (iv1 == iv2)
+	{
+		return SetNormal(v2, v3, v4);
+	}
+	else if (iv2 == iv3)
+	{
+		return SetNormal(v1, v2, v4);
+	}
+
+	return SetNormal(v1, v2, v3);
+}
+
 vec3 const &Plane::Normal() const
 {
 	return *reinterpret_cast<const vec3*>(&a);