From 3b0f19a57fafbb487894b60a2b45c83ec2ea653a Mon Sep 17 00:00:00 2001
From: RaveYard <29225776+MrRaveYard@users.noreply.github.com>
Date: Sat, 2 Sep 2023 16:49:37 +0200
Subject: [PATCH] Port BuildSmoothingGroups from ZDray

---
 .../rendering/hwrenderer/data/hw_levelmesh.h  | 47 +++++++++++++++++++
 src/rendering/hwrenderer/doom_levelmesh.cpp   | 12 +----
 2 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/src/common/rendering/hwrenderer/data/hw_levelmesh.h b/src/common/rendering/hwrenderer/data/hw_levelmesh.h
index 7f23345d51..9ac563f7aa 100644
--- a/src/common/rendering/hwrenderer/data/hw_levelmesh.h
+++ b/src/common/rendering/hwrenderer/data/hw_levelmesh.h
@@ -56,6 +56,8 @@ struct LevelMeshSurface
 	// Required for internal lightmapper:
 	//
 
+	int sectorGroup = 0;
+
 	BBox bounds;
 	int sampleDimension = 0;
 
@@ -183,4 +185,49 @@ public:
 		FVector3 end = start + direction * std::max(maxDist - 10.0f, 0.0f);
 		return !TriangleMeshShape::find_any_hit(Collision.get(), start, end);
 	}
+
+	void BuildSmoothingGroups()
+	{
+		for (size_t i = 0, count = GetSurfaceCount(); i < count; i++)
+		{
+			auto surface = GetSurface(i);
+
+			// Is this surface in the same plane as an existing smoothing group?
+			int smoothingGroupIndex = -1;
+
+			for (size_t j = 0; j < SmoothingGroups.Size(); j++)
+			{
+				if (surface->sectorGroup == SmoothingGroups[j].sectorGroup)
+				{
+					float direction = std::abs((SmoothingGroups[j].plane.XYZ() | surface->plane.XYZ()));
+					if (direction >= 0.9999f && direction <= 1.001f)
+					{
+						auto point = (surface->plane.XYZ() * surface->plane.W);
+						auto planeDistance = (SmoothingGroups[j].plane.XYZ() | point) - SmoothingGroups[j].plane.W;
+
+						float dist = std::abs(planeDistance);
+						if (dist <= 0.01f)
+						{
+							smoothingGroupIndex = (int)j;
+							break;
+						}
+					}
+				}
+			}
+
+			// Surface is in a new plane. Create a smoothing group for it
+			if (smoothingGroupIndex == -1)
+			{
+				smoothingGroupIndex = SmoothingGroups.Size();
+
+				LevelMeshSmoothingGroup group;
+				group.plane = surface->plane;
+				group.sectorGroup = surface->sectorGroup;
+				SmoothingGroups.Push(group);
+			}
+
+			SmoothingGroups[smoothingGroupIndex].surfaces.push_back(surface);
+			surface->smoothingGroupIndex = smoothingGroupIndex;
+		}
+	}
 };
diff --git a/src/rendering/hwrenderer/doom_levelmesh.cpp b/src/rendering/hwrenderer/doom_levelmesh.cpp
index e467ffca7e..81c5f78fba 100644
--- a/src/rendering/hwrenderer/doom_levelmesh.cpp
+++ b/src/rendering/hwrenderer/doom_levelmesh.cpp
@@ -627,17 +627,7 @@ void DoomLevelMesh::SetupLightmapUvs()
 		surface.texPixels.resize(surface.texWidth * surface.texHeight);
 	}
 
-	{
-		LevelMeshSmoothingGroup smoothing;
-
-		for (auto& surface : Surfaces)
-		{
-			surface.smoothingGroupIndex = 0;
-
-			smoothing.surfaces.push_back(&surface);
-		}
-		SmoothingGroups.Push(std::move(smoothing));
-	}
+	BuildSmoothingGroups();
 
 	std::sort(sortedSurfaces.begin(), sortedSurfaces.end(), [](LevelMeshSurface* a, LevelMeshSurface* b) { return a->texHeight != b->texHeight ? a->texHeight > b->texHeight : a->texWidth > b->texWidth; });