Fix smoothing groups causing pixels to bleed over portals

src/level/level.cpp

@@ -692,11 +692,16 @@ void FLevel::PostLoadInitialization()
 		}
 	}
 
-	// Discover islands of sectors
+	// Discover sector groups (graph islands)
 	{
 		int groupId = 0;
 		std::vector<IntSector*> candidates;
 
+		auto canPass = [&](IntLineDef* line) {
+			// Further conditions can be added to further split the map into groups
+			return line->special == 0 || (line->special != Line_SetPortal && line->special != Line_Horizon);
+		};
+
 		for (auto& sector : Sectors)
 		{
 			if (!sector.group)
@@ -711,15 +716,18 @@ void FLevel::PostLoadInitialization()
 
 					for (const auto& line : sector->lines)
 					{
-						if (line->frontsector && !line->frontsector->group)
+						if (canPass(line))
 						{
-							line->frontsector->group = groupId;
+							if (line->frontsector && !line->frontsector->group)
-							candidates.push_back(line->frontsector);
+							{
-						}
+								line->frontsector->group = groupId;
-						if (line->backsector && !line->backsector->group)
+								candidates.push_back(line->frontsector);
-						{
+							}
-							line->backsector->group = groupId;
+							if (line->backsector && !line->backsector->group)
-							candidates.push_back(line->backsector);
+							{
+								line->backsector->group = groupId;
+								candidates.push_back(line->backsector);
+							}
 						}
 					}
 				}

src/lightmap/levelmesh.cpp

@@ -60,6 +60,13 @@ LevelMesh::LevelMesh(FLevel &doomMap, int sampleDistance, int textureSize)
 
 	printf("Surfaces total: %i\n\n", (int)surfaces.size());
 
+	// Update sector group of the surfacesaa
+	for (auto& surface : surfaces)
+	{
+		surface->sectorGroup = surface->type == ST_CEILING || surface->type == ST_FLOOR ?
+			doomMap.GetSectorFromSubSector(&doomMap.GLSubsectors[surface->typeIndex])->group : (doomMap.Sides[surface->typeIndex].GetSectorGroup());
+	}
+
 	printf("Building level mesh...\n\n");
 
 	for (size_t i = 0; i < surfaces.size(); i++)
@@ -112,37 +119,7 @@ LevelMesh::LevelMesh(FLevel &doomMap, int sampleDistance, int textureSize)
 	}
 
 	printf("Finding smoothing groups...\n\n");
-
+	BuildSmoothingGroups(doomMap);
-	for (size_t i = 0; i < surfaces.size(); i++)
-	{
-		// Is this surface in the same plane as an existing smoothing group?
-		int smoothingGroupIndex = -1;
-		for (size_t j = 0; j < smoothingGroups.size(); j++)
-		{
-			float direction = std::abs(dot(smoothingGroups[j].Normal(), surfaces[i]->plane.Normal()));
-			if (direction >= 0.9999f && direction <= 1.001f)
-			{
-				float dist = std::abs(smoothingGroups[j].Distance(surfaces[i]->plane.Normal() * surfaces[i]->plane.d));
-				if (dist <= 0.01f)
-				{
-					smoothingGroupIndex = (int)j;
-					break;
-				}
-			}
-		}
-
-		// Surface is in a new plane. Create a smoothing group for it
-		if (smoothingGroupIndex == -1)
-		{
-			smoothingGroupIndex = smoothingGroups.size();
-			smoothingGroups.push_back(surfaces[i]->plane);
-		}
-
-		surfaces[i]->smoothingGroupIndex = smoothingGroupIndex;
-	}
-
-	printf("Created %d smoothing groups for %d surfaces\n\n", (int)smoothingGroups.size(), (int)surfaces.size());
-
 	printf("Building collision data...\n\n");
 
 	Collision = std::make_unique<TriangleMeshShape>(MeshVertices.Data(), MeshVertices.Size(), MeshElements.Data(), MeshElements.Size());
@@ -158,6 +135,50 @@ LevelMesh::LevelMesh(FLevel &doomMap, int sampleDistance, int textureSize)
 	*/
 }
 
+void LevelMesh::BuildSmoothingGroups(FLevel& doomMap)
+{
+	for (size_t i = 0; i < surfaces.size(); i++)
+	{
+		// Is this surface in the same plane as an existing smoothing group?
+		int smoothingGroupIndex = -1;
+
+		auto surface = surfaces[i].get();
+
+		for (size_t j = 0; j < smoothingGroups.size(); j++)
+		{
+			if (surface->sectorGroup == smoothingGroups[j].sectorGroup)
+			{
+				float direction = std::abs(dot(smoothingGroups[j].plane.Normal(), surface->plane.Normal()));
+				if (direction >= 0.9999f && direction <= 1.001f)
+				{
+					float dist = std::abs(smoothingGroups[j].plane.Distance(surface->plane.Normal() * surface->plane.d));
+					if (dist <= 0.01f)
+					{
+						smoothingGroupIndex = (int)j;
+						break;
+					}
+				}
+			}
+		}
+
+		// Surface is in a new plane. Create a smoothing group for it
+		if (smoothingGroupIndex == -1)
+		{
+			smoothingGroupIndex = smoothingGroups.size();
+
+			SmoothingGroup group;
+			group.plane = surface->plane;
+			group.sectorGroup = surface->sectorGroup;
+			smoothingGroups.push_back(group);
+		}
+
+		surface->smoothingGroupIndex = smoothingGroupIndex;
+	}
+
+	printf("Created %d smoothing groups for %d surfaces\n\n", (int)smoothingGroups.size(), (int)surfaces.size());
+}
+
+
 #include <set> // hack
 std::set<Portal, RejectRecursivePortals> touchedPortals;
 
@@ -179,12 +200,8 @@ void LevelMesh::PropagateLight(FLevel& doomMap, ThingLight *light)
 	for (int triangleIndex : TriangleMeshShape::find_all_hits(Collision.get(), &sphere))
 	{
 		Surface* surface = surfaces[MeshSurfaces[triangleIndex]].get();
-		
-		auto surfaceSectorGroup = surface->type == ST_CEILING || surface->type == ST_FLOOR ?
-			doomMap.GetSectorFromSubSector(&doomMap.GLSubsectors[surface->typeIndex])->group : (doomMap.Sides[surface->typeIndex].GetSectorGroup());
-
 		// Reject any surface which isn't physically connected to the sector group in which the light resided
-		if (light->sectorGroup == surfaceSectorGroup)
+		if (light->sectorGroup == surface->sectorGroup)
 		{
 			if (surface->portalIndex >= 0)
 			{			

src/lightmap/levelmesh.h

@@ -83,6 +83,9 @@ struct Surface
 	// Portal
 	int portalDestinationIndex = -1; // line or sector index
 	int portalIndex = -1;
+	
+	// Sector group
+	int sectorGroup = 0;
 
 	// Touching light sources
 	std::vector<ThingLight*> LightList;
@@ -114,6 +117,12 @@ struct Surface
 	int smoothingGroupIndex = -1;
 };
 
+struct SmoothingGroup
+{
+	Plane plane = Plane(0, 0, 1, 0);
+	int sectorGroup = 0;
+};
+
 class LevelMesh
 {
 public:
@@ -129,7 +138,7 @@ public:
 
 	std::vector<std::unique_ptr<LightmapTexture>> textures;
 
-	std::vector<Plane> smoothingGroups;
+	std::vector<SmoothingGroup> smoothingGroups;
 
 	std::vector<std::unique_ptr<Portal>> portals;
 
@@ -159,6 +168,8 @@ private:
 	void BuildLightLists(FLevel &doomMap);
 	void PropagateLight(FLevel& doomMap, ThingLight* thing);
 
+	void BuildSmoothingGroups(FLevel& doomMap);
+
 	void BlurSurfaces();
 	void FinishSurface(RectPacker& packer, Surface* surface);