From efeab91a75fdaee6ff358587de1f4fd99ba1b6bf Mon Sep 17 00:00:00 2001
From: RaveYard <29225776+MrRaveYard@users.noreply.github.com>
Date: Tue, 5 Sep 2023 15:47:48 +0200
Subject: [PATCH] Sector portals sunlight only

---
 .../rendering/hwrenderer/data/hw_levelmesh.h  | 23 +++++++++-
 .../vulkan/accelstructs/vk_raytrace.cpp       |  2 +-
 src/maploader/maploader.cpp                   |  2 +
 src/rendering/hwrenderer/doom_levelmesh.cpp   | 43 +++++++++++++++++++
 src/rendering/hwrenderer/doom_levelmesh.h     |  1 +
 5 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/src/common/rendering/hwrenderer/data/hw_levelmesh.h b/src/common/rendering/hwrenderer/data/hw_levelmesh.h
index 885c33f290..f7343c1069 100644
--- a/src/common/rendering/hwrenderer/data/hw_levelmesh.h
+++ b/src/common/rendering/hwrenderer/data/hw_levelmesh.h
@@ -55,7 +55,9 @@ struct LevelMeshSurface
 	//
 	// Required for internal lightmapper:
 	//
-
+	
+	// int portalDestinationIndex = -1; // line or sector index
+	int portalIndex = 0;
 	int sectorGroup = 0;
 
 	BBox bounds;
@@ -146,6 +148,25 @@ struct LevelMeshPortal
 	}
 };
 
+// for use with std::set to recursively go through portals and skip returning portals
+struct RecursivePortalComparator
+{
+	bool operator()(const LevelMeshPortal& a, const LevelMeshPortal& b) const
+	{
+		return !a.IsInversePortal(b) && std::memcmp(&a.transformation, &b.transformation, sizeof(VSMatrix)) < 0;
+	}
+};
+
+// for use with std::map to reject portals which have the same effect for light rays
+struct IdenticalPortalComparator
+{
+	bool operator()(const LevelMeshPortal& a, const LevelMeshPortal& b) const
+	{
+		return !a.IsEqualPortal(b) && std::memcmp(&a.transformation, &b.transformation, sizeof(VSMatrix)) < 0;
+	}
+};
+
+
 class LevelMesh
 {
 public:
diff --git a/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp b/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp
index 53f67b85c4..caa4fcd4dd 100644
--- a/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp
+++ b/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp
@@ -121,7 +121,7 @@ void VkRaytrace::CreateBuffers()
 
 		SurfaceInfo info;
 		info.Normal = surface->plane.XYZ();
-		info.PortalIndex = 0;
+		info.PortalIndex = surface->portalIndex;
 		info.SamplingDistance = (float)surface->sampleDimension;
 		info.Sky = surface->bSky;
 		surfaceInfo.Push(info);
diff --git a/src/maploader/maploader.cpp b/src/maploader/maploader.cpp
index 4eeefd0268..c029e871d0 100644
--- a/src/maploader/maploader.cpp
+++ b/src/maploader/maploader.cpp
@@ -3294,5 +3294,7 @@ void MapLoader::LoadLevel(MapData *map, const char *lumpname, int position)
 	if (!Level->IsReentering())
 		Level->FinalizePortals();	// finalize line portals after polyobjects have been initialized. This info is needed for properly flagging them.
 
+	Level->levelMesh->CreatePortals(); // [RaveYard]: needs portal data, but at the same time intializing the level mesh here breaks floor/ceiling planes!
+
 	Level->aabbTree = new DoomLevelAABBTree(Level);
 }
diff --git a/src/rendering/hwrenderer/doom_levelmesh.cpp b/src/rendering/hwrenderer/doom_levelmesh.cpp
index f089e5c18e..b3ac461cb2 100644
--- a/src/rendering/hwrenderer/doom_levelmesh.cpp
+++ b/src/rendering/hwrenderer/doom_levelmesh.cpp
@@ -86,6 +86,49 @@ DoomLevelMesh::DoomLevelMesh(FLevelLocals &doomMap)
 	Collision = std::make_unique<TriangleMeshShape>(MeshVertices.Data(), MeshVertices.Size(), MeshElements.Data(), MeshElements.Size());
 }
 
+void DoomLevelMesh::CreatePortals()
+{
+	std::map<LevelMeshPortal, int, IdenticalPortalComparator> transformationIndices; // TODO use the list of portals from the level to avoids duplicates?
+	transformationIndices.emplace(LevelMeshPortal{}, 0); // first portal is an identity matrix
+
+	for (auto& surface : Surfaces)
+	{
+		auto displacement = [&]() {
+			if (surface.Type == ST_FLOOR || surface.Type == ST_CEILING)
+			{
+				auto d = surface.Subsector->sector->GetPortalDisplacement(surface.Type == ST_FLOOR ? sector_t::floor : sector_t::ceiling);
+				return FVector3(d.X, d.Y, 0);
+			}
+			else if(surface.Type == ST_MIDDLESIDE)
+			{
+				auto d = surface.Side->linedef->getPortalDisplacement();
+				return FVector3(d.X, d.Y, 0);
+			}
+			return FVector3(0, 0, 0);
+		}();
+
+		if (!displacement.isZero())
+		{
+			LevelMeshPortal transformation;
+			transformation.transformation.translate(displacement.X, displacement.Y, displacement.Z);
+
+			auto& index = transformationIndices[transformation];
+
+			if (index == 0) // new transformation was created
+			{
+				index = Portals.Size();
+				Portals.Push(transformation);
+			}
+
+			surface.portalIndex = index;
+		}
+		else
+		{
+			surface.portalIndex = 0;
+		}
+	}
+}
+
 void DoomLevelMesh::UpdateLightLists()
 {
 	for (auto& surface : Surfaces)
diff --git a/src/rendering/hwrenderer/doom_levelmesh.h b/src/rendering/hwrenderer/doom_levelmesh.h
index c895d1bf70..93ff54407a 100644
--- a/src/rendering/hwrenderer/doom_levelmesh.h
+++ b/src/rendering/hwrenderer/doom_levelmesh.h
@@ -52,6 +52,7 @@ public:
 
 	void SetupLightmapUvs();
 
+	void CreatePortals();
 private:
 	void CreateSubsectorSurfaces(FLevelLocals &doomMap);
 	void CreateCeilingSurface(FLevelLocals &doomMap, subsector_t *sub, sector_t *sector, int typeIndex, bool is3DFloor);