Sort surfaces into smoothing groups and render the entire group into a surface tile

Extend the raytracing of a surface tile by one to get better transitions
Fix Plane.Distance(point) not actually returning the distance to the point!

src/lightmap/glsl_frag.h

@@ -48,8 +48,6 @@ layout(push_constant) uniform PushConstants
 	uint LightEnd;
 	int SurfaceIndex;
 	int PushPadding1;
-	vec2 TileTL;
-	vec2 TileBR;
 	vec3 LightmapOrigin;
 	float PushPadding2;
 	vec3 LightmapStepX;

src/lightmap/glsl_vert.h

@@ -8,8 +8,6 @@ layout(push_constant) uniform PushConstants
 	uint LightEnd;
 	int SurfaceIndex;
 	int PushPadding1;
-	vec2 TileTL;
-	vec2 TileBR;
 	vec3 LightmapOrigin;
 	float PushPadding2;
 	vec3 LightmapStepX;

src/lightmap/gpuraytracer2.cpp

@@ -59,16 +59,17 @@ void GPURaytracer2::Raytrace(LevelMesh* level)
 		.AddBuffer(uniformBuffer.get(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT)
 		.Execute(cmdbuffer.get(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
 
-	const int atlasImageSize = 512;
-	RectPacker packer(atlasImageSize, atlasImageSize, RectPacker::Spacing(0));
+	const int atlasImageSize = 2048;
+	const int spacing = 3; // Note: the spacing is here to avoid that the resolve sampler finds data from other surface tiles
+	RectPacker packer(atlasImageSize, atlasImageSize, RectPacker::Spacing(spacing));
 
 	for (size_t i = 0; i < mesh->surfaces.size(); i++)
 	{
 		Surface* surface = mesh->surfaces[i].get();
 
-		auto result = packer.insert(surface->texWidth, surface->texHeight);
-		surface->atlasX = result.pos.x;
-		surface->atlasY = result.pos.y;
+		auto result = packer.insert(surface->texWidth + 2, surface->texHeight + 2);
+		surface->atlasX = result.pos.x + 1;
+		surface->atlasY = result.pos.y + 1;
 		surface->atlasPageIndex = (int)result.pageIndex;
 	}
 
@@ -95,114 +96,81 @@ void GPURaytracer2::Raytrace(LevelMesh* level)
 
 		for (size_t i = 0; i < mesh->surfaces.size(); i++)
 		{
-			Surface* surface = mesh->surfaces[i].get();
-			if (surface->atlasPageIndex != pageIndex)
+			Surface* targetSurface = mesh->surfaces[i].get();
+			if (targetSurface->atlasPageIndex != pageIndex)
 				continue;
 
-			int sampleWidth = surface->texWidth;
-			int sampleHeight = surface->texHeight;
-
-			int firstLight = sceneLightPos;
-			int lightCount = (int)surface->LightList.size();
-			sceneLightPos += lightCount;
-
-			LightInfo2* lightinfo = &sceneLights[firstLight];
-			for (ThingLight* light : surface->LightList)
-			{
-				lightinfo->Origin = light->LightOrigin();
-				lightinfo->Radius = light->LightRadius();
-				lightinfo->Intensity = light->intensity;
-				lightinfo->InnerAngleCos = light->innerAngleCos;
-				lightinfo->OuterAngleCos = light->outerAngleCos;
-				lightinfo->SpotDir = light->SpotDir();
-				lightinfo->Color = light->rgb;
-				lightinfo++;
-			}
-
 			VkViewport viewport = {};
 			viewport.maxDepth = 1;
-			viewport.x = (float)surface->atlasX;
-			viewport.y = (float)surface->atlasY;
-			viewport.width = (float)sampleWidth;
-			viewport.height = (float)sampleHeight;
+			viewport.x = (float)targetSurface->atlasX - 1;
+			viewport.y = (float)targetSurface->atlasY - 1;
+			viewport.width = (float)(targetSurface->texWidth + 2);
+			viewport.height = (float)(targetSurface->texHeight + 2);
 			cmdbuffer->setViewport(0, 1, &viewport);
 
-			PushConstants2 pc;
-			pc.LightStart = firstLight;
-			pc.LightEnd = firstLight + lightCount;
-			pc.SurfaceIndex = (int32_t)i;
-			pc.TileTL = vec2(0.0f);
-			pc.TileBR = vec2(1.0f);
-			pc.LightmapOrigin = surface->worldOrigin;
-			pc.LightmapStepX = surface->worldStepX * (float)sampleWidth;
-			pc.LightmapStepY = surface->worldStepY * (float)sampleHeight;
-			cmdbuffer->pushConstants(raytrace.pipelineLayout.get(), VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(PushConstants2), &pc);
-
-			if (vertexList.size() < surface->lightUV.size())
-				vertexList.resize(surface->lightUV.size());
-
-#if 1
-			// Draw tile
-			int firstVertex = sceneVertexPos;
-			int vertexCount = 4;
-			sceneVertexPos += vertexCount;
-
-			SceneVertex* vertex = &sceneVertices[firstVertex];
-			vertex[0].Position = vec2(0.0f, 0.0f);
-			vertex[1].Position = vec2(1.0f, 0.0f);
-			vertex[2].Position = vec2(1.0f, 1.0f);
-			vertex[3].Position = vec2(0.0f, 1.0f);
-
-			cmdbuffer->draw(vertexCount, 1, firstVertex, 0);
-#else
-			// Convert to triangle fan and correct orientation
-			int vertexCount = (int)surface->lightUV.size();
-			if (surface->type == ST_FLOOR || surface->type == ST_CEILING)
+			// Paint all surfaces part of the smoothing group into the surface
+			for (const auto& surface : mesh->surfaces)
 			{
-				if (IsNegativelyOriented(surface->lightUV[0], surface->lightUV[1], surface->lightUV[2]))
+				if (surface->smoothingGroupIndex != targetSurface->smoothingGroupIndex)
+					continue;
+
+				vec2 minUV = ToUV(surface->bounds.min, targetSurface);
+				vec2 maxUV = ToUV(surface->bounds.max, targetSurface);
+				if (surface.get() != targetSurface && (maxUV.x < 0.0f || maxUV.y < 0.0f || minUV.x > 1.0f || minUV.y > 1.0f))
+					continue; // Bounding box not visible
+
+				int firstLight = sceneLightPos;
+				int lightCount = (int)surface->LightList.size();
+				if (sceneLightPos + lightCount > SceneLightBufferSize)
+					throw std::runtime_error("SceneLightBuffer is too small!");
+				sceneLightPos += lightCount;
+
+				LightInfo2* lightinfo = &sceneLights[firstLight];
+				for (ThingLight* light : surface->LightList)
+				{
+					lightinfo->Origin = light->LightOrigin();
+					lightinfo->Radius = light->LightRadius();
+					lightinfo->Intensity = light->intensity;
+					lightinfo->InnerAngleCos = light->innerAngleCos;
+					lightinfo->OuterAngleCos = light->outerAngleCos;
+					lightinfo->SpotDir = light->SpotDir();
+					lightinfo->Color = light->rgb;
+					lightinfo++;
+				}
+
+				PushConstants2 pc;
+				pc.LightStart = firstLight;
+				pc.LightEnd = firstLight + lightCount;
+				pc.SurfaceIndex = (int32_t)i;
+				pc.LightmapOrigin = targetSurface->worldOrigin - targetSurface->worldStepX - targetSurface->worldStepY;
+				pc.LightmapStepX = targetSurface->worldStepX * viewport.width;
+				pc.LightmapStepY = targetSurface->worldStepY * viewport.height;
+				cmdbuffer->pushConstants(raytrace.pipelineLayout.get(), VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(PushConstants2), &pc);
+
+				int firstVertex = sceneVertexPos;
+				int vertexCount = (int)surface->verts.size();
+				if (sceneVertexPos + vertexCount > SceneVertexBufferSize)
+					throw std::runtime_error("SceneVertexBuffer is too small!");
+				sceneVertexPos += vertexCount;
+				SceneVertex* vertex = &sceneVertices[firstVertex];
+
+				if (surface->type == ST_FLOOR || surface->type == ST_CEILING)
 				{
 					for (int idx = 0; idx < vertexCount; idx++)
 					{
-						vertexList[idx] = surface->lightUV[idx];
+						(vertex++)->Position = ToUV(surface->verts[idx], targetSurface);
 					}
 				}
 				else
 				{
-					for (int idx = 0; idx < vertexCount; idx++)
-					{
-						vertexList[idx] = surface->lightUV[vertexCount - 1 - idx];
-					}
+					(vertex++)->Position = ToUV(surface->verts[0], targetSurface);
+					(vertex++)->Position = ToUV(surface->verts[2], targetSurface);
+					(vertex++)->Position = ToUV(surface->verts[3], targetSurface);
+					(vertex++)->Position = ToUV(surface->verts[1], targetSurface);
 				}
-			}
-			else
-			{
-				if (IsNegativelyOriented(surface->lightUV[0], surface->lightUV[2], surface->lightUV[3]))
-				{
-					vertexList[0] = surface->lightUV[0];
-					vertexList[1] = surface->lightUV[2];
-					vertexList[2] = surface->lightUV[3];
-					vertexList[3] = surface->lightUV[1];
-				}
-				else
-				{
-					vertexList[0] = surface->lightUV[1];
-					vertexList[1] = surface->lightUV[3];
-					vertexList[2] = surface->lightUV[2];
-					vertexList[3] = surface->lightUV[0];
-				}
-			}
 
-			// Draw polygon
-			int firstVertex = sceneVertexPos;
-			sceneVertexPos += vertexCount;
-			SceneVertex* vertex = &sceneVertices[firstVertex];
-			for (int idx = 0; idx < vertexCount; idx++)
-			{
-				auto& uv = vertexList[idx];
-				(vertex++)->Position = vec2(uv.x / sampleWidth, uv.y / sampleHeight);
+				cmdbuffer->draw(vertexCount, 1, firstVertex, 0);
 			}
-			cmdbuffer->draw(vertexCount, 1, firstVertex, 0);
-#endif
 		}
 		cmdbuffer->endRenderPass();
 	}
@@ -243,8 +211,6 @@ void GPURaytracer2::Raytrace(LevelMesh* level)
 		pc.LightStart = 0;
 		pc.LightEnd = 0;
 		pc.SurfaceIndex = 0;
-		pc.TileTL = vec2(0.0f);
-		pc.TileBR = vec2(1.0f);
 		pc.LightmapOrigin = vec3(0.0f);
 		pc.LightmapStepX = vec3(0.0f);
 		pc.LightmapStepY = vec3(0.0f);
@@ -330,6 +296,14 @@ void GPURaytracer2::Raytrace(LevelMesh* level)
 	printf("Ray trace complete\n");
 }
 
+vec2 GPURaytracer2::ToUV(const vec3& vert, const Surface* targetSurface)
+{
+	vec3 localPos = vert - targetSurface->translateWorldToLocal;
+	float u = (1.0f + dot(localPos, targetSurface->projLocalToU)) / (targetSurface->texWidth + 2);
+	float v = (1.0f + dot(localPos, targetSurface->projLocalToV)) / (targetSurface->texHeight + 2);
+	return vec2(u, v);
+}
+
 void GPURaytracer2::CreateVulkanObjects()
 {
 	submitFence = std::make_unique<VulkanFence>(device.get());

src/lightmap/gpuraytracer2.h

@@ -20,8 +20,6 @@ struct PushConstants2
 	uint32_t LightEnd;
 	int32_t SurfaceIndex;
 	int32_t PushPadding1;
-	vec2 TileTL;
-	vec2 TileBR;
 	vec3 LightmapOrigin;
 	float PushPadding2;
 	vec3 LightmapStepX;
@@ -108,6 +106,7 @@ private:
 
 	std::vector<SurfaceInfo2> CreateSurfaceInfo();
 
+	static vec2 ToUV(const vec3& vert, const Surface* targetSurface);
 	static bool IsNegativelyOriented(const vec2& v1, const vec2& v2, const vec2& v3);
 
 	LevelMesh* mesh = nullptr;
@@ -124,7 +123,7 @@ private:
 	SceneVertex* sceneVertices = nullptr;
 	int sceneVertexPos = 0;
 
-	static const int SceneLightBufferSize = 1 * 1024 * 1024;
+	static const int SceneLightBufferSize = 2 * 1024 * 1024;
 	std::unique_ptr<VulkanBuffer> sceneLightBuffer;
 	LightInfo2* sceneLights = nullptr;
 	int sceneLightPos = 0;
@@ -176,6 +175,4 @@ private:
 	std::unique_ptr<VulkanFence> submitFence;
 	std::unique_ptr<VulkanCommandPool> cmdpool;
 	std::unique_ptr<VulkanCommandBuffer> cmdbuffer;
-
-	std::vector<vec2> vertexList;
 };

src/lightmap/levelmesh.cpp

@@ -113,6 +113,38 @@ LevelMesh::LevelMesh(FLevel &doomMap, int sampleDistance, int textureSize)
 		BuildSurfaceParams(surfaces[i].get());
 	}
 
+	printf("Finding smoothing groups...\n\n");
+
+	for (size_t i = 0; i < surfaces.size(); i++)
+	{
+		// Is this surface in the same plane as an existing smoothing group?
+		int smoothingGroupIndex = -1;
+		for (size_t j = 0; j < smoothingGroups.size(); j++)
+		{
+			float direction = std::abs(dot(smoothingGroups[j].Normal(), surfaces[i]->plane.Normal()));
+			if (direction >= 0.9999f && direction <= 1.001f)
+			{
+				float dist = std::abs(smoothingGroups[j].Distance(surfaces[i]->plane.Normal() * surfaces[i]->plane.d));
+				if (dist <= 0.01f)
+				{
+					smoothingGroupIndex = (int)j;
+					break;
+				}
+			}
+		}
+
+		// Surface is in a new plane. Create a smoothing group for it
+		if (smoothingGroupIndex == -1)
+		{
+			smoothingGroupIndex = smoothingGroups.size();
+			smoothingGroups.push_back(surfaces[i]->plane);
+		}
+
+		surfaces[i]->smoothingGroupIndex = smoothingGroupIndex;
+	}
+
+	printf("Created %d smoothing groups for %d surfaces\n\n", (int)smoothingGroups.size(), (int)surfaces.size());
+
 	printf("Building collision data...\n\n");
 
 	Collision = std::make_unique<TriangleMeshShape>(MeshVertices.Data(), MeshVertices.Size(), MeshElements.Data(), MeshElements.Size());
@@ -160,6 +192,7 @@ void LevelMesh::BuildSurfaceParams(Surface* surface)
 
 	plane = &surface->plane;
 	bounds = GetBoundsFromSurface(surface);
+	surface->bounds = bounds;
 
 	if (surface->sampleDimension < 0) surface->sampleDimension = 1;
 	surface->sampleDimension = Math::RoundPowerOfTwo(surface->sampleDimension);
@@ -216,24 +249,28 @@ void LevelMesh::BuildSurfaceParams(Surface* surface)
 		height = (textureHeight - 2);
 	}
 
+	surface->translateWorldToLocal = bounds.min;
+	surface->projLocalToU = tCoords[0];
+	surface->projLocalToV = tCoords[1];
+
 	surface->lightUV.resize(surface->verts.size());
 	for (i = 0; i < (int)surface->verts.size(); i++)
 	{
-		vec3 tDelta = surface->verts[i] - bounds.min;
-		surface->lightUV[i].x = dot(tDelta, tCoords[0]);
-		surface->lightUV[i].y = dot(tDelta, tCoords[1]);
+		vec3 tDelta = surface->verts[i] - surface->translateWorldToLocal;
+		surface->lightUV[i].x = dot(tDelta, surface->projLocalToU);
+		surface->lightUV[i].y = dot(tDelta, surface->projLocalToV);
 	}
 
 	tOrigin = bounds.min;
 
 	// project tOrigin and tCoords so they lie on the plane
-	d = (plane->Distance(bounds.min) - plane->d) / plane->Normal()[axis];
+	d = (plane->Distance(bounds.min)) / plane->Normal()[axis];
 	tOrigin[axis] -= d;
 
 	for (i = 0; i < 2; i++)
 	{
 		tCoords[i] = normalize(tCoords[i]);
-		d = plane->Distance(tCoords[i]) / plane->Normal()[axis];
+		d = dot(tCoords[i], plane->Normal()) / plane->Normal()[axis];
 		tCoords[i][axis] -= d;
 	}
 

src/lightmap/levelmesh.h

@@ -65,6 +65,7 @@ struct Surface
 	SurfaceType type = ST_UNKNOWN;
 	std::vector<vec3> verts;
 	Plane plane;
+	BBox bounds;
 
 	// Surface material
 	std::string material;
@@ -84,6 +85,11 @@ struct Surface
 	vec3 worldStepX = { 0.0f };
 	vec3 worldStepY = { 0.0f };
 
+	// Calculate world coordinates to UV coordinates
+	vec3 translateWorldToLocal = { 0.0f };
+	vec3 projLocalToU = { 0.0f };
+	vec3 projLocalToV = { 0.0f };
+
 	// Output lightmap for the surface
 	int texWidth = 0;
 	int texHeight = 0;
@@ -96,6 +102,9 @@ struct Surface
 	int atlasPageIndex = -1;
 	int atlasX = 0;
 	int atlasY = 0;
+
+	// Smoothing group surface is to be rendered with
+	int smoothingGroupIndex = -1;
 };
 
 class LightProbeSample
@@ -121,6 +130,8 @@ public:
 
 	std::vector<std::unique_ptr<LightmapTexture>> textures;
 
+	std::vector<Plane> smoothingGroups;
+
 	int defaultSamples = 16;
 	int textureWidth = 128;
 	int textureHeight = 128;

src/math/plane.cpp

@@ -129,7 +129,7 @@ vec3 &Plane::Normal()
 
 float Plane::Distance(const vec3 &point)
 {
-	return dot(point, Normal());
+	return dot(point, Normal()) - d;
 }
 
 Plane &Plane::SetDistance(const vec3 &point)