Minor cleanup and lightmapper fixes

src/common/rendering/hwrenderer/data/hw_levelmesh.h

@@ -312,7 +312,7 @@ public:
 			{
 				if (surface->sectorGroup == SmoothingGroups[j].sectorGroup)
 				{
-					float direction = std::abs((SmoothingGroups[j].plane.XYZ() | surface->plane.XYZ()));
+					float direction = SmoothingGroups[j].plane.XYZ() | surface->plane.XYZ();
 					if (direction >= 0.9999f && direction <= 1.001f)
 					{
 						auto point = (surface->plane.XYZ() * surface->plane.W);

src/rendering/hwrenderer/doom_levelmesh.cpp

@@ -80,6 +80,8 @@ void PrintSurfaceInfo(const DoomLevelMeshSurface* surface)
 {
 	if (!RequireLevelMesh()) return;
 
+	auto gameTexture = TexMan.GameByIndex(surface->texture.GetIndex());
+
 	Printf("Surface %d (%p)\n    Type: %d, TypeIndex: %d, ControlSector: %d\n", level.levelMesh->GetSurfaceIndex(surface), surface, surface->Type, surface->typeIndex, surface->ControlSector ? surface->ControlSector->Index() : -1);
 	Printf("    Atlas page: %d, x:%d, y:%d\n", surface->atlasPageIndex, surface->atlasX, surface->atlasY);
 	Printf("    Pixels: %dx%d (area: %d)\n", surface->texWidth, surface->texHeight, surface->Area());
@@ -87,7 +89,8 @@ void PrintSurfaceInfo(const DoomLevelMeshSurface* surface)
 	Printf("    Needs update?: %d\n", surface->needsUpdate);
 	Printf("    Sector group: %d\n", surface->sectorGroup);
 	Printf("    Smoothing group: %d\n", surface->smoothingGroupIndex);
-
+	Printf("    Texture: '%s' (id=%d)\n", gameTexture ? gameTexture->GetName().GetChars() : "<nullptr>", surface->texture.GetIndex());
+	Printf("    Alpha: %f\n", surface->alpha);
 }
 
 FVector3 RayDir(FAngle angle, FAngle pitch)
@@ -826,14 +829,14 @@ void DoomLevelMesh::CreateSideSurfaces(FLevelLocals &doomMap, side_t *side)
 		surf.bSky = false;
 		surf.plane = ToPlane(verts[0], verts[1], verts[2], verts[3]);
 
+		auto offset = FVector3(surf.plane.XYZ()) * 0.05; // for better accuracy when raytracing mid-textures from each side
+
 		if (side->linedef->sidedef[0] != side)
 		{
 			surf.plane = -surf.plane;
 			surf.plane.W = -surf.plane.W;
 		}
 
-		auto offset = surf.plane.XYZ() * 0.05; // for better accuracy when raytracing mid-textures from each side
-
 		MeshVertices.Push(verts[0] + offset);
 		MeshVertices.Push(verts[1] + offset);
 		MeshVertices.Push(verts[2] + offset);

wadsrc/static/shaders/lightmap/frag_raytrace.glsl

@@ -260,8 +260,6 @@ int TraceFirstHitTriangleNoPortal(vec3 origin, float tmin, vec3 dir, float tmax,
 		primitiveWeights.xy = rayQueryGetIntersectionBarycentricsEXT(rayQuery, true);
 		primitiveWeights.z = 1.0 - primitiveWeights.x - primitiveWeights.y;
 
-		primitiveWeights = vec3(primitiveWeights.z, primitiveWeights.x, primitiveWeights.y);
-
 		return rayQueryGetIntersectionPrimitiveIndexEXT(rayQuery, true);
 	}
 	else
@@ -348,8 +346,8 @@ float intersect_triangle_ray(RayBBox ray, int a, out float barycentricB, out flo
 	float det = dot(e1, P);
 
 	// Backface check
-	//if (det < 0.0f)
+	if (det < 0.0f)
-	//	return 1.0f;
+		return 1.0f;
 
 	// If determinant is near zero, ray lies in plane of triangle
 	if (det > -FLT_EPSILON && det < FLT_EPSILON)
@@ -384,11 +382,6 @@ float intersect_triangle_ray(RayBBox ray, int a, out float barycentricB, out flo
 	// Return hit location on triangle in barycentric coordinates
 	barycentricB = u;
 	barycentricC = v;
-
-	primitiveWeights.x = u;
-	primitiveWeights.y = v;
-	primitiveWeights.z = 1.0 - u - v;
-	
 	return t;
 }
 
@@ -464,6 +457,10 @@ TraceHit find_first_hit(RayBBox ray)
 				float t = intersect_triangle_ray(ray, a, baryB, baryC);
 				if (t < hit.fraction)
 				{
+					primitiveWeights.x = baryB;
+					primitiveWeights.y = baryC;
+					primitiveWeights.z = 1.0 - baryB - baryC;
+
 					hit.fraction = t;
 					hit.triangle = nodes[a].element_index / 3;
 					hit.b = baryB;
@@ -538,7 +535,7 @@ int TraceFirstHitTriangleT(vec3 origin, float tmin, vec3 dir, float tmax, out fl
 		{
 #if defined(USE_RAYQUERY)
 			int index = primitiveID * 3;
-			vec2 uv = vertices[elements[index + 0]].uv * primitiveWeights.x + vertices[elements[index + 1]].uv * primitiveWeights.y + vertices[elements[index + 2]].uv * primitiveWeights.z;
+			vec2 uv = vertices[elements[index + 1]].uv * primitiveWeights.x + vertices[elements[index + 2]].uv * primitiveWeights.y + vertices[elements[index + 0]].uv * primitiveWeights.z;
 
 			if (surface.TextureIndex < 0)
 			{
@@ -548,7 +545,7 @@ int TraceFirstHitTriangleT(vec3 origin, float tmin, vec3 dir, float tmax, out fl
 			vec4 color = texture(textures[surface.TextureIndex], uv);
 			color.w *= surface.Alpha;
 
-			if (color.w > 0.999 || rayColor.rgb == vec3(0))
+			if (color.w > 0.999 || all(lessThan(rayColor.rgb, vec3(0.001))))
 			{
 				break;
 			}
@@ -604,7 +601,7 @@ bool TracePoint(vec3 origin, vec3 target, float tmin, vec3 dir, float tmax)
 		{
 #if defined(USE_RAYQUERY)
 			int index = primitiveID * 3;
-			vec2 uv = vertices[elements[index + 0]].uv * primitiveWeights.x + vertices[elements[index + 1]].uv * primitiveWeights.y + vertices[elements[index + 2]].uv * primitiveWeights.z;
+			vec2 uv = vertices[elements[index + 1]].uv * primitiveWeights.x + vertices[elements[index + 2]].uv * primitiveWeights.y + vertices[elements[index + 0]].uv * primitiveWeights.z;
 
 			if (surface.TextureIndex < 0)
 			{
@@ -614,7 +611,7 @@ bool TracePoint(vec3 origin, vec3 target, float tmin, vec3 dir, float tmax)
 			vec4 color = texture(textures[surface.TextureIndex], uv);
 			color.w *= surface.Alpha;
 
-			if (color.w > 0.999 || rayColor.rgb == vec3(0))
+			if (color.w > 0.999 || all(lessThan(rayColor.rgb, vec3(0.001))))
 			{
 				break;
 			}