Calculate normals for OBJ models with smooth groups

Add smoothGroup member to OBJFace struct, and assign the current smooth group number to it
Move face normal calculation code to CalculateNormalFlat
Add AddVertFaces method, which initializes and populates the vertFaces array of arrays, which holds references to triangle references per vertex
Only initialize and populate vertFaces if the model has missing normals and smooth groups
Assign smooth groups to triangle data
Add CalculateNormalSmooth method, which calculates the normals for each face the vertex is attached to, depending on whether or not the faces are part of the given smooth group, and averages them out
Add OBJTriRef struct, which holds references to triangles on OBJ surfaces

Make {agg,cur}SurfFaceCount unsigned ints
Change nvec to a value instead of a pointer

src/r_data/models/models_obj.cpp

@@ -101,8 +101,9 @@ bool FOBJModel::Load(const char* fn, int lumpnum, const char* buffer, int length
 
 	FTextureID curMtl = FNullTextureID();
 	OBJSurface *curSurface = nullptr;
-	int aggSurfFaceCount = 0;
+	unsigned int aggSurfFaceCount = 0;
-	int curSurfFaceCount = 0;
+	unsigned int curSurfFaceCount = 0;
+	unsigned int curSmoothGroup = 0;
 
 	while(sc.GetString())
 	{
@@ -186,9 +187,25 @@ bool FOBJModel::Load(const char* fn, int lumpnum, const char* buffer, int length
 					sc.UnGet(); // No 4th side, move back
 				}
 			}
+			face.smoothGroup = curSmoothGroup;
 			faces.Push(face);
 			curSurfFaceCount += 1;
 		}
+		else if (sc.Compare("s"))
+		{
+			sc.MustGetString();
+			if (sc.Compare("off"))
+			{
+				curSmoothGroup = 0;
+			}
+			else
+			{
+				sc.UnGet();
+				sc.MustGetNumber();
+				curSmoothGroup = sc.Number;
+				hasSmoothGroups = hasSmoothGroups || curSmoothGroup > 0;
+			}
+		}
 	}
 	sc.Close();
 
@@ -277,11 +294,13 @@ bool FOBJModel::ParseFaceSide(const FString &sideStr, OBJFace &face, int sidx)
 			else
 			{
 				side.normref = -1;
+				hasMissingNormals = true;
 			}
 		}
 		else
 		{
 			side.normref = -1;
+			hasMissingNormals = true;
 		}
 	}
 	else
@@ -289,6 +308,7 @@ bool FOBJModel::ParseFaceSide(const FString &sideStr, OBJFace &face, int sidx)
 		origIdx = atoi(sideStr.GetChars());
 		side.vertref = ResolveIndex(origIdx, FaceElement::VertexIndex);
 		side.normref = -1;
+		hasMissingNormals = true;
 		side.uvref = -1;
 	}
 	face.sides[sidx] = side;
@@ -348,6 +368,11 @@ void FOBJModel::BuildVertexBuffer(FModelRenderer *renderer)
 		surfaces[i].vbStart = vbufsize;
 		vbufsize += surfaces[i].numTris * 3;
 	}
+	// Initialize/populate vertFaces
+	if (hasMissingNormals && hasSmoothGroups)
+	{
+		AddVertFaces();
+	}
 
 	auto vbuf = renderer->CreateVertexBuffer(false,true);
 	SetVertexBuffer(renderer, vbuf);
@@ -372,39 +397,40 @@ void FOBJModel::BuildVertexBuffer(FModelRenderer *renderer)
 
 				FVector3 curVvec = RealignVector(verts[vidx]);
 				FVector2 curUvec = FixUV(uvs[uvidx]);
-				FVector3 *nvec = nullptr;
+				FVector3 nvec;
 
 				mdv->Set(curVvec.X, curVvec.Y, curVvec.Z, curUvec.X, curUvec.Y);
 
 				if (nidx >= 0 && (unsigned int)nidx < norms.Size())
 				{
-					nvec = new FVector3(RealignVector(norms[nidx]));
+					nvec = RealignVector(norms[nidx]);
 				}
 				else
 				{
-					// https://www.khronos.org/opengl/wiki/Calculating_a_Surface_Normal
+					if (surfaces[i].tris[j].smoothGroup == 0)
-					// Find other sides of triangle
+					{
-					auto nextSidx = side + 2;
+						nvec = CalculateNormalFlat(i, j);
-					if (nextSidx >= 3) nextSidx -= 3;
-
-					auto lastSidx = side + 1;
-					if (lastSidx >= 3) lastSidx -= 3;
-
-					OBJFaceSide &nextSide = surfaces[i].tris[j].sides[nextSidx];
-					OBJFaceSide &lastSide = surfaces[i].tris[j].sides[lastSidx];
-
-					// Cross-multiply the U-vector and V-vector
-					FVector3 uvec = RealignVector(verts[nextSide.vertref]) - curVvec;
-					FVector3 vvec = RealignVector(verts[lastSide.vertref]) - curVvec;
-
-					nvec = new FVector3(uvec ^ vvec);
 					}
-				mdv->SetNormal(nvec->X, nvec->Y, nvec->Z);
+					else
-				delete nvec;
+					{
+						nvec = CalculateNormalSmooth(vidx, surfaces[i].tris[j].smoothGroup);
+					}
+				}
+				mdv->SetNormal(nvec.X, nvec.Y, nvec.Z);
 			}
 		}
 		delete[] surfaces[i].tris;
 	}
+
+	// Destroy vertFaces
+	if (hasMissingNormals && hasSmoothGroups)
+	{
+		for (size_t i = 0; i < verts.Size(); i++)
+		{
+			vertFaces[i].Clear();
+		}
+		delete[] vertFaces;
+	}
 	vbuf->UnlockVertexBuffer();
 }
 
@@ -432,6 +458,7 @@ void FOBJModel::ConstructSurfaceTris(OBJSurface &surf)
 		surf.tris[triIdx].sideCount = 3;
 		if (faces[i].sideCount == 3)
 		{
+			surf.tris[triIdx].smoothGroup = faces[i].smoothGroup;
 			memcpy(surf.tris[triIdx].sides, faces[i].sides, sizeof(OBJFaceSide) * 3);
 		}
 		else if (faces[i].sideCount == 4) // Triangulate face
@@ -443,6 +470,7 @@ void FOBJModel::ConstructSurfaceTris(OBJSurface &surf)
 			delete[] triangulated;
 			triIdx += 1; // Filling out two faces
 		}
+		DPrintf(DMSG_SPAMMY, "Smooth group: %d\n", surf.tris[triIdx].smoothGroup);
 	}
 }
 
@@ -455,7 +483,9 @@ void FOBJModel::ConstructSurfaceTris(OBJSurface &surf)
 void FOBJModel::TriangulateQuad(const OBJFace &quad, OBJFace *tris)
 {
 	tris[0].sideCount = 3;
+	tris[0].smoothGroup = quad.smoothGroup;
 	tris[1].sideCount = 3;
+	tris[1].smoothGroup = quad.smoothGroup;
 
 	int tsidx[2][3] = {{0, 1, 3}, {1, 2, 3}};
 
@@ -470,6 +500,26 @@ void FOBJModel::TriangulateQuad(const OBJFace &quad, OBJFace *tris)
 	}
 }
 
+/**
+ * Add the vertices of all surfaces' triangles to the array of vertex->triangle references
+ */
+void FOBJModel::AddVertFaces() {
+	// Initialize and populate vertFaces - this array stores references to triangles per vertex
+	vertFaces = new TArray<OBJTriRef>[verts.Size()];
+	for (size_t i = 0; i < surfaces.Size(); i++)
+	{
+		for (size_t j = 0; j < surfaces[i].numTris; j++)
+		{
+			OBJTriRef otr = OBJTriRef(i, j);
+			for (size_t k = 0; k < surfaces[i].tris[j].sideCount; k++)
+			{
+				int vidx = surfaces[i].tris[j].sides[k].vertref;
+				vertFaces[vidx].Push(otr);
+			}
+		}
+	}
+}
+
 /**
  * Re-align a vector to match MD3 alignment
  *
@@ -494,6 +544,65 @@ inline FVector2 FOBJModel::FixUV(FVector2 vecToRealign)
 	return vecToRealign;
 }
 
+/**
+ * Calculate the surface normal for a triangle
+ *
+ * @param surfIdx The surface index
+ * @param triIdx The triangle Index
+ * @return The surface normal vector
+ */
+FVector3 FOBJModel::CalculateNormalFlat(unsigned int surfIdx, unsigned int triIdx)
+{
+	// https://www.khronos.org/opengl/wiki/Calculating_a_Surface_Normal
+	int curVert = surfaces[surfIdx].tris[triIdx].sides[0].vertref;
+	int nextVert = surfaces[surfIdx].tris[triIdx].sides[2].vertref;
+	int lastVert = surfaces[surfIdx].tris[triIdx].sides[1].vertref;
+
+	// Cross-multiply the U-vector and V-vector
+	FVector3 curVvec = RealignVector(verts[curVert]);
+	FVector3 uvec = RealignVector(verts[nextVert]) - curVvec;
+	FVector3 vvec = RealignVector(verts[lastVert]) - curVvec;
+
+	return uvec ^ vvec;
+}
+
+/**
+ * Calculate the surface normal for a triangle
+ *
+ * @param otr A reference to the surface, and a triangle within that surface, as an OBJTriRef
+ * @return The surface normal vector
+ */
+FVector3 FOBJModel::CalculateNormalFlat(OBJTriRef otr)
+{
+	return CalculateNormalFlat(otr.surf, otr.tri);
+}
+
+/**
+ * Calculate the normal of a vertex in a specific smooth group
+ *
+ * @param vidx The index of the vertex in the array of vertices
+ * @param smoothGroup The smooth group number
+ */
+FVector3 FOBJModel::CalculateNormalSmooth(unsigned int vidx, unsigned int smoothGroup)
+{
+	unsigned int connectedFaces = 0;
+	TArray<OBJTriRef>& vTris = vertFaces[vidx];
+
+	FVector3 vNormal(0,0,0);
+	for (size_t face = 0; face < vTris.Size(); face++)
+	{
+		OBJFace& tri = surfaces[vTris[face].surf].tris[vTris[face].tri];
+		if (tri.smoothGroup == smoothGroup)
+		{
+			FVector3 fNormal = CalculateNormalFlat(vTris[face]);
+			connectedFaces += 1;
+			vNormal += fNormal;
+		}
+	}
+	vNormal /= connectedFaces;
+	return vNormal;
+}
+
 /**
  * Find the index of the frame with the given name
  *

src/r_data/models/models_obj.h

@@ -30,6 +30,8 @@ class FOBJModel : public FModel
 {
 private:
 	const char *newSideSep = "$"; // OBJ side separator is /, which is parsed as a line comment by FScanner if two of them are next to each other.
+	bool hasMissingNormals;
+	bool hasSmoothGroups;
 
 	enum class FaceElement
 	{
@@ -38,6 +40,14 @@ private:
 		VNormalIndex
 	};
 
+	struct OBJTriRef
+	{
+		unsigned int surf;
+		unsigned int tri;
+		OBJTriRef(): surf(0), tri(0) {}
+		OBJTriRef(unsigned int surf, unsigned int tri): surf(surf), tri(tri) {}
+		bool operator== (OBJTriRef other) { return surf == other.surf && tri == other.tri; }
+	};
 	struct OBJFaceSide
 	{
 		int vertref;
@@ -47,7 +57,9 @@ private:
 	struct OBJFace
 	{
 		unsigned int sideCount;
+		unsigned int smoothGroup;
 		OBJFaceSide sides[4];
+		OBJFace(): sideCount(0), smoothGroup(0) {}
 	};
 	struct OBJSurface // 1 surface per 'usemtl'
 	{
@@ -66,16 +78,21 @@ private:
 	TArray<OBJFace> faces;
 	TArray<OBJSurface> surfaces;
 	FScanner sc;
+	TArray<OBJTriRef>* vertFaces;
 
+	int ResolveIndex(int origIndex, FaceElement el);
 	template<typename T, size_t L> void ParseVector(TArray<T> &array);
 	bool ParseFaceSide(const FString &side, OBJFace &face, int sidx);
 	void ConstructSurfaceTris(OBJSurface &surf);
-	int ResolveIndex(int origIndex, FaceElement el);
+	void AddVertFaces();
 	void TriangulateQuad(const OBJFace &quad, OBJFace *tris);
 	FVector3 RealignVector(FVector3 vecToRealign);
 	FVector2 FixUV(FVector2 vecToRealign);
+	FVector3 CalculateNormalFlat(unsigned int surfIdx, unsigned int triIdx);
+	FVector3 CalculateNormalFlat(OBJTriRef otr);
+	FVector3 CalculateNormalSmooth(unsigned int vidx, unsigned int smoothGroup);
 public:
-	FOBJModel() {}
+	FOBJModel(): hasMissingNormals(false), hasSmoothGroups(false), vertFaces(nullptr) {}
 	~FOBJModel();
 	bool Load(const char* fn, int lumpnum, const char* buffer, int length) override;
 	int FindFrame(const char* name) override;