Add a stencil buffer

src/r_main.cpp

@@ -922,7 +922,7 @@ void R_RenderActorView (AActor *actor, bool dontmaplines)
 	}
 	else
 	{
-		RenderPolyBsp bsp;
+		static RenderPolyBsp bsp;
 		bsp.Render();
 	}
 	R_3D_ResetClip(); // reset clips (floor/ceiling)

src/r_poly.cpp

@@ -41,15 +41,19 @@ CVAR(Bool, r_debug_cull, 0, 0)
 
 void RenderPolyBsp::Render()
 {
+	// Setup working buffers
 	PolyVertexBuffer::Clear();
 	SolidSegments.clear();
 	SolidSegments.reserve(MAXWIDTH / 2 + 2);
 	SolidSegments.push_back({ -0x7fff, 0 });
 	SolidSegments.push_back({ viewwidth, 0x7fff });
-
 	SectorSpriteRanges.clear();
 	SectorSpriteRanges.resize(numsectors);
 	SortedSprites.clear();
+	PvsSectors.clear();
+	SectorSpriteRanges.clear();
+	ScreenSprites.clear();
+	PolyStencilBuffer::Instance()->Clear(viewwidth, viewheight, 0);
 
 	// Perspective correct:
 	float ratio = WidescreenRatio;
@@ -66,13 +70,12 @@ void RenderPolyBsp::Render()
 	// Y shearing like the Doom renderer:
 	//worldToClip = TriMatrix::viewToClip() * TriMatrix::worldToView();
 
-	// Cull front to back (ok, so we dont cull yet, but we should during this!):
+	// Cull front to back
 	if (numnodes == 0)
 		PvsSectors.push_back(subsectors); // RenderSubsector(subsectors);
 	else
 		RenderNode(nodes + numnodes - 1);	// The head node is the last node output.
 
-	static PolySkyDome skydome;
 	skydome.Render(worldToClip);
 
 	// Render back to front (we don't have a zbuffer at the moment, sniff!):
@@ -227,9 +230,6 @@ void RenderPolyBsp::AddLine(seg_t *line, sector_t *frontsector)
 	double frontceilz2 = frontsector->ceilingplane.ZatPoint(line->v2);
 	double frontfloorz2 = frontsector->floorplane.ZatPoint(line->v2);
 
-	//VisiblePlaneKey ceilingPlaneKey(frontsector->GetTexture(sector_t::ceiling), frontsector->ColorMap, frontsector->lightlevel, frontsector->ceilingplane, frontsector->planes[sector_t::ceiling].xform);
-	//VisiblePlaneKey floorPlaneKey(frontsector->GetTexture(sector_t::floor), frontsector->ColorMap, frontsector->lightlevel, frontsector->floorplane, frontsector->planes[sector_t::floor].xform);
-
 	RenderPolyWall wall;
 	wall.Line = line;
 	wall.Colormap = frontsector->ColorMap;

src/r_poly.h

@@ -69,6 +69,27 @@ public:
 	int Count = 0;
 };
 
+class PolySkyDome
+{
+public:
+	PolySkyDome() { CreateDome(); }
+	void Render(const TriMatrix &worldToClip);
+
+private:
+	TArray<TriVertex> mVertices;
+	TArray<unsigned int> mPrimStart;
+	int mRows, mColumns;
+
+	void SkyVertex(int r, int c, bool yflip);
+	void CreateSkyHemisphere(bool zflip);
+	void CreateDome();
+	void RenderRow(const TriUniforms &uniforms, FTexture *skytex, int row);
+	void RenderCapColorRow(const TriUniforms &uniforms, FTexture *skytex, int row, bool bottomCap);
+
+	TriVertex SetVertex(float xx, float yy, float zz, float uu = 0, float vv = 0);
+	TriVertex SetVertexXYZ(float xx, float yy, float zz, float uu = 0, float vv = 0);
+};
+
 // Renders a GL BSP tree in a scene
 class RenderPolyBsp
 {
@@ -121,6 +142,8 @@ private:
 	};
 
 	std::vector<SolidSegment> SolidSegments;
+
+	PolySkyDome skydome;
 };
 
 class RenderPolyWall
@@ -181,24 +204,3 @@ public:
 	static TriVertex *GetVertices(int count);
 	static void Clear();
 };
-
-class PolySkyDome
-{
-public:
-	PolySkyDome() { CreateDome(); }
-	void Render(const TriMatrix &worldToClip);
-
-private:
-	TArray<TriVertex> mVertices;
-	TArray<unsigned int> mPrimStart;
-	int mRows, mColumns;
-
-	void SkyVertex(int r, int c, bool yflip);
-	void CreateSkyHemisphere(bool zflip);
-	void CreateDome();
-	void RenderRow(const TriUniforms &uniforms, FTexture *skytex, int row);
-	void RenderCapColorRow(const TriUniforms &uniforms, FTexture *skytex, int row, bool bottomCap);
-
-	TriVertex SetVertex(float xx, float yy, float zz, float uu = 0, float vv = 0);
-	TriVertex SetVertexXYZ(float xx, float yy, float zz, float uu = 0, float vv = 0);
-};

src/r_poly_triangle.h

@@ -48,6 +48,117 @@ private:
 	friend class DrawPolyTrianglesCommand;
 };
 
+// 8x8 block of stencil values, plus a mask indicating if values are the same for early out stencil testing
+class PolyStencilBlock
+{
+public:
+	PolyStencilBlock(int block, uint8_t *values, uint32_t *masks) : Values(values + block * 64), ValueMask(masks[block])
+	{
+	}
+
+	void Set(int x, int y, uint8_t value)
+	{
+		if (ValueMask == 0xffffffff)
+		{
+			if (Values[0] == value)
+				return;
+
+			for (int i = 1; i < 8 * 8 + 4 * 4 + 2 * 2 + 1; i++)
+				Values[i] = Values[0];
+		}
+
+		if (Values[x + y * 8] == value)
+			return;
+
+		Values[x + y * 8] = value;
+
+		int leveloffset = 0;
+		for (int i = 1; i < 4; i++)
+		{
+			int iy = i + 3;
+
+			x >>= 1;
+			y >>= 1;
+
+			bool same =
+				Values[(x << i)       + (y << iy)]       != value ||
+				Values[((x + 1) << i) + (y << iy)]       != value ||
+				Values[(x << i)       + ((y + 1) << iy)] != value ||
+				Values[((x + 1) << i) + ((y + 1) << iy)] != value;
+
+			int levelbit = 1 << (leveloffset + x + y * (8 >> i));
+
+			if (same)
+				ValueMask = ValueMask & ~levelbit;
+			else
+				ValueMask = ValueMask | levelbit;
+		}
+
+		if (Values[0] != value || Values[4] != value || Values[4 * 8] != value || Values[4 * 8 + 4] != value)
+			ValueMask = ValueMask & ~(1 << 22);
+		else
+			ValueMask = ValueMask | (1 << 22);
+	}
+
+	uint8_t Get(int x, int y) const
+	{
+		if (ValueMask == 0xffffffff)
+			return Values[0];
+		else
+			return Values[x + y * 8];
+	}
+
+	void Clear(uint8_t value)
+	{
+		Values[0] = value;
+		ValueMask = 0xffffffff;
+	}
+
+private:
+	uint8_t *Values; // [8 * 8];
+	uint32_t &ValueMask; // 4 * 4 + 2 * 2 + 1 bits indicating is Values are the same
+};
+
+class PolyStencilBuffer
+{
+public:
+	static PolyStencilBuffer *Instance()
+	{
+		static PolyStencilBuffer buffer;
+		return &buffer;
+	}
+
+	void Clear(int newwidth, int newheight, uint8_t stencil_value = 0)
+	{
+		width = newwidth;
+		height = newheight;
+		int count = BlockWidth() * BlockHeight();
+		values.resize(count * 64);
+		masks.resize(count);
+
+		uint8_t *v = Values();
+		uint32_t *m = Masks();
+		for (int i = 0; i < count; i++)
+		{
+			PolyStencilBlock block(i, v, m);
+			block.Clear(stencil_value);
+		}
+	}
+
+	int Width() const { return width; }
+	int Height() const { return height; }
+	int BlockWidth() const { return (width + 7) / 8; }
+	int BlockHeight() const { return (height + 7) / 8; }
+	uint8_t *Values() { return values.data(); }
+	uint32_t *Masks() { return masks.data(); }
+
+private:
+	int width;
+	int height;
+	std::vector<uint8_t> values;
+	std::vector<uint32_t> masks;
+};
+
 struct ScreenPolyTriangleDrawerArgs
 {
 	uint8_t *dest;