- implement the depth and stencil states

src/rendering/vulkan/renderer/vk_renderpass.cpp

@@ -250,7 +250,7 @@ void VkRenderPassSetup::CreatePipeline(const VkRenderPassKey &key)
 	// builder.addDynamicState(VK_DYNAMIC_STATE_DEPTH_BOUNDS);
 	// builder.addDynamicState(VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK);
 	// builder.addDynamicState(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK);
-	// builder.addDynamicState(VK_DYNAMIC_STATE_STENCIL_REFERENCE);
+	builder.addDynamicState(VK_DYNAMIC_STATE_STENCIL_REFERENCE);
 
 	builder.setViewport(0.0f, 0.0f, (float)SCREENWIDTH, (float)SCREENHEIGHT);
 	builder.setScissor(0, 0, SCREENWIDTH, SCREENHEIGHT);
@@ -263,8 +263,6 @@ void VkRenderPassSetup::CreatePipeline(const VkRenderPassKey &key)
 		VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP
 	};
 
-	builder.setTopology(vktopology[key.DrawType]);
-
 	static const int blendstyles[] = {
 		VK_BLEND_FACTOR_ZERO,
 		VK_BLEND_FACTOR_ONE,
@@ -291,8 +289,15 @@ void VkRenderPassSetup::CreatePipeline(const VkRenderPassKey &key)
 		blendequation = VK_BLEND_OP_ADD;
 	}
 
-	builder.setDepthEnable(key.DepthTest, key.DepthWrite);
+	static const VkStencilOp op2vk[] = { VK_STENCIL_OP_KEEP, VK_STENCIL_OP_INCREMENT_AND_CLAMP, VK_STENCIL_OP_DECREMENT_AND_CLAMP };
+	static const VkCompareOp depthfunc2vk[] = { VK_COMPARE_OP_LESS, VK_COMPARE_OP_LESS_OR_EQUAL, VK_COMPARE_OP_ALWAYS };
 
+	builder.setTopology(vktopology[key.DrawType]);
+	builder.setDepthStencilEnable(key.DepthTest, key.DepthWrite, key.StencilTest);
+	builder.setDepthFunc(depthfunc2vk[key.DepthFunc]);
+	builder.setCull(key.CullMode == Cull_None ? VK_CULL_MODE_NONE : VK_CULL_MODE_FRONT_AND_BACK, key.CullMode == Cull_CW ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE);
+	builder.setColorWriteMask((VkColorComponentFlagBits)key.ColorMask);
+	builder.setStencil(VK_STENCIL_OP_KEEP, op2vk[key.StencilPassOp], VK_STENCIL_OP_KEEP, VK_COMPARE_OP_EQUAL, 0xffffffff, 0xffffffff, 0);
 	builder.setBlendMode((VkBlendOp)blendequation, (VkBlendFactor)srcblend, (VkBlendFactor)dstblend);
 
 	builder.setLayout(fb->GetRenderPassManager()->PipelineLayout.get());

src/rendering/vulkan/renderer/vk_renderpass.h

@@ -17,6 +17,11 @@ public:
 	int AlphaTest;
 	int DepthWrite;
 	int DepthTest;
+	int DepthFunc;
+	int StencilTest;
+	int StencilPassOp;
+	int ColorMask;
+	int CullMode;
 	int VertexFormat;
 	int DrawType;
 

src/rendering/vulkan/renderer/vk_renderstate.cpp

@@ -32,10 +32,8 @@ void VkRenderState::ClearScreen()
 
 void VkRenderState::Draw(int dt, int index, int count, bool apply)
 {
-	if (apply)
+	if (apply || mNeedApply)
 		Apply(dt);
-	else if (mDynamicSetChanged)
-		ApplyDynamicSet();
 
 	drawcalls.Clock();
 	mCommandBuffer->draw(count, 1, index, 0);
@@ -44,10 +42,8 @@ void VkRenderState::Draw(int dt, int index, int count, bool apply)
 
 void VkRenderState::DrawIndexed(int dt, int index, int count, bool apply)
 {
-	if (apply)
+	if (apply || mNeedApply)
 		Apply(dt);
-	else if (mDynamicSetChanged)
-		ApplyDynamicSet();
 
 	drawcalls.Clock();
 	mCommandBuffer->drawIndexed(count, 1, index, 0, 0);
@@ -58,16 +54,20 @@ bool VkRenderState::SetDepthClamp(bool on)
 {
 	bool lastValue = mLastDepthClamp;
 	mLastDepthClamp = on;
+	mNeedApply = true;
 	return lastValue;
 }
 
 void VkRenderState::SetDepthMask(bool on)
 {
 	mDepthWrite = on;
+	mNeedApply = true;
 }
 
 void VkRenderState::SetDepthFunc(int func)
 {
+	mDepthFunc = func;
+	mNeedApply = true;
 }
 
 void VkRenderState::SetDepthRange(float min, float max)
@@ -75,10 +75,14 @@ void VkRenderState::SetDepthRange(float min, float max)
 	mViewportDepthMin = min;
 	mViewportDepthMax = max;
 	mViewportChanged = true;
+	mNeedApply = true;
 }
 
 void VkRenderState::SetColorMask(bool r, bool g, bool b, bool a)
 {
+	int rr = r, gg = g, bb = b, aa = a;
+	mColorMask = (aa < 3) | (bb << 2) | (gg << 1) | rr;
+	mNeedApply = true;
 }
 
 void VkRenderState::EnableDrawBufferAttachments(bool on)
@@ -87,10 +91,24 @@ void VkRenderState::EnableDrawBufferAttachments(bool on)
 
 void VkRenderState::SetStencil(int offs, int op, int flags)
 {
+	mStencilRef = screen->stencilValue + offs;
+	mStencilRefChanged = true;
+	mStencilOp = op;
+
+	if (flags != -1)
+	{
+		bool cmon = !(flags & SF_ColorMaskOff);
+		SetColorMask(cmon, cmon, cmon, cmon); // don't write to the graphics buffer
+		mDepthWrite = cmon;
+	}
+
+	mNeedApply = true;
 }
 
 void VkRenderState::SetCulling(int mode)
 {
+	mCullMode = mode;
+	mNeedApply = true;
 }
 
 void VkRenderState::EnableClipDistance(int num, bool state)
@@ -160,6 +178,8 @@ void VkRenderState::Clear(int targets)
 
 void VkRenderState::EnableStencil(bool on)
 {
+	mStencilTest = on;
+	mNeedApply = true;
 }
 
 void VkRenderState::SetScissor(int x, int y, int w, int h)
@@ -169,6 +189,7 @@ void VkRenderState::SetScissor(int x, int y, int w, int h)
 	mScissorWidth = w;
 	mScissorHeight = h;
 	mScissorChanged = true;
+	mNeedApply = true;
 }
 
 void VkRenderState::SetViewport(int x, int y, int w, int h)
@@ -178,11 +199,13 @@ void VkRenderState::SetViewport(int x, int y, int w, int h)
 	mViewportWidth = w;
 	mViewportHeight = h;
 	mViewportChanged = true;
+	mNeedApply = true;
 }
 
 void VkRenderState::EnableDepthTest(bool on)
 {
 	mDepthTest = on;
+	mNeedApply = true;
 }
 
 void VkRenderState::EnableMultisampling(bool on)
@@ -198,12 +221,14 @@ void VkRenderState::Apply(int dt)
 	ApplyRenderPass(dt);
 	ApplyScissor();
 	ApplyViewport();
+	ApplyStencilRef();
 	ApplyStreamData();
 	ApplyMatrices();
 	ApplyPushConstants();
 	ApplyVertexBuffers();
 	ApplyDynamicSet();
 	ApplyMaterial();
+	mNeedApply = false;
 }
 
 void VkRenderState::ApplyRenderPass(int dt)
@@ -218,6 +243,11 @@ void VkRenderState::ApplyRenderPass(int dt)
 	passKey.RenderStyle = mRenderStyle;
 	passKey.DepthTest = mDepthTest;
 	passKey.DepthWrite = mDepthTest && mDepthWrite;
+	passKey.DepthFunc = mDepthFunc;
+	passKey.StencilTest = mStencilTest;
+	passKey.StencilPassOp = mStencilOp;
+	passKey.ColorMask = mColorMask;
+	passKey.CullMode = mCullMode;
 	if (mSpecialEffect > EFF_NONE)
 	{
 		passKey.SpecialEffect = mSpecialEffect;
@@ -241,6 +271,7 @@ void VkRenderState::ApplyRenderPass(int dt)
 		changingRenderPass = true;
 		mScissorChanged = true;
 		mViewportChanged = true;
+		mStencilRefChanged = true;
 	}
 	else if (changingRenderPass)
 	{
@@ -262,6 +293,15 @@ void VkRenderState::ApplyRenderPass(int dt)
 	}
 }
 
+void VkRenderState::ApplyStencilRef()
+{
+	if (mStencilRefChanged)
+	{
+		mCommandBuffer->setStencilReference(VK_STENCIL_FRONT_AND_BACK, mStencilRef);
+		mStencilRefChanged = false;
+	}
+}
+
 void VkRenderState::ApplyScissor()
 {
 	if (mScissorChanged)
@@ -530,7 +570,6 @@ void VkRenderState::ApplyDynamicSet()
 		mLastViewpointOffset = mViewpointOffset;
 		mLastLightBufferOffset = mLightBufferOffset;
 	}
-	mDynamicSetChanged = false;
 }
 
 void VkRenderState::Bind(int bindingpoint, uint32_t offset)
@@ -538,13 +577,13 @@ void VkRenderState::Bind(int bindingpoint, uint32_t offset)
 	if (bindingpoint == VIEWPOINT_BINDINGPOINT)
 	{
 		mViewpointOffset = offset;
+		mNeedApply = true;
 	}
 	else if (bindingpoint == LIGHTBUF_BINDINGPOINT)
 	{
 		mLightBufferOffset = offset;
+		mNeedApply = true;
 	}
-
-	mDynamicSetChanged = true;
 }
 
 void VkRenderState::EndRenderPass()
@@ -555,7 +594,6 @@ void VkRenderState::EndRenderPass()
 		mCommandBuffer = nullptr;
 		mRenderPassKey = {};
 
-		// To do: move this elsewhere or rename this function to make it clear this can only happen at the end of a frame
 		mMatricesOffset = 0;
 		mStreamDataOffset = 0;
 		mDataIndex = -1;

src/rendering/vulkan/renderer/vk_renderstate.h

@@ -47,6 +47,7 @@ public:
 private:
 	void Apply(int dt);
 	void ApplyRenderPass(int dt);
+	void ApplyStencilRef();
 	void ApplyScissor();
 	void ApplyViewport();
 	void ApplyStreamData();
@@ -59,7 +60,7 @@ private:
 	bool mLastDepthClamp = true;
 	VulkanCommandBuffer *mCommandBuffer = nullptr;
 	VkRenderPassKey mRenderPassKey = {};
-	bool mDynamicSetChanged = true;
+	bool mNeedApply = true;
 
 	int mScissorX = 0, mScissorY = 0, mScissorWidth = -1, mScissorHeight = -1;
 	int mViewportX = 0, mViewportY = 0, mViewportWidth = -1, mViewportHeight = -1;
@@ -69,6 +70,14 @@ private:
 
 	bool mDepthTest = false;
 	bool mDepthWrite = false;
+	bool mStencilTest = false;
+
+	bool mStencilRefChanged = false;
+	int mStencilRef = 0;
+	int mStencilOp = 0;
+	int mDepthFunc = 0;
+	int mColorMask = 15;
+	int mCullMode = 0;
 
 	MatricesUBO mMatrices = {};
 	StreamData mStreamData = {};

src/rendering/vulkan/system/vk_builders.h

@@ -186,7 +186,10 @@ public:
 	void setScissor(int x, int y, int width, int height);
 
 	void setCull(VkCullModeFlags cullMode, VkFrontFace frontFace);
-	void setDepthEnable(bool test, bool write);
+	void setDepthStencilEnable(bool test, bool write, bool stencil);
+	void setDepthFunc(VkCompareOp func);
+	void setColorWriteMask(VkColorComponentFlags mask);
+	void setStencil(VkStencilOp failOp, VkStencilOp passOp, VkStencilOp depthFailOp, VkCompareOp compareOp, uint32_t compareMask, uint32_t writeMask, uint32_t reference);
 
 	void setAdditiveBlendMode();
 	void setAlphaBlendMode();
@@ -764,12 +767,42 @@ inline void GraphicsPipelineBuilder::setCull(VkCullModeFlags cullMode, VkFrontFa
 	rasterizer.frontFace = frontFace;
 }
 
-inline void GraphicsPipelineBuilder::setDepthEnable(bool test, bool write)
+inline void GraphicsPipelineBuilder::setDepthStencilEnable(bool test, bool write, bool stencil)
 {
 	depthStencil.depthTestEnable = test ? VK_TRUE : VK_FALSE;
 	depthStencil.depthWriteEnable = write ? VK_TRUE : VK_FALSE;
+	depthStencil.stencilTestEnable = stencil ? VK_TRUE : VK_FALSE;
 
-	pipelineInfo.pDepthStencilState = (test || write) ? &depthStencil : nullptr;
+	pipelineInfo.pDepthStencilState = (test || write || stencil) ? &depthStencil : nullptr;
+}
+
+inline void GraphicsPipelineBuilder::setStencil(VkStencilOp failOp, VkStencilOp passOp, VkStencilOp depthFailOp, VkCompareOp compareOp, uint32_t compareMask, uint32_t writeMask, uint32_t reference)
+{
+	depthStencil.front.failOp = failOp;
+	depthStencil.front.passOp = passOp;
+	depthStencil.front.depthFailOp = depthFailOp;
+	depthStencil.front.compareOp = compareOp;
+	depthStencil.front.compareMask = compareMask;
+	depthStencil.front.writeMask = writeMask;
+	depthStencil.front.reference = reference;
+
+	depthStencil.back.failOp = failOp;
+	depthStencil.back.passOp = passOp;
+	depthStencil.back.depthFailOp = depthFailOp;
+	depthStencil.back.compareOp = compareOp;
+	depthStencil.back.compareMask = compareMask;
+	depthStencil.back.writeMask = writeMask;
+	depthStencil.back.reference = reference;
+}
+
+inline void GraphicsPipelineBuilder::setDepthFunc(VkCompareOp func)
+{
+	depthStencil.depthCompareOp = func;
+}
+
+inline void GraphicsPipelineBuilder::setColorWriteMask(VkColorComponentFlags mask)
+{
+	colorBlendAttachment.colorWriteMask = mask;
 }
 
 inline void GraphicsPipelineBuilder::setAdditiveBlendMode()

src/rendering/vulkan/system/vk_device.cpp

@@ -49,11 +49,10 @@ extern HWND Window;
 
 EXTERN_CVAR(Bool, vid_vsync);
 
-#ifdef NDEBUG
+CUSTOM_CVAR(Bool, vk_debug, false, CVAR_ARCHIVE | CVAR_GLOBALCONFIG | CVAR_NOINITCALL)
-CVAR(Bool, vk_debug, false, 0);	// this should be false, once the oversized model can be removed.
+{
-#else
+	Printf("This won't take effect until " GAMENAME " is restarted.\n");
-CVAR(Bool, vk_debug, true, 0);
+}
-#endif
 
 VulkanDevice::VulkanDevice()
 {