- add stencil buffer support for 2d drawing (#1660)

Co-authored-by: Christoph Oelckers <coelckers@users.noreply.github.com>

src/common/2d/v_2ddrawer.cpp

@@ -1142,6 +1142,55 @@ void F2DDrawer::AddPixel(int x1, int y1, uint32_t color)
 //
 //==========================================================================
 
+void F2DDrawer::AddEnableStencil(bool on)
+{
+	RenderCommand dg;
+
+	dg.isSpecial = SpecialDrawCommand::EnableStencil;
+	dg.stencilOn = on;
+
+	AddCommand(&dg);
+}
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+
+void F2DDrawer::AddSetStencil(int offs, int op, int flags)
+{
+	RenderCommand dg;
+
+	dg.isSpecial = SpecialDrawCommand::SetStencil;
+	dg.stencilOffs = offs;
+	dg.stencilOp = op;
+	dg.stencilFlags = flags;
+
+	AddCommand(&dg);
+}
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+
+void F2DDrawer::AddClearStencil()
+{
+	RenderCommand dg;
+
+	dg.isSpecial = SpecialDrawCommand::ClearStencil;
+
+	AddCommand(&dg);
+}
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+
 void F2DDrawer::Clear()
 {
 	if (!locked)

src/common/2d/v_2ddrawer.h

@@ -52,6 +52,14 @@ struct F2DPolygons
 class DShape2D;
 struct DShape2DBufferInfo;
 
+enum class SpecialDrawCommand {
+	NotSpecial,
+
+	EnableStencil,
+	SetStencil,
+	ClearStencil,
+};
+
 class F2DDrawer
 {
 public:
@@ -107,6 +115,14 @@ public:
 
 	struct RenderCommand
 	{
+		SpecialDrawCommand isSpecial;
+
+		bool stencilOn;
+
+		int stencilOffs;
+		int stencilOp;
+		int stencilFlags;
+
 		EDrawType mType;
 		int mVertIndex;
 		int mVertCount;
@@ -141,6 +157,10 @@ public:
 		// If these fields match, two draw commands can be batched.
 		bool isCompatible(const RenderCommand &other) const
 		{
+			if (
+				isSpecial != SpecialDrawCommand::NotSpecial ||
+				other.isSpecial != SpecialDrawCommand::NotSpecial
+			) return false;
 			if (shape2DBufInfo != nullptr || other.shape2DBufInfo != nullptr) return false;
 			return mTexture == other.mTexture &&
 				mType == other.mType &&
@@ -210,6 +230,10 @@ public:
 	void AddThickLine(int x1, int y1, int x2, int y2, double thickness, uint32_t color, uint8_t alpha = 255);
 	void AddPixel(int x1, int y1, uint32_t color);
 
+	void AddEnableStencil(bool on);
+	void AddSetStencil(int offs, int op, int flags);
+	void AddClearStencil();
+
 	void Clear();
 	void Lock() { locked = true; }
 	void SetScreenFade(float factor) { screenFade = factor; }

src/common/2d/v_draw.cpp

@@ -1626,6 +1626,40 @@ DEFINE_ACTION_FUNCTION(_Screen, SetOffset)
 	ACTION_RETURN_VEC2(twod->SetOffset(DVector2(x, y)));
 }
 
+DEFINE_ACTION_FUNCTION(_Screen, EnableStencil)
+{
+	PARAM_PROLOGUE;
+	PARAM_BOOL(on);
+
+	if (!twod->HasBegun2D()) ThrowAbortException(X_OTHER, "Attempt to draw to screen outside a draw function");
+
+	twod->AddEnableStencil(on);
+	return 0;
+}
+
+DEFINE_ACTION_FUNCTION(_Screen, SetStencil)
+{
+	PARAM_PROLOGUE;
+	PARAM_INT(offs);
+	PARAM_INT(op);
+	PARAM_INT(flags);
+
+	if (!twod->HasBegun2D()) ThrowAbortException(X_OTHER, "Attempt to draw to screen outside a draw function");
+
+	twod->AddSetStencil(offs, op, flags);
+	return 0;
+}
+
+DEFINE_ACTION_FUNCTION(_Screen, ClearStencil)
+{
+	PARAM_PROLOGUE;
+
+	if (!twod->HasBegun2D()) ThrowAbortException(X_OTHER, "Attempt to draw to screen outside a draw function");
+
+	twod->AddClearStencil();
+	return 0;
+}
+
 DEFINE_ACTION_FUNCTION(_Screen, SetTransform)
 {
 	PARAM_PROLOGUE;
@@ -1645,6 +1679,5 @@ DEFINE_ACTION_FUNCTION(_Screen, ClearTransform)
 	if (!twod->HasBegun2D()) ThrowAbortException(X_OTHER, "Attempt to draw to screen outside a draw function");
 
 	twod->ClearTransform();
-
 	return 0;
 }

src/common/rendering/gl/gl_renderbuffers.cpp

@@ -87,6 +87,7 @@ void FGLRenderBuffers::ClearScene()
 
 void FGLRenderBuffers::ClearPipeline()
 {
+	DeleteRenderBuffer(mPipelineDepthStencilBuf);
 	for (int i = 0; i < NumPipelineTextures; i++)
 	{
 		DeleteFrameBuffer(mPipelineFB[i]);
@@ -239,10 +240,11 @@ void FGLRenderBuffers::CreatePipeline(int width, int height)
 	ClearPipeline();
 	ClearEyeBuffers();
 
+	mPipelineDepthStencilBuf = CreateRenderBuffer("PipelineDepthStencil", GL_DEPTH24_STENCIL8, width, height);
 	for (int i = 0; i < NumPipelineTextures; i++)
 	{
 		mPipelineTexture[i] = Create2DTexture("PipelineTexture", GL_RGBA16F, width, height);
-		mPipelineFB[i] = CreateFrameBuffer("PipelineFB", mPipelineTexture[i]);
+		mPipelineFB[i] = CreateFrameBuffer("PipelineFB", mPipelineTexture[i], mPipelineDepthStencilBuf);
 	}
 }
 

src/common/rendering/gl/gl_renderbuffers.h

@@ -173,6 +173,7 @@ private:
 
 	static const int NumPipelineTextures = 2;
 	int mCurrentPipelineTexture = 0;
+	PPGLRenderBuffer mPipelineDepthStencilBuf;
 
 	// Buffers for the scene
 	PPGLTexture mSceneMultisampleTex;

src/common/rendering/hwrenderer/hw_draw2d.cpp

@@ -59,6 +59,9 @@ void Draw2D(F2DDrawer *drawer, FRenderState &state)
 	state.SetViewport(mScreenViewport.left, mScreenViewport.top, mScreenViewport.width, mScreenViewport.height);
 	screen->mViewpoints->Set2D(state, twod->GetWidth(), twod->GetHeight());
 
+	state.EnableStencil(false);
+	state.SetStencil(0, SOP_Keep, SF_AllOn);
+	state.Clear(CT_Stencil);
 	state.EnableDepthTest(false);
 	state.EnableMultisampling(false);
 	state.EnableLineSmooth(gl_aalines);
@@ -88,6 +91,22 @@ void Draw2D(F2DDrawer *drawer, FRenderState &state)
 
 	for(auto &cmd : commands)
 	{
+		if (cmd.isSpecial != SpecialDrawCommand::NotSpecial)
+		{
+			if (cmd.isSpecial == SpecialDrawCommand::EnableStencil)
+			{
+				state.EnableStencil(cmd.stencilOn);
+			}
+			else if (cmd.isSpecial == SpecialDrawCommand::SetStencil)
+			{
+				state.SetStencil(cmd.stencilOffs, cmd.stencilOp, cmd.stencilFlags);
+			}
+			else if (cmd.isSpecial == SpecialDrawCommand::ClearStencil)
+			{
+				state.Clear(CT_Stencil);
+			}
+			continue;
+		}
 
 		state.SetRenderStyle(cmd.mRenderStyle);
 		state.EnableBrightmap(!(cmd.mRenderStyle.Flags & STYLEF_ColorIsFixed));
@@ -126,7 +145,7 @@ void Draw2D(F2DDrawer *drawer, FRenderState &state)
 		if (cmd.mFlags & F2DDrawer::DTF_Indexed) state.SetSoftLightLevel(cmd.mLightLevel);
 		state.SetLightParms(0, 0);
 
-		state.AlphaFunc(Alpha_GEqual, 0.f);
+		state.AlphaFunc(Alpha_Greater, 0.f);
 
 		if (cmd.useTransform)
 		{
@@ -225,6 +244,8 @@ void Draw2D(F2DDrawer *drawer, FRenderState &state)
 
 	state.SetRenderStyle(STYLE_Translucent);
 	state.SetVertexBuffer(screen->mVertexData);
+	state.EnableStencil(false);
+	state.SetStencil(0, SOP_Keep, SF_AllOn);
 	state.EnableTexture(true);
 	state.EnableBrightmap(true);
 	state.SetTextureMode(TM_NORMAL);

src/common/rendering/vulkan/renderer/vk_postprocess.cpp

@@ -57,9 +57,10 @@ void VkPostprocess::SetActiveRenderTarget()
 
 	VkImageTransition()
 		.AddImage(&buffers->PipelineImage[mCurrentPipelineImage], VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, false)
+		.AddImage(&buffers->PipelineDepthStencil, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, false)
 		.Execute(fb->GetCommands()->GetDrawCommands());
 
-	fb->GetRenderState()->SetRenderTarget(&buffers->PipelineImage[mCurrentPipelineImage], nullptr, buffers->GetWidth(), buffers->GetHeight(), VK_FORMAT_R16G16B16A16_SFLOAT, VK_SAMPLE_COUNT_1_BIT);
+	fb->GetRenderState()->SetRenderTarget(&buffers->PipelineImage[mCurrentPipelineImage], buffers->PipelineDepthStencil.View.get(), buffers->GetWidth(), buffers->GetHeight(), VK_FORMAT_R16G16B16A16_SFLOAT, VK_SAMPLE_COUNT_1_BIT);
 }
 
 void VkPostprocess::PostProcessScene(int fixedcm, float flash, const std::function<void()> &afterBloomDrawEndScene2D)

src/common/rendering/vulkan/renderer/vk_pprenderstate.cpp

@@ -71,12 +71,12 @@ void VkPPRenderState::Draw()
 
 	if (Output.Type == PPTextureType::SceneColor)
 	{
-		key.StencilTest = 1;
+		key.StencilTest = WhichDepthStencil::Scene;
 		key.Samples = fb->GetBuffers()->GetSceneSamples();
 	}
 	else
 	{
-		key.StencilTest = 0;
+		key.StencilTest = WhichDepthStencil::None;
 		key.Samples = VK_SAMPLE_COUNT_1_BIT;
 	}
 
@@ -86,7 +86,7 @@ void VkPPRenderState::Draw()
 	VulkanDescriptorSet *input = fb->GetDescriptorSetManager()->GetInput(passSetup, Textures, ShadowMapBuffers);
 	VulkanFramebuffer *output = fb->GetBuffers()->GetOutput(passSetup, Output, key.StencilTest, framebufferWidth, framebufferHeight);
 
-	RenderScreenQuad(passSetup, input, output, framebufferWidth, framebufferHeight, Viewport.left, Viewport.top, Viewport.width, Viewport.height, Uniforms.Data.Data(), Uniforms.Data.Size(), key.StencilTest);
+	RenderScreenQuad(passSetup, input, output, framebufferWidth, framebufferHeight, Viewport.left, Viewport.top, Viewport.width, Viewport.height, Uniforms.Data.Data(), Uniforms.Data.Size(), key.StencilTest == WhichDepthStencil::Scene);
 
 	// Advance to next PP texture if our output was sent there
 	if (Output.Type == PPTextureType::NextPipelineTexture)

src/common/rendering/vulkan/renderer/vk_renderpass.cpp

@@ -342,7 +342,7 @@ void VkPPRenderPassSetup::CreatePipeline(const VkPPRenderPassKey& key)
 	// Note: the actual values are ignored since we use dynamic viewport+scissor states
 	builder.Viewport(0.0f, 0.0f, 320.0f, 200.0f);
 	builder.Scissor(0, 0, 320, 200);
-	if (key.StencilTest)
+	if (key.StencilTest != WhichDepthStencil::None)
 	{
 		builder.AddDynamicState(VK_DYNAMIC_STATE_STENCIL_REFERENCE);
 		builder.DepthStencilEnable(false, false, true);
@@ -364,7 +364,7 @@ void VkPPRenderPassSetup::CreateRenderPass(const VkPPRenderPassKey& key)
 		builder.AddAttachment(key.OutputFormat, key.Samples, VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
 	else
 		builder.AddAttachment(key.OutputFormat, key.Samples, VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
-	if (key.StencilTest)
+	if (key.StencilTest == WhichDepthStencil::Scene)
 	{
 		builder.AddDepthStencilAttachment(
 			fb->GetBuffers()->SceneDepthStencilFormat, key.Samples,
@@ -372,10 +372,18 @@ void VkPPRenderPassSetup::CreateRenderPass(const VkPPRenderPassKey& key)
 			VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE,
 			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
 	}
+	if (key.StencilTest == WhichDepthStencil::Pipeline)
+	{
+		builder.AddDepthStencilAttachment(
+			fb->GetBuffers()->PipelineDepthStencilFormat, key.Samples,
+			VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE,
+			VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE,
+			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
+	}
 
 	builder.AddSubpass();
 	builder.AddSubpassColorAttachmentRef(0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
-	if (key.StencilTest)
+	if (key.StencilTest != WhichDepthStencil::None)
 	{
 		builder.AddSubpassDepthStencilAttachmentRef(1, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
 		builder.AddExternalSubpassDependency(

src/common/rendering/vulkan/renderer/vk_renderpass.h

@@ -78,6 +78,8 @@ public:
 	int UseVertexData;
 };
 
+enum class WhichDepthStencil;
+
 class VkPPRenderPassKey
 {
 public:
@@ -88,7 +90,7 @@ public:
 	VkFormat OutputFormat;
 	int SwapChain;
 	int ShadowMapBuffers;
-	int StencilTest;
+	WhichDepthStencil StencilTest;
 	VkSampleCountFlagBits Samples;
 
 	bool operator<(const VkPPRenderPassKey& other) const { return memcmp(this, &other, sizeof(VkPPRenderPassKey)) < 0; }

src/common/rendering/vulkan/textures/vk_renderbuffers.cpp

@@ -83,14 +83,49 @@ void VkRenderBuffers::BeginFrame(int width, int height, int sceneWidth, int scen
 	mSceneHeight = sceneHeight;
 }
 
+void VkRenderBuffers::CreatePipelineDepthStencil(int width, int height)
+{
+	ImageBuilder builder;
+	builder.Size(width, height);
+	builder.Format(PipelineDepthStencilFormat);
+	builder.Usage(VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT);
+	if (!builder.IsFormatSupported(fb->device))
+	{
+		PipelineDepthStencilFormat = VK_FORMAT_D32_SFLOAT_S8_UINT;
+		builder.Format(PipelineDepthStencilFormat);
+		if (!builder.IsFormatSupported(fb->device))
+		{
+			I_FatalError("This device does not support any of the required depth stencil image formats.");
+		}
+	}
+	builder.DebugName("VkRenderBuffers.PipelineDepthStencil");
+
+	PipelineDepthStencil.Image = builder.Create(fb->device);
+	PipelineDepthStencil.AspectMask = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
+
+	PipelineDepthStencil.View = ImageViewBuilder()
+		.Image(PipelineDepthStencil.Image.get(), PipelineDepthStencilFormat, VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)
+		.DebugName("VkRenderBuffers.PipelineDepthStencilView")
+		.Create(fb->device);
+
+	PipelineDepthStencil.DepthOnlyView = ImageViewBuilder()
+		.Image(PipelineDepthStencil.Image.get(), PipelineDepthStencilFormat, VK_IMAGE_ASPECT_DEPTH_BIT)
+		.DebugName("VkRenderBuffers.PipelineDepthView")
+		.Create(fb->device);
+}
+
 void VkRenderBuffers::CreatePipeline(int width, int height)
 {
 	for (int i = 0; i < NumPipelineImages; i++)
 	{
 		PipelineImage[i].Reset(fb);
 	}
+	PipelineDepthStencil.Reset(fb);
+
+	CreatePipelineDepthStencil(width, height);
 
 	VkImageTransition barrier;
+	barrier.AddImage(&PipelineDepthStencil, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, true);
 	for (int i = 0; i < NumPipelineImages; i++)
 	{
 		PipelineImage[i].Image = ImageBuilder()
@@ -216,7 +251,7 @@ void VkRenderBuffers::CreateSceneNormal(int width, int height, VkSampleCountFlag
 		.Create(fb->device);
 }
 
-VulkanFramebuffer* VkRenderBuffers::GetOutput(VkPPRenderPassSetup* passSetup, const PPOutput& output, bool stencilTest, int& framebufferWidth, int& framebufferHeight)
+VulkanFramebuffer* VkRenderBuffers::GetOutput(VkPPRenderPassSetup* passSetup, const PPOutput& output, WhichDepthStencil stencilTest, int& framebufferWidth, int& framebufferHeight)
 {
 	VkTextureImage* tex = fb->GetTextureManager()->GetTexture(output.Type, output.Texture);
 
@@ -227,8 +262,12 @@ VulkanFramebuffer* VkRenderBuffers::GetOutput(VkPPRenderPassSetup* passSetup, co
 	{
 		VkImageTransition imageTransition;
 		imageTransition.AddImage(tex, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, output.Type == PPTextureType::NextPipelineTexture);
-		if (stencilTest)
+		if (stencilTest == WhichDepthStencil::Scene)
 			imageTransition.AddImage(&fb->GetBuffers()->SceneDepthStencil, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, false);
+
+		if (stencilTest == WhichDepthStencil::Pipeline)
+			imageTransition.AddImage(&fb->GetBuffers()->PipelineDepthStencil, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, false);
+
 		imageTransition.Execute(fb->GetCommands()->GetDrawCommands());
 
 		view = tex->View->view;
@@ -251,8 +290,10 @@ VulkanFramebuffer* VkRenderBuffers::GetOutput(VkPPRenderPassSetup* passSetup, co
 		builder.RenderPass(passSetup->RenderPass.get());
 		builder.Size(w, h);
 		builder.AddAttachment(view);
-		if (stencilTest)
+		if (stencilTest == WhichDepthStencil::Scene)
 			builder.AddAttachment(fb->GetBuffers()->SceneDepthStencil.View.get());
+		if (stencilTest == WhichDepthStencil::Pipeline)
+			builder.AddAttachment(fb->GetBuffers()->PipelineDepthStencil.View.get());
 		builder.DebugName("PPOutputFB");
 		framebuffer = builder.Create(fb->device);
 	}

src/common/rendering/vulkan/textures/vk_renderbuffers.h

@@ -8,6 +8,12 @@ class VulkanFrameBuffer;
 class VkPPRenderPassSetup;
 class PPOutput;
 
+enum class WhichDepthStencil {
+	None,
+	Scene,
+	Pipeline,
+};
+
 class VkRenderBuffers
 {
 public:
@@ -27,15 +33,18 @@ public:
 	VkTextureImage SceneNormal;
 	VkTextureImage SceneFog;
 
+	VkFormat PipelineDepthStencilFormat = VK_FORMAT_D24_UNORM_S8_UINT;
 	VkFormat SceneDepthStencilFormat = VK_FORMAT_D24_UNORM_S8_UINT;
 	VkFormat SceneNormalFormat = VK_FORMAT_A2R10G10B10_UNORM_PACK32;
 
 	static const int NumPipelineImages = 2;
+	VkTextureImage PipelineDepthStencil;
 	VkTextureImage PipelineImage[NumPipelineImages];
 
-	VulkanFramebuffer* GetOutput(VkPPRenderPassSetup* passSetup, const PPOutput& output, bool stencilTest, int& framebufferWidth, int& framebufferHeight);
+	VulkanFramebuffer* GetOutput(VkPPRenderPassSetup* passSetup, const PPOutput& output, WhichDepthStencil stencilTest, int& framebufferWidth, int& framebufferHeight);
 
 private:
+	void CreatePipelineDepthStencil(int width, int height);
 	void CreatePipeline(int width, int height);
 	void CreateScene(int width, int height, VkSampleCountFlagBits samples);
 	void CreateSceneColor(int width, int height, VkSampleCountFlagBits samples);

wadsrc/static/zscript/engine/base.zs

@@ -463,6 +463,19 @@ enum DrawTextureTags
 
 };
 
+enum StencilOp
+{
+	SOP_Keep = 0,
+	SOP_Increment = 1,
+	SOP_Decrement = 2
+};
+enum StencilFlags
+{
+	SF_AllOn = 0,
+	SF_ColorMaskOff = 1,
+	SF_DepthMaskOff = 2
+};
+
 class Shape2DTransform : Object native
 {
 	native void Clear();
@@ -516,6 +529,9 @@ struct Screen native
 	native static void ClearScreen(color col = 0);
 	native static void SetScreenFade(double factor);
 
+	native static void EnableStencil(bool on);
+	native static void SetStencil(int offs, int op, int flags = -1);
+	native static void ClearStencil();
 	native static void SetTransform(Shape2DTransform transform);
 	native static void ClearTransform();
 }