- fix the multisample and image transition validation errors

src/rendering/vulkan/renderer/vk_postprocess.cpp

@@ -30,7 +30,16 @@ void VkPostprocess::SetActiveRenderTarget()
 {
 	auto fb = GetVulkanFrameBuffer();
 	auto buffers = fb->GetBuffers();
-	fb->GetRenderPassManager()->SetRenderTarget(buffers->PipelineView[mCurrentPipelineImage].get(), buffers->GetWidth(), buffers->GetHeight());
+
+	if (buffers->PipelineLayout[mCurrentPipelineImage] != VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL)
+	{
+		PipelineBarrier barrier;
+		barrier.addImage(buffers->PipelineImage[mCurrentPipelineImage].get(), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT);
+		barrier.execute(fb->GetDrawCommands(), VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
+		buffers->PipelineLayout[mCurrentPipelineImage] = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+	}
+
+	fb->GetRenderPassManager()->SetRenderTarget(buffers->PipelineView[mCurrentPipelineImage].get(), buffers->GetWidth(), buffers->GetHeight(), VK_SAMPLE_COUNT_1_BIT);
 }
 
 void VkPostprocess::PostProcessScene(int fixedcm, const std::function<void()> &afterBloomDrawEndScene2D)
@@ -431,6 +440,12 @@ void VkPostprocess::RenderScreenQuad(VkPPRenderPassSetup *passSetup, VulkanDescr
 	viewport.minDepth = 0.0f;
 	viewport.maxDepth = 1.0f;
 
+	VkRect2D scissor = { };
+	scissor.offset.x = x;
+	scissor.offset.y = y;
+	scissor.extent.width = width;
+	scissor.extent.height = height;
+
 	VkBuffer vertexBuffers[] = { static_cast<VKVertexBuffer*>(screen->mVertexData->GetBufferObjects().first)->mBuffer->buffer };
 	VkDeviceSize offsets[] = { 0 };
 
@@ -439,6 +454,7 @@ void VkPostprocess::RenderScreenQuad(VkPPRenderPassSetup *passSetup, VulkanDescr
 	cmdbuffer->bindDescriptorSet(VK_PIPELINE_BIND_POINT_GRAPHICS, passSetup->PipelineLayout.get(), 0, descriptorSet);
 	cmdbuffer->bindVertexBuffers(0, 1, vertexBuffers, offsets);
 	cmdbuffer->setViewport(0, 1, &viewport);
+	cmdbuffer->setScissor(0, 1, &scissor);
 	if (pushConstantsSize > 0)
 		cmdbuffer->pushConstants(passSetup->PipelineLayout.get(), VK_SHADER_STAGE_FRAGMENT_BIT, 0, pushConstantsSize, pushConstants);
 	cmdbuffer->draw(4, 1, FFlatVertexBuffer::PRESENT_INDEX, 0);
@@ -469,13 +485,14 @@ VulkanDescriptorSet *VkPostprocess::GetInput(VkPPRenderPassSetup *passSetup, con
 		{
 			srcPipelineBits |= VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
 			barrier.addImage(tex.image, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
-			*tex.layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 			needbarrier = true;
+			*tex.layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 		}
 		else if (*tex.layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
 		{
 			srcPipelineBits |= VK_PIPELINE_STAGE_TRANSFER_BIT;
-			barrier.addImage(tex.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
+			barrier.addImage(tex.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
+			needbarrier = true;
 			*tex.layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 		}
 	}
@@ -664,7 +681,9 @@ void VkPPRenderPassSetup::CreatePipeline(const VkPPRenderPassKey &key)
 	builder.addVertexAttribute(0, 0, VK_FORMAT_R32G32B32_SFLOAT, offsetof(FFlatVertex, x));
 	builder.addVertexAttribute(1, 0, VK_FORMAT_R32G32_SFLOAT, offsetof(FFlatVertex, u));
 	builder.addDynamicState(VK_DYNAMIC_STATE_VIEWPORT);
+	builder.addDynamicState(VK_DYNAMIC_STATE_SCISSOR);
 	builder.setViewport(0.0f, 0.0f, (float)SCREENWIDTH, (float)SCREENHEIGHT);
+	builder.setScissor(0.0f, 0.0f, (float)SCREENWIDTH, (float)SCREENHEIGHT);
 	builder.setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP);
 	builder.setBlendMode(key.BlendMode);
 	builder.setLayout(PipelineLayout.get());
@@ -677,9 +696,9 @@ void VkPPRenderPassSetup::CreateRenderPass(const VkPPRenderPassKey &key)
 {
 	RenderPassBuilder builder;
 	if (key.SwapChain)
-		builder.addColorAttachment(true, key.OutputFormat, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
+		builder.addAttachment(key.OutputFormat, VK_SAMPLE_COUNT_1_BIT, VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
 	else
-		builder.addColorAttachment(false, key.OutputFormat, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+		builder.addAttachment(key.OutputFormat, VK_SAMPLE_COUNT_1_BIT, VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 	builder.addSubpass();
 	builder.addSubpassColorAttachmentRef(0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 	builder.addExternalSubpassDependency(

src/rendering/vulkan/renderer/vk_renderbuffers.cpp

@@ -78,7 +78,7 @@ void VkRenderBuffers::CreatePipeline(int width, int height)
 		ImageBuilder builder;
 		builder.setSize(width, height);
 		builder.setFormat(VK_FORMAT_R16G16B16A16_SFLOAT);
-		builder.setUsage(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
+		builder.setUsage(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT);
 		PipelineImage[i] = builder.create(fb->device);
 		PipelineImage[i]->SetDebugName("VkRenderBuffers.PipelineImage");
 
@@ -128,6 +128,7 @@ void VkRenderBuffers::CreateSceneColor(int width, int height, VkSampleCountFlagB
 
 	ImageBuilder builder;
 	builder.setSize(width, height);
+	builder.setSamples(samples);
 	builder.setFormat(VK_FORMAT_R16G16B16A16_SFLOAT);
 	builder.setUsage(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
 	SceneColor = builder.create(fb->device);

src/rendering/vulkan/renderer/vk_renderpass.cpp

@@ -28,12 +28,13 @@ void VkRenderPassManager::RenderBuffersReset()
 	RenderPassSetup.clear();
 }
 
-void VkRenderPassManager::SetRenderTarget(VulkanImageView *view, int width, int height)
+void VkRenderPassManager::SetRenderTarget(VulkanImageView *view, int width, int height, VkSampleCountFlagBits samples)
 {
 	GetVulkanFrameBuffer()->GetRenderState()->EndRenderPass();
 	mRenderTargetView = view;
 	mRenderTargetWidth = width;
 	mRenderTargetHeight = height;
+	mSamples = samples;
 }
 
 void VkRenderPassManager::BeginRenderPass(const VkRenderPassKey &key, VulkanCommandBuffer *cmdbuffer)
@@ -176,10 +177,21 @@ VkRenderPassSetup::VkRenderPassSetup(const VkRenderPassKey &key)
 
 void VkRenderPassSetup::CreateRenderPass(const VkRenderPassKey &key)
 {
+	auto buffers = GetVulkanFrameBuffer()->GetBuffers();
+
 	RenderPassBuilder builder;
-	builder.addRgba16fAttachment(false, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+	builder.addAttachment(
+		VK_FORMAT_R16G16B16A16_SFLOAT, (VkSampleCountFlagBits)key.Samples,
+		VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE,
+		VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 	if (key.DepthTest || key.DepthWrite || key.StencilTest)
-		builder.addDepthStencilAttachment(false, GetVulkanFrameBuffer()->GetBuffers()->SceneDepthStencilFormat, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
+	{
+		builder.addDepthStencilAttachment(
+			buffers->SceneDepthStencilFormat, buffers->GetSceneSamples(),
+			VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE,
+			VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_STORE_OP_STORE,
+			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
+	}
 	builder.addSubpass();
 	builder.addSubpassColorAttachmentRef(0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 	if (key.DepthTest || key.DepthWrite || key.StencilTest)
@@ -283,6 +295,7 @@ void VkRenderPassSetup::CreatePipeline(const VkRenderPassKey &key)
 	builder.setColorWriteMask((VkColorComponentFlags)key.ColorMask);
 	builder.setStencil(VK_STENCIL_OP_KEEP, op2vk[key.StencilPassOp], VK_STENCIL_OP_KEEP, VK_COMPARE_OP_EQUAL, 0xffffffff, 0xffffffff, 0);
 	builder.setBlendMode(key.RenderStyle);
+	builder.setRasterizationSamples((VkSampleCountFlagBits)key.Samples);
 
 	builder.setLayout(fb->GetRenderPassManager()->PipelineLayout.get());
 	builder.setRenderPass(RenderPass.get());

src/rendering/vulkan/renderer/vk_renderpass.h

@@ -27,6 +27,7 @@ public:
 	int CullMode;
 	int VertexFormat;
 	int DrawType;
+	int Samples;
 
 	bool operator<(const VkRenderPassKey &other) const { return memcmp(this, &other, sizeof(VkRenderPassKey)) < 0; }
 	bool operator==(const VkRenderPassKey &other) const { return memcmp(this, &other, sizeof(VkRenderPassKey)) == 0; }
@@ -64,11 +65,13 @@ public:
 	void Init();
 	void RenderBuffersReset();
 
-	void SetRenderTarget(VulkanImageView *view, int width, int height);
+	void SetRenderTarget(VulkanImageView *view, int width, int height, VkSampleCountFlagBits samples);
 	void BeginRenderPass(const VkRenderPassKey &key, VulkanCommandBuffer *cmdbuffer);
 
 	int GetVertexFormat(int numBindingPoints, int numAttributes, size_t stride, const FVertexBufferAttribute *attrs);
 
+	VkSampleCountFlagBits GetSamples() const { return mSamples; }
+
 	std::unique_ptr<VulkanDescriptorSetLayout> DynamicSetLayout;
 	std::unique_ptr<VulkanDescriptorSetLayout> TextureSetLayout;
 	std::unique_ptr<VulkanPipelineLayout> PipelineLayout;
@@ -91,4 +94,5 @@ private:
 	VulkanImageView *mRenderTargetView = nullptr;
 	int mRenderTargetWidth = 0;
 	int mRenderTargetHeight = 0;
+	VkSampleCountFlagBits mSamples = VK_SAMPLE_COUNT_1_BIT;
 };

src/rendering/vulkan/renderer/vk_renderstate.cpp

@@ -244,6 +244,8 @@ void VkRenderState::ApplyDepthBias()
 
 void VkRenderState::ApplyRenderPass(int dt)
 {
+	auto passManager = GetVulkanFrameBuffer()->GetRenderPassManager();
+
 	// Find a render pass that matches our state
 	VkRenderPassKey passKey;
 	passKey.DrawType = dt;
@@ -258,6 +260,7 @@ void VkRenderState::ApplyRenderPass(int dt)
 	passKey.StencilPassOp = mStencilOp;
 	passKey.ColorMask = mColorMask;
 	passKey.CullMode = mCullMode;
+	passKey.Samples = passManager->GetSamples();
 	if (mSpecialEffect > EFF_NONE)
 	{
 		passKey.SpecialEffect = mSpecialEffect;
@@ -291,7 +294,7 @@ void VkRenderState::ApplyRenderPass(int dt)
 
 	if (changingRenderPass)
 	{
-		GetVulkanFrameBuffer()->GetRenderPassManager()->BeginRenderPass(passKey, mCommandBuffer);
+		passManager->BeginRenderPass(passKey, mCommandBuffer);
 		mRenderPassKey = passKey;
 	}
 }

src/rendering/vulkan/system/vk_builders.h

@@ -189,6 +189,7 @@ public:
 	void setTopology(VkPrimitiveTopology topology);
 	void setViewport(float x, float y, float width, float height, float minDepth = 0.0f, float maxDepth = 1.0f);
 	void setScissor(int x, int y, int width, int height);
+	void setRasterizationSamples(VkSampleCountFlagBits samples);
 
 	void setCull(VkCullModeFlags cullMode, VkFrontFace frontFace);
 	void setDepthStencilEnable(bool test, bool write, bool stencil);
@@ -256,9 +257,8 @@ class RenderPassBuilder
 public:
 	RenderPassBuilder();
 
-	void addRgba16fAttachment(bool clear, VkImageLayout layout) { addColorAttachment(clear, VK_FORMAT_R16G16B16A16_SFLOAT, layout); }
+	void addAttachment(VkFormat format, VkSampleCountFlagBits samples, VkAttachmentLoadOp load, VkAttachmentStoreOp store, VkImageLayout initialLayout, VkImageLayout finalLayout);
-	void addColorAttachment(bool clear, VkFormat format, VkImageLayout layout);
+	void addDepthStencilAttachment(VkFormat format, VkSampleCountFlagBits samples, VkAttachmentLoadOp load, VkAttachmentStoreOp store, VkAttachmentLoadOp stencilLoad, VkAttachmentStoreOp stencilStore, VkImageLayout initialLayout, VkImageLayout finalLayout);
-	void addDepthStencilAttachment(bool clear, VkFormat format, VkImageLayout layout);
 
 	void addExternalSubpassDependency(VkPipelineStageFlags srcStageMask, VkPipelineStageFlags dstStageMask, VkAccessFlags srcAccessMask, VkAccessFlags dstAccessMask);
 
@@ -743,6 +743,11 @@ inline GraphicsPipelineBuilder::GraphicsPipelineBuilder()
 	dynamicState.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO;
 }
 
+inline void GraphicsPipelineBuilder::setRasterizationSamples(VkSampleCountFlagBits samples)
+{
+	multisampling.rasterizationSamples = samples;
+}
+
 inline void GraphicsPipelineBuilder::setSubpass(int subpass)
 {
 	pipelineInfo.subpass = subpass;
@@ -992,36 +997,34 @@ inline RenderPassBuilder::RenderPassBuilder()
 	renderPassInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO;
 }
 
-inline void RenderPassBuilder::addColorAttachment(bool clear, VkFormat format, VkImageLayout layout)
+inline void RenderPassBuilder::addAttachment(VkFormat format, VkSampleCountFlagBits samples, VkAttachmentLoadOp load, VkAttachmentStoreOp store, VkImageLayout initialLayout, VkImageLayout finalLayout)
 {
-	VkAttachmentDescription rgba16fAttachment = {};
+	VkAttachmentDescription attachment = {};
-	rgba16fAttachment.format = format;
+	attachment.format = format;
-	rgba16fAttachment.samples = VK_SAMPLE_COUNT_1_BIT;
+	attachment.samples = samples;
-	rgba16fAttachment.loadOp = clear ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD;
+	attachment.loadOp = load;
-	rgba16fAttachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
+	attachment.storeOp = store;
-	rgba16fAttachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+	attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
-	rgba16fAttachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
+	attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
-	rgba16fAttachment.initialLayout = clear ? VK_IMAGE_LAYOUT_UNDEFINED : layout;
+	attachment.initialLayout = initialLayout;
-	rgba16fAttachment.finalLayout = layout;
+	attachment.finalLayout = finalLayout;
-
+	attachments.push_back(attachment);
-	attachments.push_back(rgba16fAttachment);
 	renderPassInfo.pAttachments = attachments.data();
 	renderPassInfo.attachmentCount = (uint32_t)attachments.size();
 }
 
-inline void RenderPassBuilder::addDepthStencilAttachment(bool clear, VkFormat format, VkImageLayout layout)
+inline void RenderPassBuilder::addDepthStencilAttachment(VkFormat format, VkSampleCountFlagBits samples, VkAttachmentLoadOp load, VkAttachmentStoreOp store, VkAttachmentLoadOp stencilLoad, VkAttachmentStoreOp stencilStore, VkImageLayout initialLayout, VkImageLayout finalLayout)
 {
-	VkAttachmentDescription depthAttachment = {};
+	VkAttachmentDescription attachment = {};
-	depthAttachment.format = format;
+	attachment.format = format;
-	depthAttachment.samples = VK_SAMPLE_COUNT_1_BIT;
+	attachment.samples = samples;
-	depthAttachment.loadOp = clear ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD;
+	attachment.loadOp = load;
-	depthAttachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE/*VK_ATTACHMENT_STORE_OP_DONT_CARE*/;
+	attachment.storeOp = store;
-	depthAttachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+	attachment.stencilLoadOp = stencilLoad;
-	depthAttachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
+	attachment.stencilStoreOp = stencilStore;
-	depthAttachment.initialLayout = clear ? VK_IMAGE_LAYOUT_UNDEFINED : layout;
+	attachment.initialLayout = initialLayout;
-	depthAttachment.finalLayout = layout;
+	attachment.finalLayout = finalLayout;
-
+	attachments.push_back(attachment);
-	attachments.push_back(depthAttachment);
 	renderPassInfo.pAttachments = attachments.data();
 	renderPassInfo.attachmentCount = (uint32_t)attachments.size();
 }

src/rendering/vulkan/system/vk_framebuffer.cpp

@@ -331,7 +331,7 @@ sector_t *VulkanFrameBuffer::RenderViewpoint(FRenderViewpoint &mainvp, AActor *
 
 		if (mainview) // Bind the scene frame buffer and turn on draw buffers used by ssao
 		{
-			mRenderPassManager->SetRenderTarget(GetBuffers()->SceneColorView.get(), GetBuffers()->GetWidth(), GetBuffers()->GetHeight());
+			mRenderPassManager->SetRenderTarget(GetBuffers()->SceneColorView.get(), GetBuffers()->GetWidth(), GetBuffers()->GetHeight(), GetBuffers()->GetSceneSamples());
 #if 0
 			bool useSSAO = (gl_ssao != 0);
 			GetRenderState()->SetPassType(useSSAO ? GBUFFER_PASS : NORMAL_PASS);