- load all the shaders and use the right one for each renderpass

src/rendering/gl/shaders/gl_shader.cpp

@@ -634,55 +634,6 @@ FShader *FShaderCollection::Compile (const char *ShaderName, const char *ShaderP
 //
 //
 //==========================================================================
-struct FDefaultShader 
-{
-	const char * ShaderName;
-	const char * gettexelfunc;
-	const char * lightfunc;
-	const char * Defines;
-};
-
-// Note: the MaterialShaderIndex enum in gl_shader.h needs to be updated whenever this array is modified.
-static const FDefaultShader defaultshaders[]=
-{	
-	{"Default",	"shaders/glsl/func_normal.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Warp 1",	"shaders/glsl/func_warp1.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Warp 2",	"shaders/glsl/func_warp2.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Brightmap","shaders/glsl/func_brightmap.fp", "shaders/glsl/material_normal.fp", "#define BRIGHTMAP\n"},
-	{"Specular", "shaders/glsl/func_spec.fp", "shaders/glsl/material_specular.fp", "#define SPECULAR\n#define NORMALMAP\n"},
-	{"SpecularBrightmap", "shaders/glsl/func_spec.fp", "shaders/glsl/material_specular.fp", "#define SPECULAR\n#define NORMALMAP\n#define BRIGHTMAP\n"},
-	{"PBR","shaders/glsl/func_pbr.fp", "shaders/glsl/material_pbr.fp", "#define PBR\n#define NORMALMAP\n"},
-	{"PBRBrightmap","shaders/glsl/func_pbr.fp", "shaders/glsl/material_pbr.fp", "#define PBR\n#define NORMALMAP\n#define BRIGHTMAP\n"},
-	{"Paletted",	"shaders/glsl/func_paletted.fp", "shaders/glsl/material_nolight.fp", ""},
-	{"No Texture", "shaders/glsl/func_notexture.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Basic Fuzz", "shaders/glsl/fuzz_standard.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Smooth Fuzz", "shaders/glsl/fuzz_smooth.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Swirly Fuzz", "shaders/glsl/fuzz_swirly.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Translucent Fuzz", "shaders/glsl/fuzz_smoothtranslucent.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Jagged Fuzz", "shaders/glsl/fuzz_jagged.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Noise Fuzz", "shaders/glsl/fuzz_noise.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Smooth Noise Fuzz", "shaders/glsl/fuzz_smoothnoise.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Software Fuzz", "shaders/glsl/fuzz_software.fp", "shaders/glsl/material_normal.fp", ""},
-	{nullptr,nullptr,nullptr,nullptr}
-};
-
-struct FEffectShader
-{
-	const char *ShaderName;
-	const char *vp;
-	const char *fp1;
-	const char *fp2;
-	const char *fp3;
-	const char *defines;
-};
-
-static const FEffectShader effectshaders[]=
-{
-	{ "fogboundary", "shaders/glsl/main.vp", "shaders/glsl/fogboundary.fp", nullptr, nullptr, "#define NO_ALPHATEST\n" },
-	{ "spheremap", "shaders/glsl/main.vp", "shaders/glsl/main.fp", "shaders/glsl/func_normal.fp", "shaders/glsl/material_normal.fp", "#define SPHEREMAP\n#define NO_ALPHATEST\n" },
-	{ "burn", "shaders/glsl/main.vp", "shaders/glsl/burn.fp", nullptr, nullptr, "#define SIMPLE\n#define NO_ALPHATEST\n" },
-	{ "stencil", "shaders/glsl/main.vp", "shaders/glsl/stencil.fp", nullptr, nullptr, "#define SIMPLE\n#define NO_ALPHATEST\n" },
-};
 
 FShaderManager::FShaderManager()
 {

src/rendering/hwrenderer/utility/hw_shaderpatcher.cpp

@@ -185,3 +185,36 @@ FString RemoveSamplerBindings(FString code, TArray<std::pair<FString, int>> &sam
 	return code;
 }
 
+/////////////////////////////////////////////////////////////////////////////
+
+// Note: the MaterialShaderIndex enum in gl_shader.h needs to be updated whenever this array is modified.
+const FDefaultShader defaultshaders[] =
+{
+	{"Default",	"shaders/glsl/func_normal.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Warp 1",	"shaders/glsl/func_warp1.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Warp 2",	"shaders/glsl/func_warp2.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Brightmap","shaders/glsl/func_brightmap.fp", "shaders/glsl/material_normal.fp", "#define BRIGHTMAP\n"},
+	{"Specular", "shaders/glsl/func_spec.fp", "shaders/glsl/material_specular.fp", "#define SPECULAR\n#define NORMALMAP\n"},
+	{"SpecularBrightmap", "shaders/glsl/func_spec.fp", "shaders/glsl/material_specular.fp", "#define SPECULAR\n#define NORMALMAP\n#define BRIGHTMAP\n"},
+	{"PBR","shaders/glsl/func_pbr.fp", "shaders/glsl/material_pbr.fp", "#define PBR\n#define NORMALMAP\n"},
+	{"PBRBrightmap","shaders/glsl/func_pbr.fp", "shaders/glsl/material_pbr.fp", "#define PBR\n#define NORMALMAP\n#define BRIGHTMAP\n"},
+	{"Paletted",	"shaders/glsl/func_paletted.fp", "shaders/glsl/material_nolight.fp", ""},
+	{"No Texture", "shaders/glsl/func_notexture.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Basic Fuzz", "shaders/glsl/fuzz_standard.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Smooth Fuzz", "shaders/glsl/fuzz_smooth.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Swirly Fuzz", "shaders/glsl/fuzz_swirly.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Translucent Fuzz", "shaders/glsl/fuzz_smoothtranslucent.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Jagged Fuzz", "shaders/glsl/fuzz_jagged.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Noise Fuzz", "shaders/glsl/fuzz_noise.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Smooth Noise Fuzz", "shaders/glsl/fuzz_smoothnoise.fp", "shaders/glsl/material_normal.fp", ""},
+	{"Software Fuzz", "shaders/glsl/fuzz_software.fp", "shaders/glsl/material_normal.fp", ""},
+	{nullptr,nullptr,nullptr,nullptr}
+};
+
+const FEffectShader effectshaders[] =
+{
+	{ "fogboundary", "shaders/glsl/main.vp", "shaders/glsl/fogboundary.fp", nullptr, nullptr, "#define NO_ALPHATEST\n" },
+	{ "spheremap", "shaders/glsl/main.vp", "shaders/glsl/main.fp", "shaders/glsl/func_normal.fp", "shaders/glsl/material_normal.fp", "#define SPHEREMAP\n#define NO_ALPHATEST\n" },
+	{ "burn", "shaders/glsl/main.vp", "shaders/glsl/burn.fp", nullptr, nullptr, "#define SIMPLE\n#define NO_ALPHATEST\n" },
+	{ "stencil", "shaders/glsl/main.vp", "shaders/glsl/stencil.fp", nullptr, nullptr, "#define SIMPLE\n#define NO_ALPHATEST\n" },
+};

src/rendering/hwrenderer/utility/hw_shaderpatcher.h

@@ -7,3 +7,23 @@
 FString RemoveLegacyUserUniforms(FString code);
 FString RemoveSamplerBindings(FString code, TArray<std::pair<FString, int>> &samplerstobind);	// For GL 3.3 compatibility which cannot declare sampler bindings in the sampler source.
 
+struct FDefaultShader
+{
+	const char * ShaderName;
+	const char * gettexelfunc;
+	const char * lightfunc;
+	const char * Defines;
+};
+
+struct FEffectShader
+{
+	const char *ShaderName;
+	const char *vp;
+	const char *fp1;
+	const char *fp2;
+	const char *fp3;
+	const char *defines;
+};
+
+extern const FDefaultShader defaultshaders[];
+extern const FEffectShader effectshaders[];

src/rendering/vulkan/renderer/vk_renderpass.cpp

@@ -138,8 +138,18 @@ void VkRenderPassSetup::CreatePipeline(const VkRenderPassKey &key)
 {
 	auto fb = GetVulkanFrameBuffer();
 	GraphicsPipelineBuilder builder;
-	builder.addVertexShader(fb->GetShaderManager()->vert.get());
+
-	builder.addFragmentShader(fb->GetShaderManager()->frag.get());
+	VkShaderProgram *program;
+	if (key.SpecialEffect != EFF_NONE)
+	{
+		program = fb->GetShaderManager()->GetEffect(key.SpecialEffect);
+	}
+	else
+	{
+		program = fb->GetShaderManager()->Get(key.EffectState, key.AlphaTest);
+	}
+	builder.addVertexShader(program->vert.get());
+	builder.addFragmentShader(program->frag.get());
 
 	builder.addVertexBufferBinding(0, sizeof(F2DDrawer::TwoDVertex));
 	builder.addVertexAttribute(0, 0, VK_FORMAT_R32G32B32_SFLOAT, offsetof(F2DDrawer::TwoDVertex, x));

src/rendering/vulkan/renderer/vk_renderpass.h

@@ -11,10 +11,15 @@ class VkRenderPassKey
 {
 public:
 	FRenderStyle RenderStyle;
+	int SpecialEffect;
+	int EffectState;
+	bool AlphaTest;
 
 	bool operator<(const VkRenderPassKey &other) const
 	{
-		return RenderStyle.AsDWORD < other.RenderStyle.AsDWORD;
+		uint64_t a = RenderStyle.AsDWORD | (static_cast<uint64_t>(SpecialEffect) << 32) | (static_cast<uint64_t>(EffectState) << 40) | (static_cast<uint64_t>(AlphaTest) << 48);
+		uint64_t b = other.RenderStyle.AsDWORD | (static_cast<uint64_t>(other.SpecialEffect) << 32) | (static_cast<uint64_t>(other.EffectState) << 40) | (static_cast<uint64_t>(other.AlphaTest) << 48);
+		return a < b;
 	}
 };
 

src/rendering/vulkan/renderer/vk_renderstate.cpp

@@ -148,6 +148,19 @@ void VkRenderState::Apply(int dt)
 	// Find a render pass that matches our state
 	VkRenderPassKey passKey;
 	passKey.RenderStyle = mRenderStyle;
+	if (mSpecialEffect > EFF_NONE)
+	{
+		passKey.SpecialEffect = mSpecialEffect;
+		passKey.EffectState = 0;
+		passKey.AlphaTest = false;
+	}
+	else
+	{
+		int effectState = mMaterial.mOverrideShader >= 0 ? mMaterial.mOverrideShader : (mMaterial.mMaterial ? mMaterial.mMaterial->GetShaderIndex() : 0);
+		passKey.SpecialEffect = EFF_NONE;
+		passKey.EffectState = mTextureEnabled ? effectState : SHADER_NoTexture;
+		passKey.AlphaTest = mAlphaThreshold >= 0.f;
+	}
 	VkRenderPassSetup *passSetup = passManager->GetRenderPass(passKey);
 
 	// Is this the one we already have or do we need to change render pass?

src/rendering/vulkan/shaders/vk_shader.cpp

@@ -10,8 +10,46 @@ VkShaderManager::VkShaderManager(VulkanDevice *device) : device(device)
 {
 	ShInitialize();
 
-	vert = LoadVertShader("shaders/glsl/main.vp", "");
+	const char *mainvp = "shaders/glsl/main.vp";
-	frag = LoadFragShader("shaders/glsl/main.fp", "shaders/glsl/func_normal.fp", "shaders/glsl/material_normal.fp", "");
+	const char *mainfp = "shaders/glsl/main.fp";
+	bool gbufferpass = false;
+
+	for (int i = 0; defaultshaders[i].ShaderName != NULL; i++)
+	{
+		// To do: use defaultshaders[i].ShaderName for better debugging
+
+		VkShaderProgram prog;
+		prog.vert = LoadVertShader(defaultshaders[i].ShaderName, mainvp, defaultshaders[i].Defines);
+		prog.frag = LoadFragShader(defaultshaders[i].ShaderName, mainfp, defaultshaders[i].gettexelfunc, defaultshaders[i].lightfunc, defaultshaders[i].Defines, true, gbufferpass);
+		mMaterialShaders.push_back(std::move(prog));
+
+		if (i < SHADER_NoTexture)
+		{
+			VkShaderProgram natprog;
+			natprog.vert = LoadVertShader(defaultshaders[i].ShaderName, mainvp, defaultshaders[i].Defines);
+			natprog.frag = LoadFragShader(defaultshaders[i].ShaderName, mainfp, defaultshaders[i].gettexelfunc, defaultshaders[i].lightfunc, defaultshaders[i].Defines, false, gbufferpass);
+			mMaterialShadersNAT.push_back(std::move(natprog));
+		}
+	}
+
+	for (unsigned i = 0; i < usershaders.Size(); i++)
+	{
+		FString name = ExtractFileBase(usershaders[i].shader);
+		FString defines = defaultshaders[usershaders[i].shaderType].Defines + usershaders[i].defines;
+
+		VkShaderProgram prog;
+		prog.vert = LoadVertShader(name, mainvp, defaultshaders[i].Defines);
+		prog.frag = LoadFragShader(name, mainfp, usershaders[i].shader, defaultshaders[usershaders[i].shaderType].lightfunc, defines, true, gbufferpass);
+		mMaterialShaders.push_back(std::move(prog));
+	}
+
+	for (int i = 0; i < MAX_EFFECTS; i++)
+	{
+		VkShaderProgram prog;
+		prog.vert = LoadVertShader(effectshaders[i].ShaderName, effectshaders[i].vp, defaultshaders[i].Defines);
+		prog.frag = LoadFragShader(effectshaders[i].ShaderName, effectshaders[i].fp1, effectshaders[i].fp2, effectshaders[i].fp3, effectshaders[i].defines, true, false);
+		mEffectShaders[i] = std::move(prog);
+	}
 }
 
 VkShaderManager::~VkShaderManager()
@@ -19,6 +57,29 @@ VkShaderManager::~VkShaderManager()
 	ShFinalize();
 }
 
+VkShaderProgram *VkShaderManager::GetEffect(int effect)
+{
+	if (effect >= 0 && effect < MAX_EFFECTS && mEffectShaders[effect].frag)
+	{
+		return &mEffectShaders[effect];
+	}
+	return nullptr;
+}
+
+VkShaderProgram *VkShaderManager::Get(unsigned int eff, bool alphateston)
+{
+	// indices 0-2 match the warping modes, 3 is brightmap, 4 no texture, the following are custom
+	if (!alphateston && eff <= 3)
+	{
+		return &mMaterialShadersNAT[eff];	// Non-alphatest shaders are only created for default, warp1+2 and brightmap. The rest won't get used anyway
+	}
+	else if (eff < (unsigned int)mMaterialShaders.size())
+	{
+		return &mMaterialShaders[eff];
+	}
+	return nullptr;
+}
+
 static const char *shaderBindings = R"(
 
 	// This must match the HWViewpointUniforms struct
@@ -125,7 +186,7 @@ static const char *shaderBindings = R"(
 	#define VULKAN_COORDINATE_SYSTEM
 )";
 
-std::unique_ptr<VulkanShader> VkShaderManager::LoadVertShader(const char *vert_lump, const char *defines)
+std::unique_ptr<VulkanShader> VkShaderManager::LoadVertShader(FString shadername, const char *vert_lump, const char *defines)
 {
 	FString code = GetTargetGlslVersion();
 	code << defines << shaderBindings;
@@ -137,10 +198,14 @@ std::unique_ptr<VulkanShader> VkShaderManager::LoadVertShader(const char *vert_l
 	return builder.create(device);
 }
 
-std::unique_ptr<VulkanShader> VkShaderManager::LoadFragShader(const char *frag_lump, const char *material_lump, const char *light_lump, const char *defines)
+std::unique_ptr<VulkanShader> VkShaderManager::LoadFragShader(FString shadername, const char *frag_lump, const char *material_lump, const char *light_lump, const char *defines, bool alphatest, bool gbufferpass)
 {
 	FString code = GetTargetGlslVersion();
 	code << defines << shaderBindings;
+
+	if (!alphatest) code << "#define NO_ALPHATEST\n";
+	if (gbufferpass) code << "#define GBUFFER_PASS\n";
+
 	code << "\n#line 1\n";
 	code << LoadShaderLump(frag_lump).GetChars() << "\n";
 

src/rendering/vulkan/shaders/vk_shader.h

@@ -4,6 +4,7 @@
 #include "utility/vectors.h"
 #include "r_data/matrix.h"
 #include "name.h"
+#include "hwrenderer/scene/hw_renderstate.h"
 
 class VulkanDevice;
 class VulkanShader;
@@ -63,21 +64,32 @@ struct PushConstants
 	FVector2 uSpecularMaterial;
 };
 
+class VkShaderProgram
+{
+public:
+	std::unique_ptr<VulkanShader> vert;
+	std::unique_ptr<VulkanShader> frag;
+};
+
 class VkShaderManager
 {
 public:
 	VkShaderManager(VulkanDevice *device);
 	~VkShaderManager();
 
-	std::unique_ptr<VulkanShader> vert;
+	VkShaderProgram *GetEffect(int effect);
-	std::unique_ptr<VulkanShader> frag;
+	VkShaderProgram *Get(unsigned int eff, bool alphateston);
 
 private:
-	std::unique_ptr<VulkanShader> LoadVertShader(const char *vert_lump, const char *defines);
+	std::unique_ptr<VulkanShader> LoadVertShader(FString shadername, const char *vert_lump, const char *defines);
-	std::unique_ptr<VulkanShader> LoadFragShader(const char *frag_lump, const char *material_lump, const char *light_lump, const char *defines);
+	std::unique_ptr<VulkanShader> LoadFragShader(FString shadername, const char *frag_lump, const char *material_lump, const char *light_lump, const char *defines, bool alphatest, bool gbufferpass);
 
 	FString GetTargetGlslVersion();
 	FString LoadShaderLump(const char *lumpname);
 
 	VulkanDevice *device;
+
+	std::vector<VkShaderProgram> mMaterialShaders;
+	std::vector<VkShaderProgram> mMaterialShadersNAT;
+	VkShaderProgram mEffectShaders[MAX_EFFECTS];
 };

wadsrc/static/shaders/glsl/burn.fp

@@ -1,6 +1,6 @@
 
-in vec4 vTexCoord;
+layout(location=0) in vec4 vTexCoord;
-in vec4 vColor;
+layout(location=1) in vec4 vColor;
 layout(location=0) out vec4 FragColor;
 
 void main()

wadsrc/static/shaders/glsl/fogboundary.fp

@@ -1,4 +1,4 @@
-in vec4 pixelpos;
+layout(location=2) in vec4 pixelpos;
 layout(location=0) out vec4 FragColor;
 
 //===========================================================================

wadsrc/static/shaders/glsl/stencil.fp

@@ -1,4 +1,4 @@
-in vec4 pixelpos;
+layout(location=2) in vec4 pixelpos;
 layout(location=0) out vec4 FragColor;
 
 void main()