- major shader rework

* handle brightmaps in the main shader instead of keeping separate instances around.
* added detail and glow layers from Raze.
* fixed material setup which could not guarantee that everything was initialized correctly.
* for warped textures, warp all layers. With this brightmaps finally work on warped textures.

Note: Vulkan reports a "device lost" error with this which still needs to be investigated.

src/CMakeLists.txt

@@ -1465,8 +1465,8 @@ source_group("Common\\Scripting\\Backend" REGULAR_EXPRESSION "^${CMAKE_CURRENT_S
 source_group("Common\\Scripting\\Core" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/scripting/core/.+")
 source_group("Common\\Scripting\\JIT" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/scripting/jit/.+")
 source_group("Common\\Scripting\\VM" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/scripting/vm/.+")
-source_group("Common\\Rendering" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/rendering/.+")
+source_group("Common\\Rendering" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/rendering/.+")
-source_group("Common\\Rendering\\OpenGL Loader" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/rendering/gl_load/.+")
+source_group("Common\\Rendering\\OpenGL Loader" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/rendering/gl_load/.+")
 source_group("Common\\Textures" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/textures/.+")
 source_group("Common\\Textures\\Hires" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/textures/hires/.+")
 source_group("Common\\Textures\\Hires\\HQ Resize" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/textures/hires/hqnx/.+")

src/common/2d/v_2ddrawer.cpp

@@ -755,7 +755,7 @@ void F2DDrawer::ClearScreen(PalEntry color)
 //
 //==========================================================================
 
-void F2DDrawer::AddLine(float x1, float y1, float x2, float y2, int clipx1, int clipy1, int clipx2, int clipy2, uint32_t color, uint8_t alpha)
+void F2DDrawer::AddLine(double x1, double y1, double x2, double y2, int clipx1, int clipy1, int clipx2, int clipy2, uint32_t color, uint8_t alpha)
 {
 	PalEntry p = (PalEntry)color;
 	p.a = alpha;

src/common/2d/v_2ddrawer.h

@@ -190,7 +190,7 @@ public:
 	void AddClear(int left, int top, int right, int bottom, int palcolor, uint32_t color);
 	
 		
-	void AddLine(float x1, float y1, float x2, float y2, int cx, int cy, int cx2, int cy2, uint32_t color, uint8_t alpha = 255);
+	void AddLine(double x1, double y1, double x2, double y2, int cx, int cy, int cx2, int cy2, uint32_t color, uint8_t alpha = 255);
 	void AddThickLine(int x1, int y1, int x2, int y2, double thickness, uint32_t color, uint8_t alpha = 255);
 	void AddPixel(int x1, int y1, uint32_t color);
 

src/common/filesystem/filesystem.h

@@ -38,7 +38,7 @@ public:
 	~FileData ();
 	void *GetMem () { return Block.Len() == 0 ? NULL : (void *)Block.GetChars(); }
 	size_t GetSize () { return Block.Len(); }
-	FString GetString () { return Block; }
+	const FString &GetString () const { return Block; }
 
 private:
 	FileData (const FString &source);

src/common/textures/formats/emptytexture.cpp

@@ -69,6 +69,11 @@ FImageSource *EmptyImage_TryCreate(FileReader & file, int lumpnum)
 	return new FEmptyTexture(lumpnum);
 }
 
+FImageSource* CreateEmptyTexture()
+{
+	return new FEmptyTexture(0);
+}
+
 //==========================================================================
 //
 //

src/common/textures/hw_material.cpp

@@ -45,10 +45,6 @@ FMaterial::FMaterial(FTexture * tx, bool expanded)
 	{
 		mShaderIndex = SHADER_Paletted;
 	}
-	else if (tx->isWarped())
-	{
-		mShaderIndex = tx->isWarped(); // This picks SHADER_Warp1 or SHADER_Warp2
-	}
 	else if (tx->isHardwareCanvas())
 	{
 		if (tx->shaderindex >= FIRST_USER_SHADER)
@@ -59,7 +55,11 @@ FMaterial::FMaterial(FTexture * tx, bool expanded)
 	}
 	else
 	{
-		if (tx->Normal && tx->Specular)
+		if (tx->isWarped())
+		{
+			mShaderIndex = tx->isWarped(); // This picks SHADER_Warp1 or SHADER_Warp2
+		}
+		else if (tx->Normal && tx->Specular)
 		{
 			for (auto &texture : { tx->Normal, tx->Specular })
 			{
@@ -76,16 +76,34 @@ FMaterial::FMaterial(FTexture * tx, bool expanded)
 			mShaderIndex = SHADER_PBR;
 		}
 
+		// Note that these layers must present a valid texture even if not used, because empty TMUs in the shader are an undefined condition.
 		tx->CreateDefaultBrightmap();
 		if (tx->Brightmap)
 		{
 			mTextureLayers.Push(tx->Brightmap);
-			if (mShaderIndex == SHADER_Specular)
+			mLayerFlags |= TEXF_Brightmap;
-				mShaderIndex = SHADER_SpecularBrightmap;
+		}
-			else if (mShaderIndex == SHADER_PBR)
+		else	
-				mShaderIndex = SHADER_PBRBrightmap;
+		{ 
-			else
+			mTextureLayers.Push(TexMan.ByIndex(1));
-				mShaderIndex = SHADER_Brightmap;
+		}
+		if (tx->Detailmap)
+		{
+			mTextureLayers.Push(tx->Detailmap);
+			mLayerFlags |= TEXF_Detailmap;
+		}
+		else
+		{
+			mTextureLayers.Push(TexMan.ByIndex(1));
+		}
+		if (tx->Glowmap)
+		{
+			mTextureLayers.Push(tx->Glowmap);
+			mLayerFlags |= TEXF_Glowmap;
+		}
+		else
+		{
+			mTextureLayers.Push(TexMan.ByIndex(1));
 		}
 
 		if (tx->shaderindex >= FIRST_USER_SHADER)
@@ -347,7 +365,7 @@ again:
 		{
 			if (expand)
 			{
-				if (tex->isWarped() || tex->isHardwareCanvas() || tex->shaderindex >= FIRST_USER_SHADER || (tex->shaderindex >= SHADER_Specular && tex->shaderindex <= SHADER_PBRBrightmap))
+				if (tex->isWarped() || tex->isHardwareCanvas() || tex->shaderindex >= FIRST_USER_SHADER || tex->shaderindex == SHADER_Specular || tex->shaderindex == SHADER_PBR)
 				{
 					tex->bNoExpand = true;
 					goto again;

src/common/textures/hw_material.h

@@ -30,6 +30,7 @@ class FMaterial
 {
 	TArray<FTexture*> mTextureLayers;
 	int mShaderIndex;
+	int mLayerFlags = 0;
 
 	short mLeftOffset;
 	short mTopOffset;
@@ -52,6 +53,7 @@ public:
 	
 	FMaterial(FTexture *tex, bool forceexpand);
 	~FMaterial();
+	int GetLayerFlags() const { return mLayerFlags; }
 	void SetSpriteRect();
 	int GetShaderIndex() const { return mShaderIndex; }
 	void AddTextureLayer(FTexture *tex)

src/common/textures/texture.cpp

@@ -116,7 +116,7 @@ FTexture::FTexture (const char *name, int lumpnum)
 	: 
   Scale(1,1), SourceLump(lumpnum),
   UseType(ETextureType::Any), bNoDecals(false), bNoRemap0(false), bWorldPanning(false),
-  bMasked(true), bAlphaTexture(false), bHasCanvas(false), bWarped(0), bComplex(false), bMultiPatch(false), bFullNameTexture(false),
+  bMasked(true), bAlphaTexture(false), bHasCanvas(false), bWarped(1), bComplex(false), bMultiPatch(false), bFullNameTexture(false),
 	Rotations(0xFFFF), SkyOffset(0), Width(0), Height(0)
 {
 	bBrightmapChecked = false;
@@ -384,7 +384,7 @@ void FTexture::CreateDefaultBrightmap()
 		// Check for brightmaps
 		if (GetImage() && GetImage()->UseGamePalette() && GPalette.HasGlobalBrightmap &&
 			UseType != ETextureType::Decal && UseType != ETextureType::MiscPatch && UseType != ETextureType::FontChar &&
-			Brightmap == NULL && bWarped == 0)
+			Brightmap == NULL)
 		{
 			// May have one - let's check when we use this texture
 			auto texbuf = Get8BitPixels(false);

src/common/textures/texturemanager.cpp

@@ -1083,6 +1083,7 @@ void FTextureManager::AddLocalizedVariants()
 //==========================================================================
 FTexture *CreateShaderTexture(bool, bool);
 void InitBuildTiles();
+FImageSource* CreateEmptyTexture();
 
 void FTextureManager::Init(void (*progressFunc_)(), void (*checkForHacks)(BuildInfo&))
 {
@@ -1094,6 +1095,10 @@ void FTextureManager::Init(void (*progressFunc_)(), void (*checkForHacks)(BuildI
 	auto nulltex = new FImageTexture(nullptr, "");
 	nulltex->SetUseType(ETextureType::Null);
 	AddTexture (nulltex);
+	// This is for binding to unused texture units, because accessing an unbound texture unit is undefined. It's a one pixel empty texture.
+	auto emptytex = new FImageTexture(CreateEmptyTexture(), "");
+	emptytex->SetSize(1, 1);
+	AddTexture(emptytex);	
 	// some special textures used in the game.
 	AddTexture(CreateShaderTexture(false, false));
 	AddTexture(CreateShaderTexture(false, true));

src/common/textures/textures.h

@@ -54,11 +54,8 @@ enum MaterialShaderIndex
 	SHADER_Default,
 	SHADER_Warp1,
 	SHADER_Warp2,
-	SHADER_Brightmap,
 	SHADER_Specular,
-	SHADER_SpecularBrightmap,
 	SHADER_PBR,
-	SHADER_PBRBrightmap,
 	SHADER_Paletted,
 	SHADER_NoTexture,
 	SHADER_BasicFuzz,
@@ -72,12 +69,30 @@ enum MaterialShaderIndex
 	FIRST_USER_SHADER
 };
 
+enum texflags
+{
+	// These get Or'ed into uTextureMode because it only uses its 3 lowermost bits.
+	TEXF_Brightmap = 0x10000,
+	TEXF_Detailmap = 0x20000,
+	TEXF_Glowmap = 0x40000,
+};
+
+
+
+enum
+{
+	SFlag_Brightmap = 1,
+	SFlag_Detailmap = 2,
+	SFlag_Glowmap = 4,
+};
+
 struct UserShaderDesc
 {
 	FString shader;
 	MaterialShaderIndex shaderType;
 	FString defines;
 	bool disablealphatest = false;
+	uint8_t shaderFlags = 0;
 };
 
 extern TArray<UserShaderDesc> usershaders;
@@ -341,6 +356,8 @@ protected:
 	FTexture *PalVersion = nullptr;
 	// Material layers
 	FTexture *Brightmap = nullptr;
+	FTexture* Detailmap = nullptr;
+	FTexture* Glowmap = nullptr;
 	FTexture *Normal = nullptr;							// Normal map texture
 	FTexture *Specular = nullptr;						// Specular light texture for the diffuse+normal+specular light model
 	FTexture *Metallic = nullptr;						// Metalness texture for the physically based rendering (PBR) light model

src/r_data/gldefs.cpp

@@ -1372,18 +1372,18 @@ class GLDefsParser
 			int firstUserTexture;
 			if (tex->Normal && tex->Specular)
 			{
-				usershader.shaderType = tex->Brightmap ? SHADER_SpecularBrightmap : SHADER_Specular;
+				usershader.shaderType = SHADER_Specular;
-				firstUserTexture = tex->Brightmap ? 5 : 4;
+				firstUserTexture = 7;
 			}
 			else if (tex->Normal && tex->Metallic && tex->Roughness && tex->AmbientOcclusion)
 			{
-				usershader.shaderType = tex->Brightmap ? SHADER_PBRBrightmap : SHADER_PBR;
+				usershader.shaderType = SHADER_PBR;
-				firstUserTexture = tex->Brightmap ? 7 : 6;
+				firstUserTexture = 9;
 			}
 			else
 			{
-				usershader.shaderType = tex->Brightmap ? SHADER_Brightmap : SHADER_Default;
+				usershader.shaderType = SHADER_Default;
-				firstUserTexture = tex->Brightmap ? 3 : 2;
+				firstUserTexture = 5;
 			}
 
 			for (unsigned int i = 0; i < texNameList.Size(); i++)
@@ -1537,14 +1537,16 @@ class GLDefsParser
 				else if (sc.Compare("material"))
 				{
 					sc.MustGetString();
-					MaterialShaderIndex typeIndex[6] = { SHADER_Default, SHADER_Brightmap, SHADER_Specular, SHADER_SpecularBrightmap, SHADER_PBR, SHADER_PBRBrightmap };
+					static MaterialShaderIndex typeIndex[6] = { SHADER_Default, SHADER_Default, SHADER_Specular, SHADER_Specular, SHADER_PBR, SHADER_PBR };
-					const char *typeName[6] = { "normal", "brightmap", "specular", "specularbrightmap", "pbr", "pbrbrightmap" };
+					static bool usesBrightmap[6] = { false, true, false, true, false, true };
+					static const char *typeName[6] = { "normal", "brightmap", "specular", "specularbrightmap", "pbr", "pbrbrightmap" };
 					bool found = false;
 					for (int i = 0; i < 6; i++)
 					{
 						if (sc.Compare(typeName[i]))
 						{
 							desc.shaderType = typeIndex[i];
+							if (usesBrightmap[i]) desc.shaderFlags |= SFlag_Brightmap;
 							found = true;
 							break;
 						}
@@ -1617,12 +1619,9 @@ class GLDefsParser
 			switch (desc.shaderType)
 			{
 			default:
-			case SHADER_Default: firstUserTexture = 2; break;
+			case SHADER_Default: firstUserTexture = 5; break;
-			case SHADER_Brightmap: firstUserTexture = 3; break;
+			case SHADER_Specular: firstUserTexture = 7; break;
-			case SHADER_Specular: firstUserTexture = 4; break;
+			case SHADER_PBR: firstUserTexture = 9; break;
-			case SHADER_SpecularBrightmap: firstUserTexture = 5; break;
-			case SHADER_PBR: firstUserTexture = 6; break;
-			case SHADER_PBRBrightmap: firstUserTexture = 7; break;
 			}
 
 			for (unsigned int i = 0; i < texNameList.Size(); i++)

src/rendering/gl/renderer/gl_renderstate.cpp

@@ -126,7 +126,10 @@ bool FGLRenderState::ApplyShader()
 
 	activeShader->muDesaturation.Set(mStreamData.uDesaturationFactor);
 	activeShader->muFogEnabled.Set(fogset);
-	activeShader->muTextureMode.Set(mTextureMode == TM_NORMAL && mTempTM == TM_OPAQUE ? TM_OPAQUE : mTextureMode);
+
+	int f = mTextureModeFlags;
+	if (!mBrightmapEnabled) f &= TEXF_Detailmap;
+	activeShader->muTextureMode.Set((mTextureMode == TM_NORMAL && mTempTM == TM_OPAQUE ? TM_OPAQUE : mTextureMode) | f);
 	activeShader->muLightParms.Set(mLightParms);
 	activeShader->muFogColor.Set(mStreamData.uFogColor);
 	activeShader->muObjectColor.Set(mStreamData.uObjectColor);
@@ -141,6 +144,7 @@ bool FGLRenderState::ApplyShader()
 	activeShader->muTextureAddColor.Set(mStreamData.uTextureAddColor);
 	activeShader->muTextureModulateColor.Set(mStreamData.uTextureModulateColor);
 	activeShader->muTextureBlendColor.Set(mStreamData.uTextureBlendColor);
+	activeShader->muDetailParms.Set(&mStreamData.uDetailParms.X);
 
 	if (mGlowEnabled || activeShader->currentglowstate)
 	{
@@ -166,6 +170,7 @@ bool FGLRenderState::ApplyShader()
 		activeShader->currentsplitstate = mSplitEnabled;
 	}
 
+
 	if (mTextureMatrixEnabled)
 	{
 		matrixToGL(mTextureMatrix, activeShader->texturematrix_index);

src/rendering/gl/shaders/gl_shader.cpp

@@ -262,6 +262,8 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 	i_data += "uniform vec4 uSplitTopPlane;\n";
 	i_data += "uniform vec4 uSplitBottomPlane;\n";
 
+	i_data += "uniform vec4 uDetailParms;\n";
+
 	// Lighting + Fog
 	i_data += "uniform vec4 uLightAttr;\n";
 	i_data += "#define uLightLevel uLightAttr.a\n";
@@ -302,6 +304,8 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 	i_data += "uniform sampler2D texture4;\n";
 	i_data += "uniform sampler2D texture5;\n";
 	i_data += "uniform sampler2D texture6;\n";
+	i_data += "uniform sampler2D texture7;\n";
+	i_data += "uniform sampler2D texture8;\n";
 
 	// timer data
 	i_data += "uniform float timer;\n";
@@ -311,14 +315,20 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 	i_data += "#define normaltexture texture2\n";
 	i_data += "#define speculartexture texture3\n";
 	i_data += "#define brighttexture texture4\n";
+	i_data += "#define detailtexture texture5\n";
+	i_data += "#define glowtexture texture6\n";
 	i_data += "#elif defined(PBR)\n";
 	i_data += "#define normaltexture texture2\n";
 	i_data += "#define metallictexture texture3\n";
 	i_data += "#define roughnesstexture texture4\n";
 	i_data += "#define aotexture texture5\n";
 	i_data += "#define brighttexture texture6\n";
+	i_data += "#define detailtexture texture7\n";
+	i_data += "#define glowtexture texture8\n";
 	i_data += "#else\n";
 	i_data += "#define brighttexture texture2\n";
+	i_data += "#define detailtexture texture3\n";
+	i_data += "#define glowtexture texture4\n";
 	i_data += "#endif\n";
 
 #ifdef __APPLE__
@@ -387,22 +397,31 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 			if (pp_lump == -1) I_Error("Unable to load '%s'", proc_prog_lump);
 			FileData pp_data = fileSystem.ReadFile(pp_lump);
 
-			if (pp_data.GetString().IndexOf("ProcessMaterial") < 0)
+			if (pp_data.GetString().IndexOf("ProcessMaterial") < 0 && pp_data.GetString().IndexOf("SetupMaterial") < 0)
 			{
 				// this looks like an old custom hardware shader.
 
-				// add ProcessMaterial function that calls the older ProcessTexel function
+				if (pp_data.GetString().IndexOf("GetTexCoord") >= 0)
-				int pl_lump = fileSystem.CheckNumForFullName("shaders/glsl/func_defaultmat.fp", 0);
-				if (pl_lump == -1) I_Error("Unable to load '%s'", "shaders/glsl/func_defaultmat.fp");
-				FileData pl_data = fileSystem.ReadFile(pl_lump);
-				fp_comb << "\n" << pl_data.GetString().GetChars();
-
-				if (pp_data.GetString().IndexOf("ProcessTexel") < 0)
 				{
-					// this looks like an even older custom hardware shader.
+					int pl_lump = fileSystem.CheckNumForFullName("shaders/glsl/func_defaultmat2.fp", 0);
-					// We need to replace the ProcessTexel call to make it work.
+					if (pl_lump == -1) I_Error("Unable to load '%s'", "shaders/glsl/func_defaultmat2.fp");
+					FileData pl_data = fileSystem.ReadFile(pl_lump);
+					fp_comb << "\n" << pl_data.GetString().GetChars();
+				}
+				else
+				{
+					int pl_lump = fileSystem.CheckNumForFullName("shaders/glsl/func_defaultmat.fp", 0);
+					if (pl_lump == -1) I_Error("Unable to load '%s'", "shaders/glsl/func_defaultmat.fp");
+					FileData pl_data = fileSystem.ReadFile(pl_lump);
+					fp_comb << "\n" << pl_data.GetString().GetChars();
 
-					fp_comb.Substitute("material.Base = ProcessTexel();", "material.Base = Process(vec4(1.0));");
+					if (pp_data.GetString().IndexOf("ProcessTexel") < 0)
+					{
+						// this looks like an even older custom hardware shader.
+						// We need to replace the ProcessTexel call to make it work.
+
+						fp_comb.Substitute("material.Base = ProcessTexel();", "material.Base = Process(vec4(1.0));");
+					}
 				}
 
 				if (pp_data.GetString().IndexOf("ProcessLight") >= 0)
@@ -423,6 +442,14 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 				FileData pl_data = fileSystem.ReadFile(pl_lump);
 				fp_comb << "\n" << pl_data.GetString().GetChars();
 			}
+
+			// ProcessMaterial must be considered broken because it requires the user to fill in data they possibly cannot know all about.
+			if (pp_data.GetString().IndexOf("ProcessMaterial") >= 0 && pp_data.GetString().IndexOf("SetupMaterial") < 0)
+			{
+				// This reactivates the old logic and disables all features that cannot be supported with that method.
+				fp_comb.Insert(0, "#define LEGACY_USER_SHADER\n");
+			}
+
 		}
 		else
 		{
@@ -546,6 +573,7 @@ bool FShader::Load(const char * name, const char * vert_prog_lump, const char *
 	muGradientTopPlane.Init(hShader, "uGradientTopPlane");
 	muSplitBottomPlane.Init(hShader, "uSplitBottomPlane");
 	muSplitTopPlane.Init(hShader, "uSplitTopPlane");
+	muDetailParms.Init(hShader, "uDetailParms");
 	muInterpolationFactor.Init(hShader, "uInterpolationFactor");
 	muAlphaThreshold.Init(hShader, "uAlphaThreshold");
 	muSpecularMaterial.Init(hShader, "uSpecularMaterial");

src/rendering/gl/shaders/gl_shader.h

@@ -257,6 +257,7 @@ class FShader
 	FUniform4f muGradientTopPlane;
 	FUniform4f muSplitBottomPlane;
 	FUniform4f muSplitTopPlane;
+	FUniform4f muDetailParms;
 	FBufferedUniform1f muInterpolationFactor;
 	FBufferedUniform1f muAlphaThreshold;
 	FBufferedUniform2f muSpecularMaterial;
@@ -331,8 +332,8 @@ public:
 
 	FShader *Get(unsigned int eff, bool alphateston)
 	{
-		// indices 0-2 match the warping modes, 3 is brightmap, 4 no texture, the following are custom
+		// indices 0-2 match the warping modes, 3 no texture, the following are custom
-		if (!alphateston && eff <= 3)
+		if (!alphateston && eff <= 2)
 		{
 			return mMaterialShadersNAT[eff];	// Non-alphatest shaders are only created for default, warp1+2 and brightmap. The rest won't get used anyway
 		}

src/rendering/hwrenderer/scene/hw_renderstate.h

@@ -199,6 +199,8 @@ struct StreamData
 
 	FVector4 uSplitTopPlane;
 	FVector4 uSplitBottomPlane;
+
+	FVector4 uDetailParms;
 };
 
 class FRenderState
@@ -208,14 +210,15 @@ protected:
 	uint8_t mTextureEnabled:1;
 	uint8_t mGlowEnabled : 1;
 	uint8_t mGradientEnabled : 1;
-	uint8_t mBrightmapEnabled : 1;
 	uint8_t mModelMatrixEnabled : 1;
 	uint8_t mTextureMatrixEnabled : 1;
 	uint8_t mSplitEnabled : 1;
+	uint8_t mBrightmapEnabled : 1;
 
 	int mLightIndex;
 	int mSpecialEffect;
 	int mTextureMode;
+	int mTextureModeFlags;
 	int mSoftLight;
 	float mLightParms[4];
 
@@ -247,10 +250,11 @@ public:
 	void Reset()
 	{
 		mTextureEnabled = true;
-		mGradientEnabled = mBrightmapEnabled = mFogEnabled = mGlowEnabled = false;
+		mBrightmapEnabled = mGradientEnabled = mFogEnabled = mGlowEnabled = false;
 		mFogColor = 0xffffffff;
 		mStreamData.uFogColor = mFogColor;
 		mTextureMode = -1;
+		mTextureModeFlags = 0;
 		mStreamData.uDesaturationFactor = 0.0f;
 		mAlphaThreshold = 0.5f;
 		mModelMatrixEnabled = false;
@@ -287,6 +291,7 @@ public:
 		mStreamData.uSplitTopPlane = { 0.0f, 0.0f, 0.0f, 0.0f };
 		mStreamData.uSplitBottomPlane = { 0.0f, 0.0f, 0.0f, 0.0f };
 		mStreamData.uDynLightColor = { 0.0f, 0.0f, 0.0f, 0.0f };
+		mStreamData.uDetailParms = { 0.0f, 0.0f, 0.0f, 0.0f };
 
 		mModelMatrix.loadIdentity();
 		mTextureMatrix.loadIdentity();
@@ -338,15 +343,15 @@ public:
 	{
 		if (style.Flags & STYLEF_RedIsAlpha)
 		{
-			mTextureMode = TM_ALPHATEXTURE;
+			SetTextureMode(TM_ALPHATEXTURE);
 		}
 		else if (style.Flags & STYLEF_ColorIsFixed)
 		{
-			mTextureMode = TM_STENCIL;
+			SetTextureMode(TM_STENCIL);
 		}
 		else if (style.Flags & STYLEF_InvertSource)
 		{
-			mTextureMode = TM_INVERSE;
+			SetTextureMode(TM_INVERSE);
 		}
 	}
 
@@ -451,6 +456,11 @@ public:
 		mStreamData.uSplitBottomPlane = { (float)bn.X, (float)bn.Y, (float)bottom.negiC, (float)bottom.fD() };
 	}
 
+	void SetDetailParms(float xscale, float yscale, float bias)
+	{
+		mStreamData.uDetailParms = { xscale, yscale, bias, 0 };
+	}
+
 	void SetDynLight(float r, float g, float b)
 	{
 		mStreamData.uDynLightColor = { r, g, b, 0.0f };
@@ -556,6 +566,7 @@ public:
 		mMaterial.mTranslation = translation;
 		mMaterial.mOverrideShader = overrideshader;
 		mMaterial.mChanged = true;
+		mTextureModeFlags = mat->GetLayerFlags();
 	}
 
 	void SetClipSplit(float bottom, float top)

src/rendering/hwrenderer/utility/hw_shaderpatcher.cpp

@@ -268,11 +268,8 @@ const FDefaultShader defaultshaders[] =
 	{"Default",	"shaders/glsl/func_normal.fp", "shaders/glsl/material_normal.fp", ""},
 	{"Warp 1",	"shaders/glsl/func_warp1.fp", "shaders/glsl/material_normal.fp", ""},
 	{"Warp 2",	"shaders/glsl/func_warp2.fp", "shaders/glsl/material_normal.fp", ""},
-	{"Brightmap","shaders/glsl/func_brightmap.fp", "shaders/glsl/material_normal.fp", "#define BRIGHTMAP\n"},
 	{"Specular", "shaders/glsl/func_spec.fp", "shaders/glsl/material_specular.fp", "#define SPECULAR\n#define NORMALMAP\n"},
-	{"SpecularBrightmap", "shaders/glsl/func_spec.fp", "shaders/glsl/material_specular.fp", "#define SPECULAR\n#define NORMALMAP\n#define BRIGHTMAP\n"},
 	{"PBR","shaders/glsl/func_pbr.fp", "shaders/glsl/material_pbr.fp", "#define PBR\n#define NORMALMAP\n"},
-	{"PBRBrightmap","shaders/glsl/func_pbr.fp", "shaders/glsl/material_pbr.fp", "#define PBR\n#define NORMALMAP\n#define BRIGHTMAP\n"},
 	{"Paletted",	"shaders/glsl/func_paletted.fp", "shaders/glsl/material_nolight.fp", ""},
 	{"No Texture", "shaders/glsl/func_notexture.fp", "shaders/glsl/material_normal.fp", ""},
 	{"Basic Fuzz", "shaders/glsl/fuzz_standard.fp", "shaders/glsl/material_normal.fp", ""},

src/rendering/polyrenderer/backend/poly_renderstate.cpp

@@ -287,7 +287,7 @@ void PolyRenderState::Apply()
 
 	PolyPushConstants constants;
 	constants.uFogEnabled = fogset;
-	constants.uTextureMode = mTextureMode == TM_NORMAL && mTempTM == TM_OPAQUE ? TM_OPAQUE : mTextureMode;
+	constants.uTextureMode = (mTextureMode == TM_NORMAL && mTempTM == TM_OPAQUE ? TM_OPAQUE : mTextureMode);
 	constants.uLightDist = mLightParms[0];
 	constants.uLightFactor = mLightParms[1];
 	constants.uFogDensity = mLightParms[2];

src/rendering/vulkan/renderer/vk_renderstate.cpp

@@ -372,7 +372,9 @@ void VkRenderState::ApplyPushConstants()
 		tempTM = TM_OPAQUE;
 
 	mPushConstants.uFogEnabled = fogset;
-	mPushConstants.uTextureMode = mTextureMode == TM_NORMAL && tempTM == TM_OPAQUE ? TM_OPAQUE : mTextureMode;
+	int f = mTextureModeFlags;
+	if (!mBrightmapEnabled) f &= TEXF_Detailmap;
+	mPushConstants.uTextureMode = (mTextureMode == TM_NORMAL && tempTM == TM_OPAQUE ? TM_OPAQUE : mTextureMode) | f;
 	mPushConstants.uLightDist = mLightParms[0];
 	mPushConstants.uLightFactor = mLightParms[1];
 	mPushConstants.uFogDensity = mLightParms[2];

src/rendering/vulkan/shaders/vk_shader.cpp

@@ -90,10 +90,10 @@ VkShaderProgram *VkShaderManager::GetEffect(int effect, EPassType passType)
 
 VkShaderProgram *VkShaderManager::Get(unsigned int eff, bool alphateston, EPassType passType)
 {
-	// indices 0-2 match the warping modes, 3 is brightmap, 4 no texture, the following are custom
+	// indices 0-2 match the warping modes, 3 no texture, the following are custom
-	if (!alphateston && eff <= 3)
+	if (!alphateston && eff <= 2)
 	{
-		return &mMaterialShadersNAT[passType][eff];	// Non-alphatest shaders are only created for default, warp1+2 and brightmap. The rest won't get used anyway
+		return &mMaterialShadersNAT[passType][eff];	// Non-alphatest shaders are only created for default, warp1+2. The rest won't get used anyway
 	}
 	else if (eff < (unsigned int)mMaterialShaders[passType].size())
 	{
@@ -160,6 +160,8 @@ static const char *shaderBindings = R"(
 
 		vec4 uSplitTopPlane;
 		vec4 uSplitBottomPlane;
+
+		vec4 uDetailParms;
 	};
 
 	layout(set = 0, binding = 3, std140) uniform StreamUBO {
@@ -175,6 +177,8 @@ static const char *shaderBindings = R"(
 	layout(set = 1, binding = 3) uniform sampler2D texture4;
 	layout(set = 1, binding = 4) uniform sampler2D texture5;
 	layout(set = 1, binding = 5) uniform sampler2D texture6;
+	layout(set = 1, binding = 6) uniform sampler2D texture7;
+	layout(set = 1, binding = 7) uniform sampler2D texture8;
 
 	// This must match the PushConstants struct
 	layout(push_constant) uniform PushConstants
@@ -205,14 +209,20 @@ static const char *shaderBindings = R"(
 	#define normaltexture texture2
 	#define speculartexture texture3
 	#define brighttexture texture4
+	#define detailtexture texture5
+	#define glowtexture texture6
 	#elif defined(PBR)
 	#define normaltexture texture2
 	#define metallictexture texture3
 	#define roughnesstexture texture4
 	#define aotexture texture5
 	#define brighttexture texture6
+	#define detailtexture texture7
+	#define glowtexture texture8
 	#else
 	#define brighttexture texture2
+	#define detailtexture texture3
+	#define glowtexture texture4
 	#endif
 
 	#define uObjectColor data[uDataIndex].uObjectColor
@@ -237,6 +247,7 @@ static const char *shaderBindings = R"(
 	#define uGradientBottomPlane data[uDataIndex].uGradientBottomPlane
 	#define uSplitTopPlane data[uDataIndex].uSplitTopPlane
 	#define uSplitBottomPlane data[uDataIndex].uSplitBottomPlane
+	#define uDetailParms data[uDataIndex].uDetailParms
 
 	#define SUPPORTS_SHADOWMAPS
 	#define VULKAN_COORDINATE_SYSTEM
@@ -289,18 +300,25 @@ std::unique_ptr<VulkanShader> VkShaderManager::LoadFragShader(FString shadername
 		{
 			FString pp_code = LoadPublicShaderLump(material_lump);
 
-			if (pp_code.IndexOf("ProcessMaterial") < 0)
+			if (pp_code.IndexOf("ProcessMaterial") < 0 && pp_code.IndexOf("SetupMaterial") < 0)
 			{
 				// this looks like an old custom hardware shader.
 				// add ProcessMaterial function that calls the older ProcessTexel function
-				code << "\n" << LoadPrivateShaderLump("shaders/glsl/func_defaultmat.fp").GetChars() << "\n";
 
-				if (pp_code.IndexOf("ProcessTexel") < 0)
+				if (pp_code.IndexOf("GetTexCoord") >= 0)
 				{
-					// this looks like an even older custom hardware shader.
+					code << "\n" << LoadPrivateShaderLump("shaders/glsl/func_defaultmat2.fp").GetChars() << "\n";
-					// We need to replace the ProcessTexel call to make it work.
+				}
+				else
+				{
+					code << "\n" << LoadPrivateShaderLump("shaders/glsl/func_defaultmat.fp").GetChars() << "\n";
+					if (pp_code.IndexOf("ProcessTexel") < 0)
+					{
+						// this looks like an even older custom hardware shader.
+						// We need to replace the ProcessTexel call to make it work.
 
-					code.Substitute("material.Base = ProcessTexel();", "material.Base = Process(vec4(1.0));");
+						code.Substitute("material.Base = ProcessTexel();", "material.Base = Process(vec4(1.0));");
+					}
 				}
 
 				if (pp_code.IndexOf("ProcessLight") >= 0)
@@ -319,6 +337,13 @@ std::unique_ptr<VulkanShader> VkShaderManager::LoadFragShader(FString shadername
 			{
 				code << "\n" << LoadPrivateShaderLump("shaders/glsl/func_defaultlight.fp").GetChars() << "\n";
 			}
+
+			// ProcessMaterial must be considered broken because it requires the user to fill in data they possibly cannot know all about.
+			if (pp_code.IndexOf("ProcessMaterial") >= 0 && pp_code.IndexOf("SetupMaterial") < 0)
+			{
+				// This reactivates the old logic and disables all features that cannot be supported with that method.
+				code.Insert(0, "#define LEGACY_USER_SHADER\n");
+			}
 		}
 		else
 		{

wadsrc/static/shaders/glsl/func_brightmap.fp

@@ -1,9 +0,0 @@
-
-Material ProcessMaterial()
-{
-	Material material;
-	material.Base = getTexel(vTexCoord.st);
-	material.Normal = ApplyNormalMap(vTexCoord.st);
-	material.Bright = texture(brighttexture, vTexCoord.st);
-	return material;
-}

wadsrc/static/shaders/glsl/func_defaultlight.fp

@@ -1,17 +1,5 @@
 
-#if defined(BRIGHTMAP)
-
-vec4 ProcessLight(Material material, vec4 color)
-{
-	vec4 brightpix = desaturate(material.Bright);
-	return vec4(min(color.rgb + brightpix.rgb, 1.0), color.a);
-}
-
-#else
-
 vec4 ProcessLight(Material material, vec4 color)
 {
 	return color;
 }
-
-#endif

wadsrc/static/shaders/glsl/func_defaultmat.fp

@@ -1,11 +1,7 @@
 
-Material ProcessMaterial()
+void SetupMaterial(inout Material material)
 {
-	Material material;
 	material.Base = ProcessTexel();
 	material.Normal = ApplyNormalMap(vTexCoord.st);
-#if defined(BRIGHTMAP)
 	material.Bright = texture(brighttexture, vTexCoord.st);
-#endif
-	return material;
 }

wadsrc/static/shaders/glsl/func_defaultmat2.fp

@@ -0,0 +1,6 @@
+
+void SetupMaterial(inout Material material)
+{
+	vec2 texCoord = GetTexCoord();
+	SetMaterialProps(material, texCoord);
+}

wadsrc/static/shaders/glsl/func_normal.fp

@@ -1,8 +1,5 @@
 
-Material ProcessMaterial()
+void SetupMaterial(inout Material material)
 {
-	Material material;
+	SetMaterialProps(material, vTexCoord.st);
-	material.Base = getTexel(vTexCoord.st);
-	material.Normal = ApplyNormalMap(vTexCoord.st);
-	return material;
 }

wadsrc/static/shaders/glsl/func_pbr.fp

@@ -1,14 +1,8 @@
 
-Material ProcessMaterial()
+void SetupMaterial(inout Material material)
 {
-	Material material;
+	SetMaterialProps(material, vTexCoord.st);
-	material.Base = getTexel(vTexCoord.st);
-	material.Normal = ApplyNormalMap(vTexCoord.st);
 	material.Metallic = texture(metallictexture, vTexCoord.st).r;
 	material.Roughness = texture(roughnesstexture, vTexCoord.st).r;
 	material.AO = texture(aotexture, vTexCoord.st).r;
-#if defined(BRIGHTMAP)
-	material.Bright = texture(brighttexture, vTexCoord.st);
-#endif
-	return material;
 }

wadsrc/static/shaders/glsl/func_spec.fp

@@ -1,14 +1,8 @@
 
-Material ProcessMaterial()
+void SetupMaterial(inout Material material)
 {
-	Material material;
+	SetMaterialProps(material, vTexCoord.st);
-	material.Base = getTexel(vTexCoord.st);
-	material.Normal = ApplyNormalMap(vTexCoord.st);
 	material.Specular = texture(speculartexture, vTexCoord.st).rgb;
 	material.Glossiness = uSpecularMaterial.x;
 	material.SpecularLevel = uSpecularMaterial.y;
-#if defined(BRIGHTMAP)
-	material.Bright = texture(brighttexture, vTexCoord.st);
-#endif
-	return material;
 }

wadsrc/static/shaders/glsl/func_warp1.fp

@@ -1,5 +1,5 @@
 
-vec4 ProcessTexel()
+vec2 GetTexCoord()
 {
 	vec2 texCoord = vTexCoord.st;
 
@@ -9,8 +9,11 @@ vec4 ProcessTexel()
 	offset.y = sin(pi * 2.0 * (texCoord.x + timer * 0.125)) * 0.1;
 	offset.x = sin(pi * 2.0 * (texCoord.y + timer * 0.125)) * 0.1;
 
-	texCoord += offset;
+	return texCoord + offset;
-
+}
-	return getTexel(texCoord);
+
+vec4 ProcessTexel()
+{
+	return getTexel(GetTexCoord());
 }
 

wadsrc/static/shaders/glsl/func_warp2.fp

@@ -1,5 +1,6 @@
 
-vec4 ProcessTexel()
+
+vec2 GetTexCoord()
 {
 	vec2 texCoord = vTexCoord.st;
 
@@ -9,8 +10,11 @@ vec4 ProcessTexel()
 	offset.y = 0.5 + sin(pi * 2.0 * (texCoord.y + timer * 0.61 + 900.0/8192.0)) + sin(pi * 2.0 * (texCoord.x * 2.0 + timer * 0.36 + 300.0/8192.0));
 	offset.x = 0.5 + sin(pi * 2.0 * (texCoord.y + timer * 0.49 + 700.0/8192.0)) + sin(pi * 2.0 * (texCoord.x * 2.0 + timer * 0.49 + 1200.0/8192.0));
 
-	texCoord += offset * 0.025;
+	return texCoord + offset * 0.025;
-
+}
-	return getTexel(texCoord);
+
+vec4 ProcessTexel()
+{
+	return getTexel(GetTexCoord());
 }
 

wadsrc/static/shaders/glsl/func_warp3.fp

@@ -1,5 +1,6 @@
 
-vec4 ProcessTexel()
+
+vec2 GetTexCoord()
 {
 	vec2 texCoord = vTexCoord.st;
 
@@ -10,8 +11,11 @@ vec4 ProcessTexel()
 	offset.y = siny + sin(pi * 2.0 * (texCoord.x + timer * 0.75)) * 0.03;
 	offset.x = siny + sin(pi * 2.0 * (texCoord.x + timer * 0.45)) * 0.02;
 
-	texCoord += offset;
+	return texCoord + offset;
-
+}
-	return getTexel(texCoord);
+
+vec4 ProcessTexel()
+{
+	return getTexel(GetTexCoord());
 }
 

wadsrc/static/shaders/glsl/func_wavex.fp

@@ -1,5 +1,5 @@
 
-vec4 ProcessTexel()
+vec2 GetTexCoord()
 {
 	vec2 texCoord = vTexCoord.st;
 
@@ -7,6 +7,11 @@ vec4 ProcessTexel()
 
 	texCoord.x += sin(pi * 2.0 * (texCoord.y + timer * 0.125)) * 0.1;
 
-	return getTexel(texCoord);
+	return texCoord;
+}
+
+vec4 ProcessTexel()
+{
+	return getTexel(GetTexCoord());
 }
 

wadsrc/static/shaders/glsl/main.fp

@@ -22,6 +22,7 @@ struct Material
 {
 	vec4 Base;
 	vec4 Bright;
+	vec4 Glow;
 	vec3 Normal;
 	vec3 Specular;
 	float Glossiness;
@@ -33,9 +34,16 @@ struct Material
 
 vec4 Process(vec4 color);
 vec4 ProcessTexel();
-Material ProcessMaterial();
+Material ProcessMaterial(); // note that this is deprecated. Use SetupMaterial!
+void SetupMaterial(inout Material mat);
 vec4 ProcessLight(Material mat, vec4 color);
 vec3 ProcessMaterialLight(Material material, vec3 color);
+vec2 GetTexCoord();
+
+// These get Or'ed into uTextureMode because it only uses its 3 lowermost bits.
+const int TEXF_Brightmap = 0x10000;
+const int TEXF_Detailmap = 0x20000;
+const int TEXF_Glowmap = 0x40000;
 
 //===========================================================================
 //
@@ -157,7 +165,7 @@ vec4 getTexel(vec2 st)
 	//
 	// Apply texture modes
 	//
-	switch (uTextureMode)
+	switch (uTextureMode & 0xffff)
 	{
 		case 1:	// TM_STENCIL
 			texel.rgb = vec3(1.0,1.0,1.0);
@@ -524,6 +532,30 @@ vec3 ApplyNormalMap(vec2 texcoord)
 }
 #endif
 
+//===========================================================================
+//
+// Sets the common material properties.
+//
+//===========================================================================
+
+void SetMaterialProps(inout Material material, vec2 texCoord)
+{
+	material.Base = getTexel(texCoord.st); 
+	material.Normal = ApplyNormalMap(texCoord.st);
+
+	if ((uTextureMode & TEXF_Brightmap) != 0)
+		material.Bright = texture(brighttexture, texCoord.st);
+		
+	if ((uTextureMode & TEXF_Detailmap) != 0)
+	{
+		vec4 Detail = texture(detailtexture, texCoord.st * uDetailParms.xy) * uDetailParms.z;
+		material.Base *= Detail;
+	}
+	
+	if ((uTextureMode & TEXF_Glowmap) != 0)
+		material.Glow = texture(glowtexture, texCoord.st);
+}
+
 //===========================================================================
 //
 // Calculate light
@@ -574,8 +606,21 @@ vec4 getLightColor(Material material, float fogdist, float fogfactor)
 	}
 	color = min(color, 1.0);
 
+	// these cannot be safely applied by the legacy format where the implementation cannot guarantee that the values are set.
+#ifndef LEGACY_USER_SHADER
 	//
-	// apply brightmaps (or other light manipulation by custom shaders.
+	// apply glow 
+	//
+	color.rgb = mix(color.rgb, material.Glow.rgb, material.Glow.a);
+
+	//
+	// apply brightmaps 
+	//
+	color.rgb = min(color.rgb + material.Bright.rgb, 1.0);
+#endif
+	
+	//
+	// apply other light manipulation by custom shaders, default is a NOP.
 	//
 	color = ProcessLight(material, color);
 
@@ -635,7 +680,23 @@ void main()
 	if (ClipDistanceA.x < 0 || ClipDistanceA.y < 0 || ClipDistanceA.z < 0 || ClipDistanceA.w < 0 || ClipDistanceB.x < 0) discard;
 #endif
 
+#ifndef LEGACY_USER_SHADER
+	Material material;
+	
+	material.Base = vec4(0.0);
+	material.Bright = vec4(0.0);
+	material.Glow = vec4(0.0);
+	material.Normal = vec3(0.0);
+	material.Specular = vec3(0.0);
+	material.Glossiness = 0.0;
+	material.SpecularLevel = 0.0;
+	material.Metallic = 0.0;
+	material.Roughness = 0.0;
+	material.AO = 0.0;
+	SetupMaterial(material);
+#else
 	Material material = ProcessMaterial();
+#endif
 	vec4 frag = material.Base;
 	
 #ifndef NO_ALPHATEST
@@ -663,9 +724,10 @@ void main()
 			fogfactor = exp2 (uFogDensity * fogdist);
 		}
 		
-		if (uTextureMode != 7)
+		if ((uTextureMode & 0xffff) != 7)
 		{
 			frag = getLightColor(material, fogdist, fogfactor);
+
 			//
 			// colored fog
 			//
@@ -681,7 +743,7 @@ void main()
 	}
 	else // simple 2D (uses the fog color to add a color overlay)
 	{
-		if (uTextureMode == 7)
+		if ((uTextureMode & 0xffff) == 7)
 		{
 			float gray = grayscale(frag);
 			vec4 cm = (uObjectColor + gray * (uAddColor - uObjectColor)) * 2;