- Added auto-detection scheme for r_vanillatrans

It now works the following way:
(0) - Force off (ZDoom defaults)
(1) - Force on (Doom defaults)
(2) - Auto off (Prefer ZDoom defaults - if DEHACKED is detected with no ZSCRIPT it will turn on) (default)
(3) - Auto on (Prefer Doom defaults - if DECORATE is detected with no ZSCRIPT it will turn off)

src/CMakeLists.txt

@@ -1071,6 +1071,7 @@ set (PCH_SOURCES
 	r_data/voxels.cpp
 	r_data/renderstyle.cpp
 	r_data/r_interpolate.cpp
+	r_data/r_vanillatrans.cpp
 	scripting/symbols.cpp
 	scripting/types.cpp
 	scripting/thingdef.cpp

src/d_main.cpp

@@ -116,6 +116,7 @@
 #include "r_utility.h"
 #include "vm.h"
 #include "types.h"
+#include "r_data/r_vanillatrans.h"
 
 EXTERN_CVAR(Bool, hud_althud)
 void DrawHUD();
@@ -2606,6 +2607,9 @@ void D_DoomMain (void)
 			D_CheckNetGame ();
 		}
 
+		// [SP] Force vanilla transparency auto-detection to re-detect our game lumps now
+		UpdateVanillaTransparency();
+
 		// [RH] Lock any cvars that should be locked now that we're
 		// about to begin the game.
 		FBaseCVar::EnableNoSet ();

src/gl/scene/gl_sprite.cpp

@@ -39,6 +39,7 @@
 #include "g_levellocals.h"
 #include "events.h"
 #include "actorinlines.h"
+#include "r_data/r_vanillatrans.h"
 
 #include "gl/system/gl_interface.h"
 #include "gl/system/gl_framebuffer.h"
@@ -75,7 +76,6 @@ CUSTOM_CVAR(Int, gl_fuzztype, 0, CVAR_ARCHIVE)
 }
 
 EXTERN_CVAR (Float, transsouls)
-EXTERN_CVAR (Bool, r_vanillatrans)
 
 extern TArray<spritedef_t> sprites;
 extern TArray<spriteframe_t> SpriteFrames;
@@ -993,7 +993,7 @@ void GLSprite::Process(AActor* thing, sector_t * sector, int thruportal)
 	{
 		trans = 1.f;
 	}
-	if (r_vanillatrans)
+	if (UseVanillaTransparency())
 	{
 		// [SP] "canonical transparency" - with the flip of a CVar, disable transparency for Doom objects,
 		//   and disable 'additive' translucency for certain objects from other games.

src/polyrenderer/scene/poly_sprite.cpp

@@ -28,10 +28,10 @@
 #include "poly_sprite.h"
 #include "polyrenderer/poly_renderer.h"
 #include "polyrenderer/scene/poly_light.h"
+#include "r_data/r_vanillatrans.h"
 
 EXTERN_CVAR(Float, transsouls)
 EXTERN_CVAR(Int, r_drawfuzz)
-EXTERN_CVAR (Bool, r_vanillatrans)
 
 bool RenderPolySprite::GetLine(AActor *thing, DVector2 &left, DVector2 &right)
 {
@@ -146,7 +146,7 @@ void RenderPolySprite::Render(const TriMatrix &worldToClip, const PolyClipPlane
 	args.SetStencilTestValue(stencilValue);
 	args.SetWriteStencil(true, stencilValue);
 	args.SetClipPlane(clipPlane);
-	if ((thing->renderflags & RF_ZDOOMTRANS) && r_vanillatrans)
+	if ((thing->renderflags & RF_ZDOOMTRANS) && UseVanillaTransparency())
 		args.SetStyle(LegacyRenderStyles[STYLE_Normal], 1.0f, thing->fillcolor, thing->Translation, tex, fullbrightSprite);
 	else
 		args.SetStyle(thing->RenderStyle, thing->Alpha, thing->fillcolor, thing->Translation, tex, fullbrightSprite);

src/r_data/r_vanillatrans.cpp

@@ -0,0 +1,67 @@
+
+#include "templates.h"
+#include "c_cvars.h"
+#include "w_wad.h"
+#ifdef _DEBUG
+#include "c_dispatch.h"
+#endif
+
+CVAR (Int, r_vanillatrans, 2, CVAR_ARCHIVE)
+
+namespace
+{
+	bool firstTime = true;
+	bool foundDehacked = false;
+	bool foundDecorate = false;
+	bool foundZScript = false;
+}
+#ifdef _DEBUG
+CCMD (debug_checklumps)
+{
+	Printf("firstTime: %d\n", firstTime);
+	Printf("foundDehacked: %d\n", foundDehacked);
+	Printf("foundDecorate: %d\n", foundDecorate);
+	Printf("foundZScript: %d\n", foundZScript);
+}
+#endif
+
+void UpdateVanillaTransparency()
+{
+	firstTime = true;
+}
+
+bool UseVanillaTransparency()
+{
+	if (firstTime)
+	{
+		int lastlump = 0;
+		Wads.FindLump("ZSCRIPT", &lastlump); // ignore first ZScript
+		if (Wads.FindLump("ZSCRIPT", &lastlump) == -1) // no loaded ZScript
+		{
+			lastlump = 0;
+			foundDehacked = Wads.FindLump("DEHACKED", &lastlump) != -1;
+			lastlump = 0;
+			foundDecorate = Wads.FindLump("DECORATE", &lastlump) != -1;
+			foundZScript = false;
+		}
+		else
+		{
+			foundZScript = true;
+			foundDehacked = false;
+			foundDecorate = false;
+		}
+		firstTime = false;
+	}
+
+	switch (r_vanillatrans)
+	{
+		case 0: return false;
+		case 1: return true;
+		default:
+		if (foundDehacked)
+			return true;
+		if (foundDecorate)
+			return false;
+		return r_vanillatrans == 3;
+	}
+}

src/r_data/r_vanillatrans.h

@@ -0,0 +1,3 @@
+
+void UpdateVanillaTransparency();
+bool UseVanillaTransparency();

src/r_data/renderstyle.cpp

@@ -38,7 +38,6 @@
 #include "serializer.h"
 
 CVAR (Bool, r_drawtrans, true, 0)
-CVAR (Bool, r_vanillatrans, false, CVAR_ARCHIVE)
 CVAR (Int, r_drawfuzz, 1, CVAR_ARCHIVE)
 
 // Convert legacy render styles to flexible render styles.

src/swrenderer/things/r_sprite.cpp

@@ -65,10 +65,10 @@
 #include "swrenderer/r_memory.h"
 #include "swrenderer/r_renderthread.h"
 #include "a_dynlight.h"
+#include "r_data/r_vanillatrans.h"
 
 EXTERN_CVAR(Bool, r_fullbrightignoresectorcolor)
 EXTERN_CVAR(Bool, gl_light_sprites)
-EXTERN_CVAR (Bool, r_vanillatrans)
 
 namespace swrenderer
 {
@@ -213,7 +213,7 @@ namespace swrenderer
 		if (thing->flags5 & MF5_BRIGHT)
 			vis->renderflags |= RF_FULLBRIGHT; // kg3D
 		vis->RenderStyle = thing->RenderStyle;
-		if (r_vanillatrans)
+		if (UseVanillaTransparency())
 		{
 			if (thing->renderflags & RF_ZDOOMTRANS)
 				vis->RenderStyle = LegacyRenderStyles[STYLE_Normal];

wadsrc/static/language.enu

@@ -2369,6 +2369,10 @@ OPTVAL_SWDOOM				= "Doom Software Renderer";
 OPTVAL_DEDICATED			= "High-Performance";
 OPTVAL_INTEGRATED			= "Power-Saving";
 OPTVAL_VANILLA				= "Vanilla";
+OPTVAL_VTFZDOOM				= "ZDoom (Forced)";
+OPTVAL_VTFVANILLA			= "Vanilla (Forced)";
+OPTVAL_VTAZDOOM				= "Auto (Vanilla Preferred)";
+OPTVAL_VTAVANILLA			= "Auto (ZDoom Preferred)";
 
 // Colors
 C_BRICK					= "\cabrick";

wadsrc/static/menudef.txt

@@ -692,8 +692,10 @@ OptionValue Fuzziness
 
 OptionValue VanillaTrans
 {
-	0.0, "$OPTVAL_ZDOOM"
-	1.0, "$OPTVAL_VANILLA"
+	0.0, "$OPTVAL_VTFZDOOM"
+	1.0, "$OPTVAL_VTFVANILLA"
+	0.0, "$OPTVAL_VTAZDOOM"
+	1.0, "$OPTVAL_VTAVANILLA"
 }
 
 OptionValue GPUSwitch