From ae0878ca618a737d7f8e843e8f4867bd34712cbe Mon Sep 17 00:00:00 2001
From: Zack Middleton <zack@cloemail.com>
Date: Wed, 20 Dec 2023 23:12:56 -0600
Subject: [PATCH] OpenGL2: Fix q3map2 lightstyles effects

Fixes World of Padman wop_trashmap.

For r_mergeLightmaps 1
- Fix tcMod transform on "map $lightmap" stages
- Fix external lightmap image texcoords if shader also has an internal
  lightmap

For r_sunlightMode 1
- Fix "tcGen lightmap" stages with blendFunc GL_SRC_ALPHA GL_ONE being
  converted to white image + modulate lightmap and drawing incorrectly
---
 code/renderergl2/tr_bsp.c    |   1 +
 code/renderergl2/tr_shader.c | 107 ++++++++++++++++++++++++++++++++---
 2 files changed, 99 insertions(+), 9 deletions(-)

diff --git a/code/renderergl2/tr_bsp.c b/code/renderergl2/tr_bsp.c
index ad5fe3a3..b316351e 100644
--- a/code/renderergl2/tr_bsp.c
+++ b/code/renderergl2/tr_bsp.c
@@ -497,6 +497,7 @@ static	void R_LoadLightmaps( lump_t *l, lump_t *surfs ) {
 }
 
 
+// If FatPackU() or FatPackV() changes, update FixFatLightmapTexCoords()
 static float FatPackU(float input, int lightmapnum)
 {
 	if (lightmapnum < 0)
diff --git a/code/renderergl2/tr_shader.c b/code/renderergl2/tr_shader.c
index 0cd78a78..c899363e 100644
--- a/code/renderergl2/tr_shader.c
+++ b/code/renderergl2/tr_shader.c
@@ -2580,13 +2580,15 @@ static int CollapseStagesToGLSL(void)
 		numStages++;
 	}
 
-	// convert any remaining lightmap stages to a lighting pass with a white texture
+	// convert any remaining lightmap stages with no blending or blendfunc filter
+	// to a lighting pass with a white texture
 	// only do this with r_sunlightMode non-zero, as it's only for correct shadows.
 	if (r_sunlightMode->integer && shader.numDeforms == 0)
 	{
 		for (i = 0; i < MAX_SHADER_STAGES; i++)
 		{
 			shaderStage_t *pStage = &stages[i];
+			int blendBits;
 
 			if (!pStage->active)
 				continue;
@@ -2594,15 +2596,23 @@ static int CollapseStagesToGLSL(void)
 			if (pStage->adjustColorsForFog)
 				continue;
 
-			if (pStage->bundle[TB_DIFFUSEMAP].tcGen == TCGEN_LIGHTMAP)
-			{
-				pStage->glslShaderGroup = tr.lightallShader;
-				pStage->glslShaderIndex = LIGHTDEF_USE_LIGHTMAP;
-				pStage->bundle[TB_LIGHTMAP] = pStage->bundle[TB_DIFFUSEMAP];
-				pStage->bundle[TB_DIFFUSEMAP].image[0] = tr.whiteImage;
-				pStage->bundle[TB_DIFFUSEMAP].isLightmap = qfalse;
-				pStage->bundle[TB_DIFFUSEMAP].tcGen = TCGEN_TEXTURE;
+			if (pStage->bundle[TB_DIFFUSEMAP].tcGen != TCGEN_LIGHTMAP)
+				continue;
+
+			blendBits = pStage->stateBits & (GLS_DSTBLEND_BITS | GLS_SRCBLEND_BITS);
+
+			if (blendBits != 0 &&
+				blendBits != (GLS_DSTBLEND_SRC_COLOR | GLS_SRCBLEND_ZERO) &&
+				blendBits != (GLS_DSTBLEND_ZERO | GLS_SRCBLEND_DST_COLOR)) {
+				continue;
 			}
+
+			pStage->glslShaderGroup = tr.lightallShader;
+			pStage->glslShaderIndex = LIGHTDEF_USE_LIGHTMAP;
+			pStage->bundle[TB_LIGHTMAP] = pStage->bundle[TB_DIFFUSEMAP];
+			pStage->bundle[TB_DIFFUSEMAP].image[0] = tr.whiteImage;
+			pStage->bundle[TB_DIFFUSEMAP].isLightmap = qfalse;
+			pStage->bundle[TB_DIFFUSEMAP].tcGen = TCGEN_TEXTURE;
 		}
 	}
 
@@ -2890,6 +2900,83 @@ static void VertexLightingCollapse( void ) {
 	}
 }
 
+/*
+=================
+FixFatLightmapTexCoords
+
+Handle edge cases of altering lightmap texcoords for fat lightmap atlas
+=================
+*/
+static void FixFatLightmapTexCoords(void)
+{
+	texModInfo_t *tmi;
+	int lightmapnum;
+	int stage;
+	int size;
+	int i;
+
+	if ( !r_mergeLightmaps->integer || tr.fatLightmapCols <= 0) {
+		return;
+	}
+
+	if ( shader.lightmapIndex < 0 ) {
+		// no internal lightmap, texcoords were not modified
+		return;
+	}
+
+	lightmapnum = shader.lightmapIndex;
+
+	if (tr.worldDeluxeMapping)
+		lightmapnum >>= 1;
+
+	lightmapnum %= (tr.fatLightmapCols * tr.fatLightmapRows);
+
+	for ( stage = 0; stage < MAX_SHADER_STAGES; stage++ ) {
+		shaderStage_t *pStage = &stages[stage];
+
+		if ( !pStage->active ) {
+			break;
+		}
+
+		// fix tcMod transform for internal lightmaps, it may be used by q3map2 lightstyles
+		if ( pStage->bundle[0].isLightmap ) {
+			for ( i = 0; i < pStage->bundle[0].numTexMods; i++ ) {
+				tmi = &pStage->bundle[0].texMods[i];
+
+				if ( tmi->type == TMOD_TRANSFORM ) {
+					tmi->translate[0] /= (float)tr.fatLightmapCols;
+					tmi->translate[1] /= (float)tr.fatLightmapRows;
+				}
+			}
+		}
+		// add a tcMod transform for external lightmaps to convert back to the original texcoords
+		else if ( pStage->bundle[0].tcGen == TCGEN_LIGHTMAP ) {
+			if ( pStage->bundle[0].numTexMods == TR_MAX_TEXMODS ) {
+				ri.Printf( PRINT_DEVELOPER, "WARNING: too many tcmods to fix external lightmap texcoords for r_mergeLightmaps in shader '%s'", shader.name );
+			} else {
+				size = pStage->bundle[0].numTexMods * sizeof( texModInfo_t );
+
+				if ( size ) {
+					memmove( &pStage->bundle[0].texMods[1], &pStage->bundle[0].texMods[0], size );
+				}
+
+				tmi = &pStage->bundle[0].texMods[0];
+				pStage->bundle[0].numTexMods++;
+
+				tmi->matrix[0][0] = tr.fatLightmapCols;
+				tmi->matrix[0][1] = 0;
+				tmi->matrix[1][0] = 0;
+				tmi->matrix[1][1] = tr.fatLightmapRows;
+
+				tmi->translate[0] = -(lightmapnum % tr.fatLightmapCols);
+				tmi->translate[1] = -(lightmapnum / tr.fatLightmapCols);
+
+				tmi->type = TMOD_TRANSFORM;
+			}
+		}
+	}
+}
+
 /*
 ===============
 InitShader
@@ -3081,6 +3168,8 @@ static shader_t *FinishShader( void ) {
 		hasLightmapStage = qfalse;
 	}
 
+	FixFatLightmapTexCoords();
+
 	//
 	// look for multitexture potential
 	//