Merge colormap selection into one function

src/swrenderer/things/r_particle.cpp

@@ -67,7 +67,6 @@ namespace swrenderer
 		double	 			xscale, yscale;
 		int 				x1, x2, y1, y2;
 		sector_t*			heightsec = NULL;
-		FSWColormap*			map;
 		
 		RenderPortal *renderportal = RenderPortal::Instance();
 
@@ -135,6 +134,7 @@ namespace swrenderer
 		const secplane_t *botplane;
 		FTextureID toppic;
 		FTextureID botpic;
+		FDynamicColormap *map;
 
 		if (heightsec)	// only clip things which are in special sectors
 		{
@@ -200,27 +200,8 @@ namespace swrenderer
 		vis->ColormapNum = 0;
 		vis->foggy = foggy;
 
-		if (fixedlightlev >= 0)
+		// Particles are slightly more visible than regular sprites.
-		{
+		vis->SetColormap(tiz * r_SpriteVisibility * 0.5, shade, map, particle->bright != 0, false, false);
-			vis->BaseColormap = map;
-			vis->ColormapNum = fixedlightlev >> COLORMAPSHIFT;
-		}
-		else if (fixedcolormap)
-		{
-			vis->BaseColormap = fixedcolormap;
-			vis->ColormapNum = 0;
-		}
-		else if (particle->bright)
-		{
-			vis->BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : map;
-			vis->ColormapNum = 0;
-		}
-		else
-		{
-			// Particles are slightly more visible than regular sprites.
-			vis->ColormapNum = GETPALOOKUP(tiz * r_SpriteVisibility * 0.5, shade);
-			vis->BaseColormap = map;
-		}
 
 		VisibleSpriteList::Instance()->Push(vis);
 	}

src/swrenderer/things/r_playersprite.cpp

@@ -440,55 +440,20 @@ namespace swrenderer
 			// The software renderer cannot invert the source without inverting the overlay
 			// too. That means if the source is inverted, we need to do the reverse of what
 			// the invert overlay flag says to do.
-			INTBOOL invertcolormap = (vis->RenderStyle.Flags & STYLEF_InvertOverlay);
+			bool invertcolormap = (vis->RenderStyle.Flags & STYLEF_InvertOverlay) != 0;
 
 			if (vis->RenderStyle.Flags & STYLEF_InvertSource)
 			{
 				invertcolormap = !invertcolormap;
 			}
 
-			FDynamicColormap *mybasecolormap = basecolormap;
+			bool fullbright = !vis->foggy && pspr->GetState()->GetFullbright();
+			bool fadeToBlack = (vis->RenderStyle.Flags & STYLEF_FadeToBlack) != 0;
 
-			if (vis->RenderStyle.Flags & STYLEF_FadeToBlack)
+			vis->SetColormap(0, spriteshade, basecolormap, fullbright, invertcolormap, fadeToBlack);
-			{
+
-				if (invertcolormap)
+			colormap_to_use = (FDynamicColormap*)vis->BaseColormap;
-				{ // Fade to white
-					mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(255, 255, 255), mybasecolormap->Desaturate);
-					invertcolormap = false;
-				}
-				else
-				{ // Fade to black
-					mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(0, 0, 0), mybasecolormap->Desaturate);
-				}
-			}
 
-			if (realfixedcolormap != nullptr && (!r_swtruecolor || (r_shadercolormaps && screen->Accel2D)))
-			{ // fixed color
-				vis->BaseColormap = realfixedcolormap;
-				vis->ColormapNum = 0;
-			}
-			else
-			{
-				if (invertcolormap)
-				{
-					mybasecolormap = GetSpecialLights(mybasecolormap->Color, mybasecolormap->Fade.InverseColor(), mybasecolormap->Desaturate);
-				}
-				if (fixedlightlev >= 0)
-				{
-					vis->BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : mybasecolormap;
-					vis->ColormapNum = fixedlightlev >> COLORMAPSHIFT;
-				}
-				else if (!vis->foggy && pspr->GetState()->GetFullbright())
-				{ // full bright
-					vis->BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : mybasecolormap;	// [RH] use basecolormap
-					vis->ColormapNum = 0;
-				}
-				else
-				{ // local light
-					vis->BaseColormap = mybasecolormap;
-					vis->ColormapNum = GETPALOOKUP(0, spriteshade);
-				}
-			}
 			if (camera->Inventory != nullptr)
 			{
 				visstyle_t visstyle;
@@ -509,10 +474,7 @@ namespace swrenderer
 				{
 					vis->BaseColormap = &SpecialColormaps[INVERSECOLORMAP];
 					vis->ColormapNum = 0;
-					if (vis->BaseColormap->Maps < mybasecolormap->Maps || vis->BaseColormap->Maps >= mybasecolormap->Maps + NUMCOLORMAPS * 256)
+					noaccel = true;
-					{
-						noaccel = true;
-					}
 				}
 			}
 			// If we're drawing with a special colormap, but shaders for them are disabled, do
@@ -523,24 +485,18 @@ namespace swrenderer
 				noaccel = true;
 			}
 			// If drawing with a BOOM colormap, disable acceleration.
-			if (mybasecolormap == &NormalLight && NormalLight.Maps != realcolormaps.Maps)
+			if (vis->BaseColormap == &NormalLight && NormalLight.Maps != realcolormaps.Maps)
 			{
 				noaccel = true;
 			}
 			// If the main colormap has fixed lights, and this sprite is being drawn with that
 			// colormap, disable acceleration so that the lights can remain fixed.
 			if (!noaccel && realfixedcolormap == nullptr &&
-				NormalLightHasFixedLights && mybasecolormap == &NormalLight &&
+				NormalLightHasFixedLights && vis->BaseColormap == &NormalLight &&
 				vis->pic->UseBasePalette())
 			{
 				noaccel = true;
 			}
-			// [SP] If emulating GZDoom fullbright, disable acceleration
-			if (r_fullbrightignoresectorcolor && fixedlightlev >= 0)
-				mybasecolormap = &FullNormalLight;
-			if (r_fullbrightignoresectorcolor && !vis->foggy && pspr->GetState()->GetFullbright())
-				mybasecolormap = &FullNormalLight;
-			colormap_to_use = mybasecolormap;
 		}
 		else
 		{

src/swrenderer/things/r_sprite.cpp

@@ -212,12 +212,9 @@ namespace swrenderer
 		// The software renderer cannot invert the source without inverting the overlay
 		// too. That means if the source is inverted, we need to do the reverse of what
 		// the invert overlay flag says to do.
-		INTBOOL invertcolormap = (vis->RenderStyle.Flags & STYLEF_InvertOverlay);
+		bool invertcolormap = (vis->RenderStyle.Flags & STYLEF_InvertOverlay) != 0;
-
 		if (vis->RenderStyle.Flags & STYLEF_InvertSource)
-		{
 			invertcolormap = !invertcolormap;
-		}
 
 		FDynamicColormap *mybasecolormap = basecolormap;
 		if (current_sector->sectornum != thing->Sector->sectornum)	// compare sectornums to account for R_FakeFlat copies.
@@ -225,53 +222,15 @@ namespace swrenderer
 			// Todo: The actor is from a different sector so we have to retrieve the proper basecolormap for that sector.
 		}
 
-		// Sprites that are added to the scene must fade to black.
 		if (vis->RenderStyle == LegacyRenderStyles[STYLE_Add] && mybasecolormap->Fade != 0)
 		{
 			mybasecolormap = GetSpecialLights(mybasecolormap->Color, 0, mybasecolormap->Desaturate);
 		}
 
-		if (vis->RenderStyle.Flags & STYLEF_FadeToBlack)
+		bool fullbright = !vis->foggy && ((renderflags & RF_FULLBRIGHT) || (thing->flags5 & MF5_BRIGHT));
-		{
+		bool fadeToBlack = (vis->RenderStyle.Flags & STYLEF_FadeToBlack) != 0;
-			if (invertcolormap)
-			{ // Fade to white
-				mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(255, 255, 255), mybasecolormap->Desaturate);
-				invertcolormap = false;
-			}
-			else
-			{ // Fade to black
-				mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(0, 0, 0), mybasecolormap->Desaturate);
-			}
-		}
 
-		// get light level
+		vis->SetColormap(r_SpriteVisibility / MAX(tz, MINZ), spriteshade, mybasecolormap, fullbright, invertcolormap, fadeToBlack);
-		if (fixedcolormap != nullptr)
-		{ // fixed map
-			vis->BaseColormap = fixedcolormap;
-			vis->ColormapNum = 0;
-		}
-		else
-		{
-			if (invertcolormap)
-			{
-				mybasecolormap = GetSpecialLights(mybasecolormap->Color, mybasecolormap->Fade.InverseColor(), mybasecolormap->Desaturate);
-			}
-			if (fixedlightlev >= 0)
-			{
-				vis->BaseColormap = mybasecolormap;
-				vis->ColormapNum = fixedlightlev >> COLORMAPSHIFT;
-			}
-			else if (!vis->foggy && ((renderflags & RF_FULLBRIGHT) || (thing->flags5 & MF5_BRIGHT)))
-			{ // full bright
-				vis->BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : mybasecolormap;
-				vis->ColormapNum = 0;
-			}
-			else
-			{ // diminished light
-				vis->ColormapNum = GETPALOOKUP(r_SpriteVisibility / MAX(tz, MINZ), spriteshade);
-				vis->BaseColormap = mybasecolormap;
-			}
-		}
 
 		VisibleSpriteList::Instance()->Push(vis);
 	}

src/swrenderer/things/r_visiblesprite.cpp

@@ -55,7 +55,6 @@ namespace swrenderer
 		FSWColormap *colormap = spr->BaseColormap;
 		int colormapnum = spr->ColormapNum;
 		F3DFloor *rover;
-		FDynamicColormap *mybasecolormap;
 
 		Clip3DFloors *clip3d = Clip3DFloors::Instance();
 
@@ -93,6 +92,7 @@ namespace swrenderer
 				clip3d->sclipTop = spr->sector->ceilingplane.ZatPoint(ViewPos);
 			}
 			sector_t *sec = nullptr;
+			FDynamicColormap *mybasecolormap = nullptr;
 			for (i = spr->sector->e->XFloor.lightlist.Size() - 1; i >= 0; i--)
 			{
 				if (clip3d->sclipTop <= spr->sector->e->XFloor.lightlist[i].plane.Zat0())
@@ -120,53 +120,21 @@ namespace swrenderer
 			// found new values, recalculate
 			if (sec)
 			{
-				INTBOOL invertcolormap = (spr->RenderStyle.Flags & STYLEF_InvertOverlay);
+				bool invertcolormap = (spr->RenderStyle.Flags & STYLEF_InvertOverlay) != 0;
-
 				if (spr->RenderStyle.Flags & STYLEF_InvertSource)
-				{
 					invertcolormap = !invertcolormap;
-				}
 
-				// Sprites that are added to the scene must fade to black.
+				if (RenderStyle == LegacyRenderStyles[STYLE_Add] && mybasecolormap->Fade != 0)
-				if (spr->RenderStyle == LegacyRenderStyles[STYLE_Add] && mybasecolormap->Fade != 0)
 				{
 					mybasecolormap = GetSpecialLights(mybasecolormap->Color, 0, mybasecolormap->Desaturate);
 				}
 
-				if (spr->RenderStyle.Flags & STYLEF_FadeToBlack)
+				bool isFullBright = !foggy && (renderflags & RF_FULLBRIGHT);
-				{
+				bool fadeToBlack = spr->RenderStyle == LegacyRenderStyles[STYLE_Add] && mybasecolormap->Fade != 0;
-					if (invertcolormap)
-					{ // Fade to white
-						mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(255, 255, 255), mybasecolormap->Desaturate);
-						invertcolormap = false;
-					}
-					else
-					{ // Fade to black
-						mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(0, 0, 0), mybasecolormap->Desaturate);
-					}
-				}
 
-				// get light level
+				int spriteshade = LIGHT2SHADE(sec->lightlevel + R_ActualExtraLight(spr->foggy));
-				if (invertcolormap)
+
-				{
+				SetColormap(r_SpriteVisibility / MAX(MINZ, (double)spr->depth), spriteshade, mybasecolormap, isFullBright, invertcolormap, fadeToBlack);
-					mybasecolormap = GetSpecialLights(mybasecolormap->Color, mybasecolormap->Fade.InverseColor(), mybasecolormap->Desaturate);
-				}
-				if (fixedlightlev >= 0)
-				{
-					spr->BaseColormap = mybasecolormap;
-					spr->ColormapNum = fixedlightlev >> COLORMAPSHIFT;
-				}
-				else if (!spr->foggy && (spr->renderflags & RF_FULLBRIGHT))
-				{ // full bright
-					spr->BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : mybasecolormap;
-					spr->ColormapNum = 0;
-				}
-				else
-				{ // diminished light
-					int spriteshade = LIGHT2SHADE(sec->lightlevel + R_ActualExtraLight(spr->foggy));
-					spr->BaseColormap = mybasecolormap;
-					spr->ColormapNum = GETPALOOKUP(r_SpriteVisibility / MAX(MINZ, (double)spr->depth), spriteshade);
-				}
 			}
 		}
 
@@ -440,4 +408,46 @@ namespace swrenderer
 		spr->BaseColormap = colormap;
 		spr->ColormapNum = colormapnum;
 	}
+
+	void VisibleSprite::SetColormap(double visibility, int shade, FDynamicColormap *basecolormap, bool fullbright, bool invertColormap, bool fadeToBlack)
+	{
+		if (fadeToBlack)
+		{
+			if (invertColormap) // Fade to white
+			{
+				basecolormap = GetSpecialLights(basecolormap->Color, MAKERGB(255, 255, 255), basecolormap->Desaturate);
+				invertColormap = false;
+			}
+			else // Fade to black
+			{
+				basecolormap = GetSpecialLights(basecolormap->Color, MAKERGB(0, 0, 0), basecolormap->Desaturate);
+			}
+		}
+
+		if (invertColormap)
+		{
+			basecolormap = GetSpecialLights(basecolormap->Color, basecolormap->Fade.InverseColor(), basecolormap->Desaturate);
+		}
+
+		if (fixedcolormap)
+		{
+			BaseColormap = fixedcolormap;
+			ColormapNum = 0;
+		}
+		else if (fixedlightlev >= 0)
+		{
+			BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : basecolormap;
+			ColormapNum = fixedlightlev >> COLORMAPSHIFT;
+		}
+		else if (fullbright)
+		{
+			BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : basecolormap;
+			ColormapNum = 0;
+		}
+		else
+		{
+			BaseColormap = basecolormap;
+			ColormapNum = GETPALOOKUP(visibility, shade);
+		}
+	}
 }

src/swrenderer/things/r_visiblesprite.h

@@ -37,6 +37,8 @@ namespace swrenderer
 		float SortDist() const { return idepth; }
 
 	protected:
+		void SetColormap(double visibility, int shade, FDynamicColormap *basecolormap, bool fullbright, bool invertColormap, bool fadeToBlack);
+
 		virtual bool IsParticle() const { return false; }
 		virtual bool IsVoxel() const { return false; }
 		virtual bool IsWallSprite() const { return false; }

src/swrenderer/things/r_voxel.cpp

@@ -159,7 +159,7 @@ namespace swrenderer
 		// The software renderer cannot invert the source without inverting the overlay
 		// too. That means if the source is inverted, we need to do the reverse of what
 		// the invert overlay flag says to do.
-		INTBOOL invertcolormap = (vis->RenderStyle.Flags & STYLEF_InvertOverlay);
+		bool invertcolormap = (vis->RenderStyle.Flags & STYLEF_InvertOverlay) != 0;
 
 		if (vis->RenderStyle.Flags & STYLEF_InvertSource)
 		{
@@ -178,48 +178,10 @@ namespace swrenderer
 			mybasecolormap = GetSpecialLights(mybasecolormap->Color, 0, mybasecolormap->Desaturate);
 		}
 
-		if (vis->RenderStyle.Flags & STYLEF_FadeToBlack)
+		bool fullbright = !vis->foggy && ((renderflags & RF_FULLBRIGHT) || (thing->flags5 & MF5_BRIGHT));
-		{
+		bool fadeToBlack = (vis->RenderStyle.Flags & STYLEF_FadeToBlack) != 0;
-			if (invertcolormap)
-			{ // Fade to white
-				mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(255, 255, 255), mybasecolormap->Desaturate);
-				invertcolormap = false;
-			}
-			else
-			{ // Fade to black
-				mybasecolormap = GetSpecialLights(mybasecolormap->Color, MAKERGB(0, 0, 0), mybasecolormap->Desaturate);
-			}
-		}
 
-		// get light level
+		vis->SetColormap(r_SpriteVisibility / MAX(tz, MINZ), spriteshade, mybasecolormap, fullbright, invertcolormap, fadeToBlack);
-		if (fixedcolormap != nullptr)
-		{ // fixed map
-			vis->BaseColormap = fixedcolormap;
-			vis->ColormapNum = 0;
-		}
-		else
-		{
-			if (invertcolormap)
-			{
-				mybasecolormap = GetSpecialLights(mybasecolormap->Color, mybasecolormap->Fade.InverseColor(), mybasecolormap->Desaturate);
-			}
-
-			if (fixedlightlev >= 0)
-			{
-				vis->BaseColormap = mybasecolormap;
-				vis->ColormapNum = fixedlightlev >> COLORMAPSHIFT;
-			}
-			else if (!vis->foggy && ((renderflags & RF_FULLBRIGHT) || (thing->flags5 & MF5_BRIGHT)))
-			{ // full bright
-				vis->BaseColormap = (r_fullbrightignoresectorcolor) ? &FullNormalLight : mybasecolormap;
-				vis->ColormapNum = 0;
-			}
-			else
-			{ // diminished light
-				vis->ColormapNum = GETPALOOKUP(r_SpriteVisibility / MAX(tz, MINZ), spriteshade);
-				vis->BaseColormap = mybasecolormap;
-			}
-		}
 
 		VisibleSpriteList::Instance()->Push(vis);
 		RenderTranslucentPass::DrewAVoxel = true;

src/swrenderer/things/r_wallsprite.cpp

@@ -129,11 +129,11 @@ namespace swrenderer
 		vis->fakeceiling = NULL;
 		//vis->bInMirror = renderportal->MirrorFlags & RF_XFLIP;
 		vis->pic = pic;
-		vis->ColormapNum = GETPALOOKUP(r_SpriteVisibility / MAX(tz, MINZ), spriteshade);
-		vis->BaseColormap = basecolormap;
 		vis->wallc = wallc;
 		vis->foggy = foggy;
 
+		vis->SetColormap(r_SpriteVisibility / MAX(tz, MINZ), spriteshade, basecolormap, false, false, false);
+
 		VisibleSpriteList::Instance()->Push(vis);
 	}