- did a complete workover of the weapon sprite translucency code that got inherited from QZDoom.

This was very poorly done without ever addressing the issues a composite render style can bring, it merely dealt with the known legacy render styles.
The same, identical code was also present in two different places.
The oversight that AlterWeaponSprite overrode even forced styles was also fixed.

OpenGL is not implemented yet but with the problems eliminated should be doable now.

src/p_pspr.cpp

@@ -122,7 +122,6 @@ DEFINE_FIELD(DPSprite, oldy)
 DEFINE_FIELD(DPSprite, firstTic)
 DEFINE_FIELD(DPSprite, Tics)
 DEFINE_FIELD(DPSprite, alpha)
-DEFINE_FIELD(DPSprite, RenderStyle)
 DEFINE_FIELD_BIT(DPSprite, Flags, bAddWeapon, PSPF_ADDWEAPON)
 DEFINE_FIELD_BIT(DPSprite, Flags, bAddBob, PSPF_ADDBOB)
 DEFINE_FIELD_BIT(DPSprite, Flags, bPowDouble, PSPF_POWDOUBLE)
@@ -147,7 +146,7 @@ DPSprite::DPSprite(player_t *owner, AActor *caller, int id)
   processPending(true)
 {
 	alpha = 1;
-	RenderStyle = STYLE_Normal;
+	Renderstyle = STYLE_Normal;
 
 	DPSprite *prev = nullptr;
 	DPSprite *next = Owner->psprites;
@@ -298,6 +297,79 @@ DEFINE_ACTION_FUNCTION(_PlayerInfo, GetPSprite)	// the underscore is needed to g
 }
 
 
+
+std::pair<FRenderStyle, float> DPSprite::GetRenderStyle(FRenderStyle ownerstyle, double owneralpha)
+{
+	FRenderStyle returnstyle, mystyle;
+	double returnalpha;
+
+	ownerstyle.CheckFuzz();
+	mystyle = Renderstyle;
+	mystyle.CheckFuzz();
+	if (Flags & PSPF_RENDERSTYLE)
+	{
+		ownerstyle.CheckFuzz();
+		if (Flags & PSPF_FORCESTYLE)
+		{
+			returnstyle = mystyle;
+		}
+		else if (ownerstyle.BlendOp != STYLEOP_Add)
+		{
+			// all styles that do more than simple blending need to be fully preserved.
+			returnstyle = ownerstyle;
+		}
+		else
+		{
+			returnstyle = mystyle;
+			if (ownerstyle.DestAlpha == STYLEALPHA_One)
+			{
+				// If the owner is additive and the overlay translucent, force additive result.
+				returnstyle.DestAlpha = STYLEALPHA_One;
+			}
+			if (ownerstyle.Flags & (STYLEF_ColorIsFixed|STYLEF_RedIsAlpha))
+			{
+				// If the owner's style is a stencil type, this must be preserved.
+				returnstyle.Flags = ownerstyle.Flags;
+			}
+		}
+	}
+	else
+	{
+		returnstyle = ownerstyle;
+	}
+
+	if ((Flags & PSPF_FORCEALPHA) && returnstyle.BlendOp != STYLEOP_Fuzz && returnstyle.BlendOp != STYLEOP_Shadow)
+	{
+		// ALWAYS take priority if asked for, except fuzz. Fuzz does absolutely nothing
+		// no matter what way it's changed.
+		returnalpha = alpha;
+		returnstyle.Flags &= ~(STYLEF_Alpha1 | STYLEF_TransSoulsAlpha);
+	}
+	else if (Flags & PSPF_ALPHA)
+	{
+		// Set the alpha based on if using the overlay's own or not. Also adjust
+		// and override the alpha if not forced.
+		if (returnstyle.BlendOp != STYLEOP_Add)
+		{
+			returnalpha = owneralpha;
+		}
+		else
+		{
+			returnalpha = owneralpha * alpha;
+		}
+	}
+
+	// Should normal renderstyle come out on top at the end and we desire alpha,
+	// switch it to translucent. Normal never applies any sort of alpha.
+	if (returnstyle.BlendOp == STYLEOP_Add && returnstyle.SrcAlpha == STYLEALPHA_One && returnstyle.DestAlpha == STYLEALPHA_Zero && returnalpha < 1.)
+	{
+		returnstyle = LegacyRenderStyles[STYLE_Translucent];
+		returnalpha = owneralpha * alpha;
+	}
+
+	return{ returnstyle, clamp<float>(float(alpha), 0.f, 1.f) };
+}
+
 //---------------------------------------------------------------------------
 //
 // PROC P_NewPspriteTick
@@ -1237,7 +1309,7 @@ DEFINE_ACTION_FUNCTION(AActor, A_OverlayRenderStyle)
 		if (pspr == nullptr || style >= STYLE_Count || style < 0)
 			return 0;
 
-		pspr->RenderStyle = style;
+		pspr->Renderstyle = ERenderStyle(style);
 	}
 	return 0;
 }
@@ -1472,7 +1544,7 @@ void DPSprite::Serialize(FSerializer &arc)
 		("oldx", oldx)
 		("oldy", oldy)
 		("alpha", alpha)
-		("renderstyle", RenderStyle);
+		("renderstyle_", Renderstyle);	// The underscore is intentional to avoid problems with old savegames which had this as an ERenderStyle (which is not future proof.)
 }
 
 //------------------------------------------------------------------------

src/p_pspr.h

@@ -23,6 +23,8 @@
 #ifndef __P_PSPR_H__
 #define __P_PSPR_H__
 
+#include "r_data/renderstyle.h"
+
 // Basic data types.
 // Needs fixed point, and BAM angles.
 
@@ -81,13 +83,14 @@ public:
 	void		SetCaller(AActor *newcaller) { Caller = newcaller; }
 	void		ResetInterpolation() { oldx = x; oldy = y; }
 	void OnDestroy() override;
+	std::pair<FRenderStyle, float> GetRenderStyle(FRenderStyle ownerstyle, double owneralpha);
 
 	double x, y, alpha;
 	double oldx, oldy;
 	bool firstTic;
 	int Tics;
 	int Flags;
-	int RenderStyle;
+	FRenderStyle Renderstyle;
 
 private:
 	DPSprite () {}

src/polyrenderer/scene/poly_playersprite.cpp

@@ -217,7 +217,7 @@ void RenderPolyPlayerSprites::RenderSprite(DPSprite *pspr, AActor *owner, float
 	flip = sprframe->Flip & 1;
 	tex = TexMan(picnum);
 
-	if (tex->UseType == FTexture::TEX_Null || pspr->RenderStyle == STYLE_None)
+	if (tex->UseType == FTexture::TEX_Null)
 		return;
 
 	if (pspr->firstTic)
@@ -319,112 +319,11 @@ void RenderPolyPlayerSprites::RenderSprite(DPSprite *pspr, AActor *owner, float
 	FDynamicColormap *colormap_to_use = nullptr;
 	if (pspr->GetID() < PSP_TARGETCENTER)
 	{
-		// [MC] Set the render style 
+		auto rs = pspr->GetRenderStyle(owner->RenderStyle, owner->Alpha);
+		vis.RenderStyle = rs.first;
+		vis.Alpha = rs.second;
 
-		if (pspr->Flags & PSPF_RENDERSTYLE)
-		{
-			const int rs = clamp<int>(pspr->RenderStyle, 0, STYLE_Count);
-
-			if (pspr->Flags & PSPF_FORCESTYLE)
-			{
-				vis.RenderStyle = LegacyRenderStyles[rs];
-			}
-			else if (owner->RenderStyle == LegacyRenderStyles[STYLE_Fuzzy])
-			{
-				vis.RenderStyle = LegacyRenderStyles[STYLE_Fuzzy];
-			}
-			else if (owner->RenderStyle == LegacyRenderStyles[STYLE_OptFuzzy])
-			{
-				vis.RenderStyle = LegacyRenderStyles[STYLE_OptFuzzy];
-				vis.RenderStyle.CheckFuzz();
-			}
-			else if (owner->RenderStyle == LegacyRenderStyles[STYLE_Subtract])
-			{
-				vis.RenderStyle = LegacyRenderStyles[STYLE_Subtract];
-			}
-			else
-			{
-				vis.RenderStyle = LegacyRenderStyles[rs];
-			}
-		}
-		else
-		{
-			vis.RenderStyle = owner->RenderStyle;
-		}
-
-		// Set the alpha based on if using the overlay's own or not. Also adjust
-		// and override the alpha if not forced.
-		if (pspr->Flags & PSPF_ALPHA)
-		{
-			if (vis.RenderStyle == LegacyRenderStyles[STYLE_Fuzzy])
-			{
-				alpha = owner->Alpha;
-			}
-			else if (vis.RenderStyle == LegacyRenderStyles[STYLE_OptFuzzy])
-			{
-				FRenderStyle style = vis.RenderStyle;
-				style.CheckFuzz();
-				switch (style.BlendOp)
-				{
-				default:
-					alpha = pspr->alpha * owner->Alpha;
-					break;
-				case STYLEOP_Fuzz:
-				case STYLEOP_Sub:
-					alpha = owner->Alpha;
-					break;
-				}
-
-			}
-			else if (vis.RenderStyle == LegacyRenderStyles[STYLE_Subtract])
-			{
-				alpha = owner->Alpha;
-			}
-			else if (vis.RenderStyle == LegacyRenderStyles[STYLE_Add] ||
-				vis.RenderStyle == LegacyRenderStyles[STYLE_Translucent] ||
-				vis.RenderStyle == LegacyRenderStyles[STYLE_TranslucentStencil] ||
-				vis.RenderStyle == LegacyRenderStyles[STYLE_AddStencil] ||
-				vis.RenderStyle == LegacyRenderStyles[STYLE_AddShaded])
-			{
-				alpha = owner->Alpha * pspr->alpha;
-			}
-			else
-			{
-				alpha = owner->Alpha;
-			}
-		}
-
-		// Should normal renderstyle come out on top at the end and we desire alpha,
-		// switch it to translucent. Normal never applies any sort of alpha.
-		if ((pspr->Flags & PSPF_ALPHA) &&
-			vis.RenderStyle == LegacyRenderStyles[STYLE_Normal] &&
-			vis.Alpha < 1.0)
-		{
-			vis.RenderStyle = LegacyRenderStyles[STYLE_Translucent];
-			alpha = owner->Alpha * pspr->alpha;
-		}
-
-		// ALWAYS take priority if asked for, except fuzz. Fuzz does absolutely nothing
-		// no matter what way it's changed.
-		if (pspr->Flags & PSPF_FORCEALPHA)
-		{
-			//Due to lack of != operators...
-			if (vis.RenderStyle == LegacyRenderStyles[STYLE_Fuzzy] ||
-				vis.RenderStyle == LegacyRenderStyles[STYLE_SoulTrans] ||
-				vis.RenderStyle == LegacyRenderStyles[STYLE_Stencil])
-			{
-			}
-			else
-			{
-				alpha = pspr->alpha;
-				vis.RenderStyle.Flags |= STYLEF_ForceAlpha;
-			}
-		}
-		vis.Alpha = clamp<float>(float(alpha), 0.f, 1.f);
-
-		// Due to how some of the effects are handled, going to 0 or less causes some
-		// weirdness to display. There's no point rendering it anyway if it's 0.
-		if (vis.Alpha <= 0.)
+		if (!vis.RenderStyle.IsVisible(vis.Alpha))
 			return;
 
 		//-----------------------------------------------------------------------------
@@ -455,9 +354,9 @@ void RenderPolyPlayerSprites::RenderSprite(DPSprite *pspr, AActor *owner, float
 
 			viewpoint.camera->Inventory->AlterWeaponSprite(&visstyle);
 
-			vis.Alpha = visstyle.Alpha;
+			if (!(pspr->Flags & PSPF_FORCEALPHA)) vis.Alpha = visstyle.Alpha;
 
-			if (visstyle.RenderStyle != STYLE_Count)
+			if (visstyle.RenderStyle != STYLE_Count && !(pspr->Flags & PSPF_FORCESTYLE))
 			{
 				vis.RenderStyle = visstyle.RenderStyle;
 			}

src/r_data/renderstyle.h

@@ -119,9 +119,6 @@ enum ERenderFlags
 	// Actors only: Ignore sector fade and fade to black. To fade to white,
 	// combine this with STYLEF_InvertOverlay.
 	STYLEF_FadeToBlack = 64,
-
-	// Force alpha.
-	STYLEF_ForceAlpha = 128,
 };
 
 union FRenderStyle

src/swrenderer/things/r_playersprite.cpp

@@ -217,7 +217,7 @@ namespace swrenderer
 		flip = sprframe->Flip & 1;
 		tex = TexMan(picnum);
 
-		if (tex->UseType == FTexture::TEX_Null || pspr->RenderStyle == STYLE_None)
+		if (tex->UseType == FTexture::TEX_Null)
 			return;
 
 		if (pspr->firstTic)
@@ -320,111 +320,11 @@ namespace swrenderer
 		if (pspr->GetID() < PSP_TARGETCENTER)
 		{
 			// [MC] Set the render style 
+			auto rs = pspr->GetRenderStyle(owner->RenderStyle, owner->Alpha);
+			vis.RenderStyle = rs.first;
+			vis.Alpha = rs.second;
 
-			if (pspr->Flags & PSPF_RENDERSTYLE)
-			{
-				const int rs = clamp<int>(pspr->RenderStyle, 0, STYLE_Count);
-
-				if (pspr->Flags & PSPF_FORCESTYLE)
-				{
-					vis.RenderStyle = LegacyRenderStyles[rs];
-				}
-				else if (owner->RenderStyle == LegacyRenderStyles[STYLE_Fuzzy])
-				{
-					vis.RenderStyle = LegacyRenderStyles[STYLE_Fuzzy];
-				}
-				else if (owner->RenderStyle == LegacyRenderStyles[STYLE_OptFuzzy])
-				{
-					vis.RenderStyle = LegacyRenderStyles[STYLE_OptFuzzy];
-					vis.RenderStyle.CheckFuzz();
-				}
-				else if (owner->RenderStyle == LegacyRenderStyles[STYLE_Subtract])
-				{
-					vis.RenderStyle = LegacyRenderStyles[STYLE_Subtract];
-				}
-				else
-				{
-					vis.RenderStyle = LegacyRenderStyles[rs];
-				}
-			}
-			else
-			{
-				vis.RenderStyle = owner->RenderStyle;
-			}
-
-			// Set the alpha based on if using the overlay's own or not. Also adjust
-			// and override the alpha if not forced.
-			if (pspr->Flags & PSPF_ALPHA)
-			{
-				if (vis.RenderStyle == LegacyRenderStyles[STYLE_Fuzzy])
-				{
-					alpha = owner->Alpha;
-				}
-				else if (vis.RenderStyle == LegacyRenderStyles[STYLE_OptFuzzy])
-				{
-					FRenderStyle style = vis.RenderStyle;
-					style.CheckFuzz();
-					switch (style.BlendOp)
-					{
-					default:
-						alpha = pspr->alpha * owner->Alpha;
-						break;
-					case STYLEOP_Fuzz:
-					case STYLEOP_Sub:
-						alpha = owner->Alpha;
-						break;
-					}
-
-				}
-				else if (vis.RenderStyle == LegacyRenderStyles[STYLE_Subtract])
-				{
-					alpha = owner->Alpha;
-				}
-				else if (vis.RenderStyle == LegacyRenderStyles[STYLE_Add] ||
-					vis.RenderStyle == LegacyRenderStyles[STYLE_Translucent] ||
-					vis.RenderStyle == LegacyRenderStyles[STYLE_TranslucentStencil] ||
-					vis.RenderStyle == LegacyRenderStyles[STYLE_AddStencil] ||
-					vis.RenderStyle == LegacyRenderStyles[STYLE_AddShaded])
-				{
-					alpha = owner->Alpha * pspr->alpha;
-				}
-				else
-				{
-					alpha = owner->Alpha;
-				}
-			}
-
-			// Should normal renderstyle come out on top at the end and we desire alpha,
-			// switch it to translucent. Normal never applies any sort of alpha.
-			if ((pspr->Flags & PSPF_ALPHA) &&
-				vis.RenderStyle == LegacyRenderStyles[STYLE_Normal] &&
-				vis.Alpha < 1.0)
-			{
-				vis.RenderStyle = LegacyRenderStyles[STYLE_Translucent];
-				alpha = owner->Alpha * pspr->alpha;
-			}
-
-			// ALWAYS take priority if asked for, except fuzz. Fuzz does absolutely nothing
-			// no matter what way it's changed.
-			if (pspr->Flags & PSPF_FORCEALPHA)
-			{
-				//Due to lack of != operators...
-				if (vis.RenderStyle == LegacyRenderStyles[STYLE_Fuzzy] ||
-					vis.RenderStyle == LegacyRenderStyles[STYLE_SoulTrans] ||
-					vis.RenderStyle == LegacyRenderStyles[STYLE_Stencil])
-				{
-				}
-				else
-				{
-					alpha = pspr->alpha;
-					vis.RenderStyle.Flags |= STYLEF_ForceAlpha;
-				}
-			}
-			vis.Alpha = clamp<float>(float(alpha), 0.f, 1.f);
-
-			// Due to how some of the effects are handled, going to 0 or less causes some
-			// weirdness to display. There's no point rendering it anyway if it's 0.
-			if (vis.Alpha <= 0.)
+			if (!vis.RenderStyle.IsVisible(vis.Alpha))
 				return;
 
 			//-----------------------------------------------------------------------------
@@ -455,9 +355,9 @@ namespace swrenderer
 				
 				Thread->Viewport->viewpoint.camera->Inventory->AlterWeaponSprite(&visstyle);
 				
-				vis.Alpha = visstyle.Alpha;
+				if (!(pspr->Flags & PSPF_FORCEALPHA)) vis.Alpha = visstyle.Alpha;
 
-				if (visstyle.RenderStyle != STYLE_Count)
+				if (visstyle.RenderStyle != STYLE_Count && !(pspr->Flags & PSPF_FORCESTYLE))
 				{
 					vis.RenderStyle = visstyle.RenderStyle;
 				}

src/swrenderer/viewport/r_spritedrawer.cpp

@@ -393,11 +393,7 @@ namespace swrenderer
 			color = 0;
 		}
 
-		if (style.Flags & STYLEF_ForceAlpha)
-		{
-			alpha = clamp<fixed_t>(alpha, 0, OPAQUE);
-		}
-		else if (style.Flags & STYLEF_TransSoulsAlpha)
+		if (style.Flags & STYLEF_TransSoulsAlpha)
 		{
 			alpha = fixed_t(transsouls * OPAQUE);
 		}

wadsrc/static/zscript/shared/player.txt

@@ -223,7 +223,7 @@ class PSprite : Object native play
 	native readonly PlayerInfo Owner;
 	native SpriteID Sprite;
 	native int Frame;
-	native readonly int RenderStyle;
+	//native readonly int RenderStyle;	had to be blocked because the internal representation was not ok. Renderstyle is still pending a proper solution.
 	native readonly int ID;
 	native Bool processPending;
 	native double x;