From 593627f0499ce3cda972368fe07dacaa00eac68d Mon Sep 17 00:00:00 2001
From: nashmuhandes <nashbackslash@gmail.com>
Date: Sun, 19 Jun 2022 17:25:59 +0800
Subject: [PATCH] Add +ONLYVISIBLEINMIRRORS and +INVISIBLEINMIRRORS actor
 flags. The former makes the actor only visible in reflections, while the
 latter makes the actor not cast reflections in mirrors.

---
 src/playsim/actor.h                              |  9 +++++++++
 src/playsim/p_mobj.cpp                           |  1 +
 src/rendering/hwrenderer/scene/hw_sprites.cpp    | 11 +++++++++++
 src/rendering/swrenderer/scene/r_opaque_pass.cpp | 11 +++++++++++
 src/rendering/swrenderer/scene/r_portal.cpp      |  3 +++
 src/rendering/swrenderer/scene/r_portal.h        |  4 ++++
 src/scripting/thingdef_data.cpp                  |  2 ++
 7 files changed, 41 insertions(+)

diff --git a/src/playsim/actor.h b/src/playsim/actor.h
index 722f59b0c..ced98b3a7 100644
--- a/src/playsim/actor.h
+++ b/src/playsim/actor.h
@@ -477,6 +477,12 @@ enum ActorRenderFlag
 	RF_NOSPRITESHADOW = 0x80000000,		// actor will not cast a sprite shadow
 };
 
+enum ActorRenderFlag2
+{
+	RF2_INVISIBLEINMIRRORS		= 0x0001,	// [Nash] won't render in mirrors
+	RF2_ONLYVISIBLEINMIRRORS	= 0x0002,	// [Nash] only renders in mirrors
+};
+
 // This translucency value produces the closest match to Heretic's TINTTAB.
 // ~40% of the value of the overlaid image shows through.
 const double HR_SHADOW = (0x6800 / 65536.);
@@ -579,6 +585,7 @@ typedef TFlags<ActorFlag6> ActorFlags6;
 typedef TFlags<ActorFlag7> ActorFlags7;
 typedef TFlags<ActorFlag8> ActorFlags8;
 typedef TFlags<ActorRenderFlag> ActorRenderFlags;
+typedef TFlags<ActorRenderFlag2> ActorRenderFlags2;
 typedef TFlags<ActorBounceFlag> ActorBounceFlags;
 typedef TFlags<ActorRenderFeatureFlag> ActorRenderFeatureFlags;
 DEFINE_TFLAGS_OPERATORS (ActorFlags)
@@ -590,6 +597,7 @@ DEFINE_TFLAGS_OPERATORS (ActorFlags6)
 DEFINE_TFLAGS_OPERATORS (ActorFlags7)
 DEFINE_TFLAGS_OPERATORS (ActorFlags8)
 DEFINE_TFLAGS_OPERATORS (ActorRenderFlags)
+DEFINE_TFLAGS_OPERATORS (ActorRenderFlags2)
 DEFINE_TFLAGS_OPERATORS (ActorBounceFlags)
 DEFINE_TFLAGS_OPERATORS (ActorRenderFeatureFlags)
 
@@ -1037,6 +1045,7 @@ public:
 	uint32_t			RenderHidden;		// current renderer must *not* have any of these features
 
 	ActorRenderFlags	renderflags;		// Different rendering flags
+	ActorRenderFlags2	renderflags2;		// More rendering flags...
 	ActorFlags		flags;
 	ActorFlags2		flags2;			// Heretic flags
 	ActorFlags3		flags3;			// [RH] Hexen/Heretic actor-dependant behavior made flaggable
diff --git a/src/playsim/p_mobj.cpp b/src/playsim/p_mobj.cpp
index 52771f786..bb2f6042d 100644
--- a/src/playsim/p_mobj.cpp
+++ b/src/playsim/p_mobj.cpp
@@ -205,6 +205,7 @@ void AActor::Serialize(FSerializer &arc)
 		A("scale", Scale)
 		A("renderstyle", RenderStyle)
 		A("renderflags", renderflags)
+		A("renderflags2", renderflags2)
 		A("picnum", picnum)
 		A("floorpic", floorpic)
 		A("ceilingpic", ceilingpic)
diff --git a/src/rendering/hwrenderer/scene/hw_sprites.cpp b/src/rendering/hwrenderer/scene/hw_sprites.cpp
index c60f472a6..6afff0610 100644
--- a/src/rendering/hwrenderer/scene/hw_sprites.cpp
+++ b/src/rendering/hwrenderer/scene/hw_sprites.cpp
@@ -735,6 +735,17 @@ void HWSprite::Process(HWDrawInfo *di, AActor* thing, sector_t * sector, area_t
 		viewmaster = thing->master;
 	}
 
+	// [Nash] filter visibility in mirrors
+	bool isInMirror = di->mCurrentPortal && (di->mCurrentPortal->mState->MirrorFlag > 0 || di->mCurrentPortal->mState->PlaneMirrorFlag > 0);
+	if (thing->renderflags2 & RF2_INVISIBLEINMIRRORS && isInMirror)
+	{
+		return;
+	}
+	else if (thing->renderflags2 & RF2_ONLYVISIBLEINMIRRORS && !isInMirror)
+	{
+		return;
+	}
+
 	// Some added checks if the camera actor is not supposed to be seen. It can happen that some portal setup has this actor in view in which case it may not be skipped here
 	if (viewmaster == camera && !vp.showviewer)
 	{
diff --git a/src/rendering/swrenderer/scene/r_opaque_pass.cpp b/src/rendering/swrenderer/scene/r_opaque_pass.cpp
index 14c4703e3..e598c15ac 100644
--- a/src/rendering/swrenderer/scene/r_opaque_pass.cpp
+++ b/src/rendering/swrenderer/scene/r_opaque_pass.cpp
@@ -1028,6 +1028,17 @@ namespace swrenderer
 		if (!renderportal->CurrentPortalInSkybox && renderportal->CurrentPortal && !!P_PointOnLineSidePrecise(thing->Pos(), renderportal->CurrentPortal->dst))
 			return false;
 
+		// [Nash] filter visibility in mirrors
+		bool isInMirror = renderportal != nullptr && renderportal->IsInMirrorRecursively;
+		if (thing->renderflags2 & RF2_INVISIBLEINMIRRORS && isInMirror)
+		{
+			return false;
+		}
+		else if (thing->renderflags2 & RF2_ONLYVISIBLEINMIRRORS && !isInMirror)
+		{
+			return false;
+		}
+
 		double distanceSquared = (thing->Pos() - Thread->Viewport->viewpoint.Pos).LengthSquared();
 		if (distanceSquared > sprite_distance_cull)
 			return false;
diff --git a/src/rendering/swrenderer/scene/r_portal.cpp b/src/rendering/swrenderer/scene/r_portal.cpp
index 2e94df39d..697f610aa 100644
--- a/src/rendering/swrenderer/scene/r_portal.cpp
+++ b/src/rendering/swrenderer/scene/r_portal.cpp
@@ -339,6 +339,8 @@ namespace swrenderer
 
 		if (pds->mirror)
 		{
+			IsInMirrorRecursively = true;
+
 			//vertex_t *v1 = ds->curline->v1;
 			vertex_t *v1 = pds->src->v1;
 
@@ -465,6 +467,7 @@ namespace swrenderer
 
 		CurrentPortal = prevpds;
 		MirrorFlags = prevmf;
+		IsInMirrorRecursively = false;
 		viewpoint.Angles.Yaw = startang;
 		viewpoint.Pos = startpos;
 		viewpoint.Path[0] = savedpath[0];
diff --git a/src/rendering/swrenderer/scene/r_portal.h b/src/rendering/swrenderer/scene/r_portal.h
index 34a49a4ff..cf1f075d5 100644
--- a/src/rendering/swrenderer/scene/r_portal.h
+++ b/src/rendering/swrenderer/scene/r_portal.h
@@ -48,6 +48,10 @@ namespace swrenderer
 		int WindowRight = 0;
 		uint16_t MirrorFlags = 0;
 
+		// [Nash] this is set when first entering a mirror portal, and is unset when leaving the final mirror portal recursion
+		// Used for the RF2_INVISIBLEINMIRRORS and RF2_ONLYVISIBLEINMIRRORS features
+		bool IsInMirrorRecursively = false;
+
 		PortalDrawseg* CurrentPortal = nullptr;
 		int CurrentPortalUniq = 0;
 		bool CurrentPortalInSkybox = false;
diff --git a/src/scripting/thingdef_data.cpp b/src/scripting/thingdef_data.cpp
index 0f7abaac6..a10d6e449 100644
--- a/src/scripting/thingdef_data.cpp
+++ b/src/scripting/thingdef_data.cpp
@@ -368,6 +368,8 @@ static FFlagDef ActorFlagDefs[]=
 	DEFINE_FLAG(RF, ZDOOMTRANS, AActor, renderflags),
 	DEFINE_FLAG(RF, CASTSPRITESHADOW, AActor, renderflags),
 	DEFINE_FLAG(RF, NOSPRITESHADOW, AActor, renderflags),
+	DEFINE_FLAG(RF2, INVISIBLEINMIRRORS, AActor, renderflags2),
+	DEFINE_FLAG(RF2, ONLYVISIBLEINMIRRORS, AActor, renderflags2),
 
 	// Bounce flags
 	DEFINE_FLAG2(BOUNCE_Walls, BOUNCEONWALLS, AActor, BounceFlags),