diff --git a/src/polyrenderer/scene/poly_cull.cpp b/src/polyrenderer/scene/poly_cull.cpp
index c39eb66cc5..e759bdda43 100644
--- a/src/polyrenderer/scene/poly_cull.cpp
+++ b/src/polyrenderer/scene/poly_cull.cpp
@@ -33,13 +33,24 @@ void PolyCull::CullScene(const TriMatrix &worldToClip, const PolyClipPlane &port
 	ClearSolidSegments();
 	MarkViewFrustum();
 
-	for (const auto &sub : PvsSectors)
-		SubsectorDepths[sub->Index()] = 0xffffffff;
-	SubsectorDepths.resize(level.subsectors.Size(), 0xffffffff);
+	if (level.LevelName != lastLevelName) // Is this the best way to detect a level change?
+	{
+		lastLevelName = level.LevelName;
+		SubsectorDepths.clear();
+		SubsectorDepths.resize(level.subsectors.Size(), 0xffffffff);
+		SectorSeen.clear();
+		SectorSeen.resize(level.sectors.Size());
+	}
+	else
+	{
+		for (const auto &sub : PvsSectors)
+			SubsectorDepths[sub->Index()] = 0xffffffff;
+		SubsectorDepths.resize(level.subsectors.Size(), 0xffffffff);
 
-	for (const auto &sector : SeenSectors)
-		SectorSeen[sector->Index()] = false;
-	SectorSeen.resize(level.sectors.Size());
+		for (const auto &sector : SeenSectors)
+			SectorSeen[sector->Index()] = false;
+		SectorSeen.resize(level.sectors.Size());
+	}
 
 	PvsSectors.clear();
 	SeenSectors.clear();
diff --git a/src/polyrenderer/scene/poly_cull.h b/src/polyrenderer/scene/poly_cull.h
index 549f4c3753..fe80005012 100644
--- a/src/polyrenderer/scene/poly_cull.h
+++ b/src/polyrenderer/scene/poly_cull.h
@@ -70,6 +70,8 @@ private:
 
 	void MarkSegmentCulled(angle_t angle1, angle_t angle2);
 
+	FString lastLevelName;
+
 	std::vector<SolidSegment> SolidSegments;
 	std::vector<SolidSegment> TempInvertSolidSegments;
 	const int SolidCullScale = 3000;