diff --git a/src/polyrenderer/scene/poly_cull.cpp b/src/polyrenderer/scene/poly_cull.cpp
index a3361fd627..d53e0e1bfc 100644
--- a/src/polyrenderer/scene/poly_cull.cpp
+++ b/src/polyrenderer/scene/poly_cull.cpp
@@ -33,26 +33,15 @@ void PolyCull::CullScene(const PolyClipPlane &portalClipPlane)
 	ClearSolidSegments();
 	MarkViewFrustum();
 
-	if (level.LevelName != lastLevelName) // Is this the best way to detect a level change?
-	{
-		lastLevelName = level.LevelName;
-		SubsectorDepths.clear();
-		SubsectorDepths.resize(level.subsectors.Size(), 0xffffffff);
-		SectorSeen.clear();
-		SectorSeen.resize(level.sectors.Size());
-	}
-	else
-	{
-		for (const auto &sub : PvsSectors)
-			SubsectorDepths[sub->Index()] = 0xffffffff;
-		SubsectorDepths.resize(level.subsectors.Size(), 0xffffffff);
+	for (uint32_t sub : PvsSubsectors)
+		SubsectorDepths[sub] = 0xffffffff;
+	SubsectorDepths.resize(level.subsectors.Size(), 0xffffffff);
 
-		for (const auto &sector : SeenSectors)
-			SectorSeen[sector->Index()] = false;
-		SectorSeen.resize(level.sectors.Size());
-	}
+	for (uint32_t sector : SeenSectors)
+		SectorSeen[sector] = false;
+	SectorSeen.resize(level.sectors.Size());
 
-	PvsSectors.clear();
+	PvsSubsectors.clear();
 	SeenSectors.clear();
 
 	NextPvsLineStart = 0;
@@ -125,10 +114,10 @@ void PolyCull::CullSubsector(subsector_t *sub)
 		FirstSkyHeight = false;
 	}
 
-	uint32_t subsectorDepth = (uint32_t)PvsSectors.size();
+	uint32_t subsectorDepth = (uint32_t)PvsSubsectors.size();
 
 	// Mark that we need to render this
-	PvsSectors.push_back(sub);
+	PvsSubsectors.push_back(sub->Index());
 	PvsLineStart.push_back(NextPvsLineStart);
 
 	DVector3 viewpos = PolyRenderer::Instance()->Viewpoint.Pos;
@@ -169,7 +158,7 @@ void PolyCull::CullSubsector(subsector_t *sub)
 	if (!SectorSeen[sub->sector->Index()])
 	{
 		SectorSeen[sub->sector->Index()] = true;
-		SeenSectors.push_back(sub->sector);
+		SeenSectors.push_back(sub->sector->Index());
 	}
 
 	SubsectorDepths[sub->Index()] = subsectorDepth;
diff --git a/src/polyrenderer/scene/poly_cull.h b/src/polyrenderer/scene/poly_cull.h
index 474a33bb6c..4b851373e6 100644
--- a/src/polyrenderer/scene/poly_cull.h
+++ b/src/polyrenderer/scene/poly_cull.h
@@ -36,11 +36,11 @@ public:
 		return PvsLineVisible[PvsLineStart[subsectorDepth] + lineIndex];
 	}
 
-	std::vector<subsector_t *> PvsSectors;
+	std::vector<uint32_t> PvsSubsectors;
 	double MaxCeilingHeight = 0.0;
 	double MinFloorHeight = 0.0;
 
-	std::vector<sector_t *> SeenSectors;
+	std::vector<uint32_t> SeenSectors;
 	std::vector<bool> SectorSeen;
 	std::vector<uint32_t> SubsectorDepths;
 
diff --git a/src/polyrenderer/scene/poly_scene.cpp b/src/polyrenderer/scene/poly_scene.cpp
index 02ad1842ed..42f6e815b0 100644
--- a/src/polyrenderer/scene/poly_scene.cpp
+++ b/src/polyrenderer/scene/poly_scene.cpp
@@ -69,8 +69,8 @@ void RenderPolyScene::RenderSectors()
 {
 	PolyRenderThread *mainthread = PolyRenderer::Instance()->Threads.MainThread();
 
-	int totalcount = (int)Cull.PvsSectors.size();
-	auto subsectors = Cull.PvsSectors.data();
+	int totalcount = (int)Cull.PvsSubsectors.size();
+	uint32_t *subsectors = Cull.PvsSubsectors.data();
 
 	TranslucentObjects.resize(PolyRenderer::Instance()->Threads.NumThreads());
 
@@ -82,7 +82,7 @@ void RenderPolyScene::RenderSectors()
 		int end = thread->End;
 		for (int i = start; i < end; i++)
 		{
-			RenderSubsector(thread, subsectors[i], i);
+			RenderSubsector(thread, &level.subsectors[subsectors[i]], i);
 		}
 	}, [&](PolyRenderThread *thread)
 	{
@@ -372,8 +372,9 @@ void RenderPolyScene::RenderTranslucent(int portalDepth)
 	}
 
 	const auto &viewpoint = PolyRenderer::Instance()->Viewpoint;
-	for (sector_t *sector : Cull.SeenSectors)
+	for (uint32_t sectorIndex : Cull.SeenSectors)
 	{
+		sector_t *sector = &level.sectors[sectorIndex];
 		for (AActor *thing = sector->thinglist; thing != nullptr; thing = thing->snext)
 		{
 			DVector2 left, right;