diff --git a/src/common/utility/palette.cpp b/src/common/utility/palette.cpp
index 5d7be22728..e7bc61f098 100644
--- a/src/common/utility/palette.cpp
+++ b/src/common/utility/palette.cpp
@@ -418,29 +418,15 @@ void MakeRemap(uint32_t* BaseColors, const uint32_t* colors, uint8_t* remap, con
 // color, so find a duplicate pair of palette entries, make one of them a
 // duplicate of color 0, and remap every graphic so that it uses that entry
 // instead of entry 0.
-void MakeGoodRemap(uint32_t* BaseColors, uint8_t* Remap)
+void MakeGoodRemap(uint32_t* BaseColors, uint8_t* Remap, const uint8_t* lastcolormap)
 {
 	for (int i = 0; i < 256; i++) Remap[i] = i;
 	PalEntry color0 = BaseColors[0];
 	int i;
 
-	// we have to load the colormap ourselves, because at this stage the colormaps have not yet been loaded
-	int lump = fileSystem.CheckNumForFullName ("COLORMAP", true, ns_colormaps);
-	if (lump == -1)
-		lump = fileSystem.CheckNumForName ("COLORMAP", ns_global);
-
-	FileReader readlump;
-	TArray<uint8_t> maincolormap(8192);
-	uint8_t *lastcolormap = maincolormap.Data() + 7936;
-
-	if (lump != -1)
-	{
-		readlump = fileSystem.OpenFileReader (lump);
-		readlump.Read (maincolormap.Data(), 8192);
-	}
 
 	// First try for an exact match of color 0. Only Hexen does not have one.
-	if ((lump == -1) || !lastcolormap)
+	if (!lastcolormap)
 	{
 		for (i = 1; i < 256; ++i)
 		{
@@ -477,7 +463,7 @@ void MakeGoodRemap(uint32_t* BaseColors, uint8_t* Remap)
 			sortcopy[i] = (BaseColors[i] & 0xffffff) | (i << 24);
 		}
 		qsort(sortcopy, 256, 4, sortforremap);
-		if ((lump == -1) || !lastcolormap)
+		if (!lastcolormap)
 		{
 			for (i = 255; i > 0; --i)
 			{
diff --git a/src/common/utility/palutil.h b/src/common/utility/palutil.h
index bbd8f9a2d0..6faa7b5df3 100644
--- a/src/common/utility/palutil.h
+++ b/src/common/utility/palutil.h
@@ -14,7 +14,7 @@ void DoBlending(const PalEntry* from, PalEntry* to, int count, int r, int g, int
 // Given an array of colors, fills in remap with values to remap the
 // passed array of colors to BaseColors. Used for loading palette downconversions of PNGs.
 void MakeRemap(uint32_t* BaseColors, const uint32_t* colors, uint8_t* remap, const uint8_t* useful, int numcolors);
-void MakeGoodRemap(uint32_t* BaseColors, uint8_t* Remap);
+void MakeGoodRemap(uint32_t* BaseColors, uint8_t* Remap, const uint8_t* cmapdata = nullptr);
 
 // Colorspace conversion RGB <-> HSV
 void RGBtoHSV (float r, float g, float b, float *h, float *s, float *v);
diff --git a/src/r_data/v_palette.cpp b/src/r_data/v_palette.cpp
index aa3702bea3..333cb7579b 100644
--- a/src/r_data/v_palette.cpp
+++ b/src/r_data/v_palette.cpp
@@ -65,9 +65,11 @@ void InitPalette ()
 		FileData cmap = fileSystem.ReadFile(lump);
 		const unsigned char* cmapdata = (const unsigned char*)cmap.GetMem();
 		GPalette.GenerateGlobalBrightmapFromColormap(cmapdata, 32);
+		MakeGoodRemap((uint32_t*)GPalette.BaseColors, GPalette.Remap, cmapdata + 7936);	// last entry in colormap
 	}
+	else
+		MakeGoodRemap ((uint32_t*)GPalette.BaseColors, GPalette.Remap);
 
-	MakeGoodRemap ((uint32_t*)GPalette.BaseColors, GPalette.Remap);
 	ColorMatcher.SetPalette ((uint32_t *)GPalette.BaseColors);
 
 	if (GPalette.Remap[0] == 0)