diff --git a/source/blood/src/view.cpp b/source/blood/src/view.cpp
index 86f734fa5..9ce0f2444 100644
--- a/source/blood/src/view.cpp
+++ b/source/blood/src/view.cpp
@@ -3673,9 +3673,6 @@ void viewLoadingScreen(int nTile, const char *pText, const char *pText2, const c
     viewLoadingScreenUpdate(NULL, -1);
 }
 
-#define COLOR_RED redcol
-#define COLOR_WHITE whitecol
-
 #define LOW_FPS 60
 #define SLOW_FRAME_TIME 20
 
@@ -3685,8 +3682,6 @@ void viewLoadingScreen(int nTile, const char *pText, const char *pText2, const c
 # define FPS_YOFFSET 0
 #endif
 
-#define FPS_COLOR(x) ((x) ? COLOR_RED : COLOR_WHITE)
-
 FString GameInterface::statFPS(void)
 {
 	FString output;
@@ -3773,8 +3768,6 @@ FString GameInterface::GetCoordString()
 }
 
 
-#undef FPS_COLOR
-
 class ViewLoadSave : public LoadSave {
 public:
     void Load(void);
diff --git a/source/build/include/build.h b/source/build/include/build.h
index 4b8aca091..76db19d28 100644
--- a/source/build/include/build.h
+++ b/source/build/include/build.h
@@ -539,11 +539,9 @@ static inline int32_t BGetTime(void) { return (int32_t) totalclock; }
 EXTERN int32_t numframes, randomseed;
 EXTERN int16_t sintable[2048];
 
-EXTERN uint8_t palette[768];
 EXTERN int16_t numshades;
 EXTERN char *lookuptables[MAXPALOOKUPS];
 EXTERN uint8_t paletteloaded;
-EXTERN uint8_t whitecol, redcol, blackcol;
 
 EXTERN int32_t maxspritesonscreen;
 
diff --git a/source/build/include/palette.h b/source/build/include/palette.h
index 3254f7f13..94c4e56ac 100644
--- a/source/build/include/palette.h
+++ b/source/build/include/palette.h
@@ -81,15 +81,10 @@ void videoFadePalette(uint8_t r, uint8_t g, uint8_t b, uint8_t offset);
 void videoTintBlood(int32_t r, int32_t g, int32_t b);
 #endif
 
-extern int32_t realmaxshade;
-extern float frealmaxshade;
-
 extern int32_t globalpal;
 extern int32_t globalblend;
-extern uint32_t g_lastpalettesum;
 extern void paletteLoadFromDisk(void);
 extern void palettePostLoadTables(void);
-extern uint8_t basepalreset;
 extern int32_t curbrightness;
 
 extern int32_t paletteLoadLookupTable(FileReader &fp);
diff --git a/source/build/src/defs.cpp b/source/build/src/defs.cpp
index 08dfdb225..6b1c14a0b 100644
--- a/source/build/src/defs.cpp
+++ b/source/build/src/defs.cpp
@@ -500,8 +500,8 @@ static int32_t defsparser(scriptfile *script)
         }
         break;
         case T_SHADEFACTOR:
-            scriptfile_getnumber(script, &realmaxshade);
-            frealmaxshade = (float)realmaxshade;
+            //scriptfile_getnumber(script, &realmaxshade);
+            //frealmaxshade = (float)realmaxshade;
             break;
         case T_ARTFILE:
         {
diff --git a/source/build/src/engine.cpp b/source/build/src/engine.cpp
index f8f79d54c..c6c1e409d 100644
--- a/source/build/src/engine.cpp
+++ b/source/build/src/engine.cpp
@@ -3717,8 +3717,6 @@ int32_t videoSetGameMode(char davidoption, int32_t daupscaledxdim, int32_t daups
 
     j = bpp;
 
-    g_lastpalettesum = 0;
-
     rendmode = REND_POLYMOST;
 
     upscalefactor = 1;
@@ -5495,8 +5493,6 @@ int32_t videoSetRenderMode(int32_t renderer)
 
     renderer = REND_POLYMOST;
 
-    basepalreset = 1;
-
     rendmode = renderer;
     if (videoGetRenderMode() >= REND_POLYMOST)
         glrendmode = rendmode;
diff --git a/source/build/src/palette.cpp b/source/build/src/palette.cpp
index b0a2b6f62..d0ac4b251 100644
--- a/source/build/src/palette.cpp
+++ b/source/build/src/palette.cpp
@@ -17,22 +17,19 @@
 #include "common.h"
 #include "memarena.h"
 #include "palettecontainer.h"
+#include "palutil.h"
+#include "colormatcher.h"
 #include "../../glbackend/glbackend.h"
 
 FMemArena lookuparena;
 
-uint8_t basepalreset=1;
 uint8_t curbasepal;
 int32_t globalblend;
 
-uint32_t g_lastpalettesum = 0;
 palette_t palfadergb = { 0, 0, 0, 0 };
 unsigned char palfadedelta = 0;
 ESetPalFlags curpaletteflags;
 
-int32_t realmaxshade;
-float frealmaxshade;
-
 #if defined(USE_OPENGL)
 palette_t palookupfog[MAXPALOOKUPS];
 #endif
@@ -135,6 +132,7 @@ void paletteLoadFromDisk(void)
 
     // PALETTE_MAIN
 
+    uint8_t palette[768];
     if (768 != fil.Read(palette, 768))
         return;
 
@@ -255,55 +253,26 @@ void palettePostLoadTables(void)
 
     char const * const palookup0 = lookuptables[0];
 
-#ifdef DEBUG_TILESIZY_512
-    // Bump shade 1 by 16.
-    for (bssize_t i=256; i<512; i++)
-        palookup0[i] = palookup0[i+(16<<8)];
-#endif
-    PalEntry pe[256];
-    for (int i = 0; i < 256; i++) pe[i] = PalEntry(palette[i * 3], palette[i * 3 + 1], palette[i * 3 + 2]);
-    ImageHelpers::SetPalette(pe);
-    blackcol = ImageHelpers::BestColor(0, 0, 0);
-    whitecol = ImageHelpers::BestColor(255, 255, 255);
-    redcol = ImageHelpers::BestColor(255, 0, 0);
-
+    ImageHelpers::SetPalette(GPalette.BaseColors);
+    
     // Bmemset(PaletteIndexFullbrights, 0, sizeof(PaletteIndexFullbrights));
     for (bssize_t c = 0; c < 255; ++c) // skipping transparent color
     {
         uint8_t const index = palookup0[c];
-        rgb24_t const & color = *(rgb24_t *)&palette[index*3];
+        PalEntry color = GPalette.BaseColors[index];
 
         // don't consider #000000 fullbright
-        if (EDUKE32_PREDICT_FALSE(color.r == 0 && color.g == 0 && color.b == 0))
+        if (color.r == 0 && color.g == 0 && color.b == 0)
             continue;
 
         for (size_t s = c + 256, s_end = 256*numshades; s < s_end; s += 256)
-            if (EDUKE32_PREDICT_FALSE(palookup0[s] != index))
+            if (palookup0[s] != index)
                 goto PostLoad_NotFullbright;
 
-        Printf("%d is fullbright\n", c);
         SetPaletteIndexFullbright(c);
 
         PostLoad_NotFullbright: ;
     }
-
-    if (realmaxshade == 0)
-    {
-        uint8_t const * const blackcolor = &palette[blackcol*3];
-        size_t s;
-        for (s = numshades < 2 ? 0 : numshades-2; s > 0; --s)
-        {
-            for (size_t c = s*256, c_end = c+255; c < c_end; ++c) // skipping transparent color
-            {
-                uint8_t const index = palookup0[c];
-                uint8_t const * const color = &palette[index*3];
-                if (!IsPaletteIndexFullbright(index) && memcmp(blackcolor, color, sizeof(rgb24_t)))
-                    goto PostLoad_FoundShade;
-            }
-        }
-        PostLoad_FoundShade: ;
-        frealmaxshade = (float)(realmaxshade = s+1);
-    }
 }
 
 void paletteFixTranslucencyMask(void)
@@ -575,10 +544,10 @@ void paletteMakeLookupTable(int32_t palnum, const char *remapbuf, uint8_t r, uin
 
             for (j=0; j<256; j++)
             {
-                const char *ptr = (const char *) &palette[remapbuf[j]*3];
-                *ptr2++ = ImageHelpers::BestColor(ptr[0] + mulscale16(r-ptr[0], palscale),
-                    ptr[1] + mulscale16(g-ptr[1], palscale),
-                    ptr[2] + mulscale16(b-ptr[2], palscale));
+                PalEntry pe = GPalette.BaseColors[remapbuf[j]];
+                *ptr2++ = ColorMatcher.Pick(pe.r + mulscale16(r-pe.r, palscale),
+                    pe.g + mulscale16(g-pe.g, palscale),
+                    pe.b + mulscale16(b-pe.b, palscale));
             }
         }
     }
@@ -606,7 +575,6 @@ void videoSetPalette(int dabrightness, int dapalid, ESetPalFlags flags)
 	if (GPalette.GetTranslation(Translation_BasePalettes, dapalid) == nullptr)
 		dapalid = 0;
 	curbasepal = dapalid;
-	basepalreset = 0;
 
 	// In-scene brightness mode for RR's thunderstorm. This shouldn't affect the global gamma ramp.
 	if ((videoGetRenderMode() >= REND_POLYMOST) && (flags & Pal_SceneBrightness))
diff --git a/source/common/engine/palettecontainer.cpp b/source/common/engine/palettecontainer.cpp
index e6408197a..3103e7775 100644
--- a/source/common/engine/palettecontainer.cpp
+++ b/source/common/engine/palettecontainer.cpp
@@ -58,6 +58,7 @@ void PaletteContainer::Init(int numslots)	// This cannot be a constructor!!!
 	remap.Inactive = true;
 	AddRemap(&remap);
 	TranslationTables.Resize(numslots);
+	ColorMatcher.SetPalette(BaseColors);
 }
 
 void PaletteContainer::SetPalette(const uint8_t* colors, int transparent_index)
diff --git a/source/common/utility/colormatcher.h b/source/common/utility/colormatcher.h
index 9d0826ea4..e17db4b5b 100644
--- a/source/common/utility/colormatcher.h
+++ b/source/common/utility/colormatcher.h
@@ -51,6 +51,7 @@ public:
 	FColorMatcher (const uint32_t *palette) { Pal = reinterpret_cast<const PalEntry*>(palette); }
 	FColorMatcher (const FColorMatcher &other) = default;
 
+	void SetPalette(PalEntry* palette) { Pal = palette; }
 	void SetPalette (const uint32_t *palette) { Pal = reinterpret_cast<const PalEntry*>(palette); }
 	uint8_t Pick (int r, int g, int b)
 	{
diff --git a/source/core/2d/v_2ddrawer.cpp b/source/core/2d/v_2ddrawer.cpp
index 0fc4b62e8..8f66118dc 100644
--- a/source/core/2d/v_2ddrawer.cpp
+++ b/source/core/2d/v_2ddrawer.cpp
@@ -33,6 +33,7 @@
 #include "vectors.h"
 #include "gamecvars.h"
 #include "earcut.hpp"
+#include "palettecontainer.h"
 //#include "doomtype.h"
 #include "templates.h"
 //#include "r_utility.h"
@@ -849,8 +850,7 @@ void drawlinergb(int32_t x1, int32_t y1, int32_t x2, int32_t y2, palette_t p)
 
 void renderDrawLine(int32_t x1, int32_t y1, int32_t x2, int32_t y2, uint8_t col)
 {
-	PalEntry color(255, palette[col * 3], palette[col * 3 + 1], palette[col * 3 + 2]);
-	drawlinergb(x1, y1, x2, y2, color);
+	drawlinergb(x1, y1, x2, y2, GPalette.BaseColors[GPalette.Remap[col]]);
 }
 
 
diff --git a/source/core/fonts/font.cpp b/source/core/fonts/font.cpp
index 5f64a4bca..672afbb81 100644
--- a/source/core/fonts/font.cpp
+++ b/source/core/fonts/font.cpp
@@ -53,6 +53,7 @@
 #include "fontchars.h"
 #include "imagehelpers.h"
 #include "glbackend/glbackend.h"
+#include "palettecontainer.h"
 
 #include "fontinternals.h"
 
@@ -229,8 +230,8 @@ void FFont::SetDefaultTranslation(uint32_t *othercolors)
 		{
 			if (myluminosity[l] >= otherluminosity[o] && myluminosity[l] <= otherluminosity[o+1])
 			{
-				PalEntry color1 = ImageHelpers::BasePalette[otherreverse[o]];
-				PalEntry color2 = ImageHelpers::BasePalette[otherreverse[o+1]];
+				PalEntry color1 = GPalette.BaseColors[otherreverse[o]];
+				PalEntry color2 = GPalette.BaseColors[otherreverse[o+1]];
 				double weight = 0;
 				if (otherluminosity[o] != otherluminosity[o + 1])
 				{
@@ -263,12 +264,12 @@ void FFont::SetDefaultTranslation(uint32_t *othercolors)
 
 static int compare (const void *arg1, const void *arg2)
 {
-	if (RPART(ImageHelpers::BasePalette[*((uint8_t *)arg1)]) * 299 +
-		GPART(ImageHelpers::BasePalette[*((uint8_t *)arg1)]) * 587 +
-		BPART(ImageHelpers::BasePalette[*((uint8_t *)arg1)]) * 114  <
-		RPART(ImageHelpers::BasePalette[*((uint8_t *)arg2)]) * 299 +
-		GPART(ImageHelpers::BasePalette[*((uint8_t *)arg2)]) * 587 +
-		BPART(ImageHelpers::BasePalette[*((uint8_t *)arg2)]) * 114)
+	if (RPART(GPalette.BaseColors[*((uint8_t *)arg1)]) * 299 +
+		GPART(GPalette.BaseColors[*((uint8_t *)arg1)]) * 587 +
+		BPART(GPalette.BaseColors[*((uint8_t *)arg1)]) * 114  <
+		RPART(GPalette.BaseColors[*((uint8_t *)arg2)]) * 299 +
+		GPART(GPalette.BaseColors[*((uint8_t *)arg2)]) * 587 +
+		BPART(GPalette.BaseColors[*((uint8_t *)arg2)]) * 114)
 		return -1;
 	else
 		return 1;
@@ -319,9 +320,9 @@ int FFont::SimpleTranslation (uint32_t *colorsused, uint8_t *translation, uint8_
 	{
 		translation[reverse[i]] = i;
 
-		Luminosity[i] = RPART(ImageHelpers::BasePalette[reverse[i]]) * 0.299 +
-						   GPART(ImageHelpers::BasePalette[reverse[i]]) * 0.587 +
-						   BPART(ImageHelpers::BasePalette[reverse[i]]) * 0.114;
+		Luminosity[i] = RPART(GPalette.BaseColors[reverse[i]]) * 0.299 +
+						   GPART(GPalette.BaseColors[reverse[i]]) * 0.587 +
+						   BPART(GPalette.BaseColors[reverse[i]]) * 0.114;
 		if (Luminosity[i] > max)
 			max = Luminosity[i];
 		if (Luminosity[i] < min)
@@ -372,10 +373,10 @@ void FFont::BuildTranslations (const double *luminosity, const uint8_t *identity
 				}
 				else
 				{
-					remap.Palette[0] = ImageHelpers::BasePalette[identity[0]] & MAKEARGB(0,255,255,255);
+					remap.Palette[0] = GPalette.BaseColors[identity[0]] & MAKEARGB(0,255,255,255);
 					for (j = 1; j < ActiveColors; ++j)
 					{
-						remap.Palette[j] = ImageHelpers::BasePalette[identity[j]] | MAKEARGB(255,0,0,0);
+						remap.Palette[j] = GPalette.BaseColors[identity[j]] | MAKEARGB(255,0,0,0);
 					}
 				}
 			}
diff --git a/source/core/fonts/hexfont.cpp b/source/core/fonts/hexfont.cpp
index 34dda32b2..5916ca207 100644
--- a/source/core/fonts/hexfont.cpp
+++ b/source/core/fonts/hexfont.cpp
@@ -40,6 +40,7 @@
 #include "imagehelpers.h"
 #include "v_draw.h"
 #include "glbackend/glbackend.h"
+#include "palettecontainer.h"
 
 #include "fontinternals.h"
 
@@ -386,8 +387,8 @@ public:
 			{
 				if (myluminosity[l] >= otherluminosity[o] && myluminosity[l] <= otherluminosity[o + 1])
 				{
-					PalEntry color1 = ImageHelpers::BasePalette[otherreverse[o]];
-					PalEntry color2 = ImageHelpers::BasePalette[otherreverse[o + 1]];
+					PalEntry color1 = GPalette.BaseColors[otherreverse[o]];
+					PalEntry color2 = GPalette.BaseColors[otherreverse[o + 1]];
 					double weight = 0;
 					if (otherluminosity[o] != otherluminosity[o + 1])
 					{
diff --git a/source/core/fonts/singlelumpfont.cpp b/source/core/fonts/singlelumpfont.cpp
index 66c48de75..e02000732 100644
--- a/source/core/fonts/singlelumpfont.cpp
+++ b/source/core/fonts/singlelumpfont.cpp
@@ -41,6 +41,7 @@
 #include "printf.h"
 #include "imagehelpers.h"
 #include "filesystem.h"
+#include "colormatcher.h"
 
 #include "fontinternals.h"
 
@@ -598,7 +599,7 @@ void FSingleLumpFont::FixupPalette (uint8_t *identity, double *luminosity, const
 		int g = palette[1];
 		int b = palette[2];
 		double lum = r*0.299 + g*0.587 + b*0.114;
-		identity[i] = ImageHelpers::BestColor(r, g, b);
+		identity[i] = ColorMatcher.Pick(r, g, b);
 		luminosity[i] = lum;
 		out_palette[i].r = r;
 		out_palette[i].g = g;
diff --git a/source/core/fonts/v_font.h b/source/core/fonts/v_font.h
index 0c35c698a..21264a266 100644
--- a/source/core/fonts/v_font.h
+++ b/source/core/fonts/v_font.h
@@ -190,11 +190,4 @@ FFont *V_GetFont(const char *fontname, const char *fontlumpname = nullptr);
 void V_InitFontColors();
 char* CleanseString(char* str);
 
-
-struct FRemapTable
-{
-	PalEntry Palette[256] = { };			// The ideal palette this maps to
-};
-
-
 #endif //__V_FONT_H__
diff --git a/source/core/rendering/hwrenderer/postprocessing/hw_postprocess.cpp b/source/core/rendering/hwrenderer/postprocessing/hw_postprocess.cpp
index c8ad01257..b6e0a2092 100644
--- a/source/core/rendering/hwrenderer/postprocessing/hw_postprocess.cpp
+++ b/source/core/rendering/hwrenderer/postprocessing/hw_postprocess.cpp
@@ -27,6 +27,7 @@
 #include "hwrenderer/utility/hw_cvars.h"
 #include "hwrenderer/postprocessing/hw_postprocess_cvars.h"
 #include "palutil.h"
+#include "palettecontainer.h"
 #include <random>
 
 Postprocess hw_postprocess;
@@ -543,7 +544,7 @@ void PPTonemap::UpdateTextures()
 			{
 				for (int b = 0; b < 64; b++)
 				{
-					PalEntry color = ImageHelpers::BasePalette[(uint8_t)ImageHelpers::PTM_BestColor((r << 2) | (r >> 4), (g << 2) | (g >> 4), (b << 2) | (b >> 4),
+					PalEntry color = GPalette.BaseColors[(uint8_t)PTM_BestColor((uint32_t*)GPalette.BaseColors, (r << 2) | (r >> 4), (g << 2) | (g >> 4), (b << 2) | (b >> 4),
 						gl_paltonemap_reverselookup, gl_paltonemap_powtable, 0, 256)];
 					int index = ((r * 64 + g) * 64 + b) * 4;
 					lut[index] = color.r;
diff --git a/source/core/textures/formats/arttexture.cpp b/source/core/textures/formats/arttexture.cpp
index a7c98abce..e84aeac76 100644
--- a/source/core/textures/formats/arttexture.cpp
+++ b/source/core/textures/formats/arttexture.cpp
@@ -37,6 +37,7 @@
 #include "templates.h"
 #include "bitmap.h"
 #include "image.h"
+#include "palettecontainer.h"
 
 #include "build.h"
 
@@ -122,6 +123,11 @@ void FArtTexture::CreatePalettedPixels(uint8_t* buffer)
 	if (!fr.isOpen()) return;
 	int numpixels = Width * Height;
 	fr.Read(buffer, numpixels);
+	auto remap = GPalette.Remap;
+	for (int i = 0; i < numpixels; i++)
+	{
+		buffer[i] = remap[buffer[i]];
+	}
 }
 
 //===========================================================================
@@ -145,18 +151,20 @@ int FArtTexture::CopyPixels(FBitmap *bmp, int conversion)
 	fr.Read(source.Data(), numpixels);
 	auto dest = bmp->GetPixels();
 	
+	auto remap = GPalette.Remap;
+	auto pal = GPalette.BaseColors;
     for (int y = 0; y < numpixels; ++y)
     {
-		int index = source[y];
+		int index = remap[source[y]];
         if (index == TRANSPARENT_INDEX)
 		{
 			hasalpha = true;
             continue;
 		}
 		
-		dest[0] = palette[index];
-		dest[1] = palette[index+1];
-		dest[2] = palette[index+2];
+		dest[0] = pal[index].b;
+		dest[1] = pal[index].g;
+		dest[2] = pal[index].r;
 		dest[3] = 255;
 		dest += 4;
     }
diff --git a/source/core/textures/formats/pngtexture.cpp b/source/core/textures/formats/pngtexture.cpp
index 7c18f0424..2aaa14b7b 100644
--- a/source/core/textures/formats/pngtexture.cpp
+++ b/source/core/textures/formats/pngtexture.cpp
@@ -42,6 +42,7 @@
 #include "image.h"
 #include "printf.h"
 #include "filesystem.h"
+#include "colormatcher.h"
 
 //==========================================================================
 //
@@ -265,7 +266,7 @@ FPNGTexture::FPNGTexture (FileReader &lump, int width, int height,
 			}
 			else
 			{
-				PaletteMap[i] = ImageHelpers::BestColor(p.palette[i].r, p.palette[i].g, p.palette[i].b);
+				PaletteMap[i] = ColorMatcher.Pick(p.palette[i].r, p.palette[i].g, p.palette[i].b);
 			}
 		}
 		break;
diff --git a/source/core/textures/imagehelpers.cpp b/source/core/textures/imagehelpers.cpp
index 56eae26d3..067fc8f82 100644
--- a/source/core/textures/imagehelpers.cpp
+++ b/source/core/textures/imagehelpers.cpp
@@ -36,88 +36,22 @@
  */
  
  #include "imagehelpers.h"
+#include "palettecontainer.h"
+#include "colormatcher.h"
 
 namespace ImageHelpers
 {
 	uint8_t GrayMap[256];
-	int WhiteIndex, BlackIndex;
 	int alphaThreshold;
 	ColorTable256k RGB256k;
-	_BasePalette BasePalette;
 
-	int BestColor(int r, int g, int b, int first, int num)
-	{
-		int bestcolor = first;
-		int bestdist = 257 * 257 + 257 * 257 + 257 * 257;
-
-		for (int color = first; color < num; color++)
-		{
-			int x = r - palette[color * 3 + 0];
-			int y = g - palette[color * 3 + 1];
-			int z = b - palette[color * 3 + 2];
-			int dist = x * x + y * y + z * z;
-			if (dist < bestdist)
-			{
-				if (dist == 0)
-					return color;
-
-				bestdist = dist;
-				bestcolor = color;
-			}
-		}
-		return bestcolor;
-	}
-
-	// [SP] Re-implemented BestColor for more precision rather than speed. This function is only ever called once until the game palette is changed.
-
-	int PTM_BestColor(int r, int g, int b, bool reverselookup, float powtable_val, int first, int num)
-	{
-		static double powtable[256];
-		static bool firstTime = true;
-		static float trackpowtable = 0.;
-
-		double fbestdist = DBL_MAX, fdist;
-		int bestcolor = 0;
-
-		if (firstTime || trackpowtable != powtable_val)
-		{
-			auto pt = powtable_val;
-			trackpowtable = pt;
-			firstTime = false;
-			for (int x = 0; x < 256; x++) powtable[x] = pow((double)x / 255, (double)pt);
-		}
-
-		for (int color = first; color < num; color++)
-		{
-			double x = powtable[abs(r - palette[color * 3 + 0])];
-			double y = powtable[abs(g - palette[color * 3 + 1])];
-			double z = powtable[abs(b - palette[color * 3 + 2])];
-			fdist = x + y + z;
-			if (color == first || (reverselookup ? (fdist <= fbestdist) : (fdist < fbestdist)))
-			{
-				if (fdist == 0 && !reverselookup)
-					return color;
-
-				fbestdist = fdist;
-				bestcolor = color;
-			}
-		}
-		return bestcolor;
-	}
-	
 	void SetPalette(const PalEntry* colors)
 	{
-		// Find white and black from the original palette so that they can be
-		// used to make an educated guess of the translucency % for a
-		// translucency map.
-		WhiteIndex = BestColor(255, 255, 255);
-		BlackIndex = BestColor(0, 0, 0);
-
 		// create the RGB666 lookup table
 		for (int r = 0; r < 64; r++)
 			for (int g = 0; g < 64; g++)
 				for (int b = 0; b < 64; b++)
-					RGB256k.RGB[r][g][b] = BestColor((r<<2)|(r>>4), (g<<2)|(g>>4), (b<<2)|(b>>4));
+					RGB256k.RGB[r][g][b] = ColorMatcher.Pick((r<<2)|(r>>4), (g<<2)|(g>>4), (b<<2)|(b>>4));
 	}
 	
 	
diff --git a/source/core/textures/imagehelpers.h b/source/core/textures/imagehelpers.h
index fa5206662..c5f6b5f2e 100644
--- a/source/core/textures/imagehelpers.h
+++ b/source/core/textures/imagehelpers.h
@@ -41,9 +41,8 @@
 #include "tarray.h"
 #include "palentry.h"
 #include "bitmap.h"
-
-// we do not want to pull in the entirety of build.h here.
-extern uint8_t palette[768];
+#include "palutil.h"
+#include "palettecontainer.h"
 
 namespace ImageHelpers
 {
@@ -59,8 +58,6 @@ namespace ImageHelpers
 	extern int alphaThreshold;
 
 	// Todo: This should not pick fullbright colors.
-	int BestColor(int r, int g, int b, int first = 0, int num = 255);
-	int PTM_BestColor(int r, int g, int b, bool reverselookup, float powtable_val, int first, int num);
 	void SetPalette(const PalEntry* colors);
 
 
@@ -72,7 +69,7 @@ namespace ImageHelpers
 	
 	inline uint8_t RGBToPalettePrecise(bool wantluminance, int r, int g, int b, int a = 255)
 	{
-		return BestColor(r, g, b);
+		return BestColor((uint32_t*)GPalette.BaseColors, r, g, b);
 	}
 	
 	inline uint8_t RGBToPalette(bool wantluminance, int r, int g, int b, int a = 255)
@@ -143,18 +140,4 @@ namespace ImageHelpers
 			}
 		}
 	}
-	
-	struct _BasePalette
-	{
-		PalEntry operator[](int index)
-		{
-			return PalEntry(
-				palette[index * 3 + 0],
-				palette[index * 3 + 1],
-				palette[index * 3 + 2]
-			);
-		}
-	};
-	
-	extern _BasePalette BasePalette;
 }
diff --git a/source/duke3d/src/gameexec.cpp b/source/duke3d/src/gameexec.cpp
index 75a887cb3..dbc9b07e5 100644
--- a/source/duke3d/src/gameexec.cpp
+++ b/source/duke3d/src/gameexec.cpp
@@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "mapinfo.h"
 #include "version.h"
 #include "v_video.h"
+#include "colormatcher.h"
 
 #include "debugbreak.h"
 
@@ -5835,7 +5836,8 @@ badindex:
                 {
                     tw = *insptr++;
                     int32_t const rgb = Gv_GetVar(*insptr++);
-                    Gv_SetVar(tw, ImageHelpers::BestColor(rgb & 0xFF, (rgb >> 8) & 0xFF, (rgb >> 16) & 0xFF, Gv_GetVar(*insptr++)));
+                    Gv_GetVar(*insptr++);
+                    Gv_SetVar(tw, ColorMatcher.Pick(rgb & 0xFF, (rgb >> 8) & 0xFF, (rgb >> 16) & 0xFF));
                 }
                 dispatch();
 
diff --git a/source/duke3d/src/screens.cpp b/source/duke3d/src/screens.cpp
index cdfc482a0..0314ebd68 100644
--- a/source/duke3d/src/screens.cpp
+++ b/source/duke3d/src/screens.cpp
@@ -37,9 +37,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 BEGIN_DUKE_NS
 
-#define COLOR_RED redcol
-#define COLOR_WHITE whitecol
-
 #define quotepulseshade (sintable[((uint32_t)totalclock<<5)&2047]>>11)
 
 int32_t g_crosshairSum = -1;
@@ -580,8 +577,6 @@ FString GameInterface::GetCoordString()
 # define FPS_YOFFSET 0
 #endif
 
-#define FPS_COLOR(x) ((x) ? COLOR_RED : COLOR_WHITE)
-
 FString GameInterface::statFPS(void)
 {
 	FString output;
@@ -654,8 +649,6 @@ FString GameInterface::statFPS(void)
 	return output;
 }
 
-#undef FPS_COLOR
-
 void G_DisplayRest(int32_t smoothratio)
 {
     int32_t i, j;
diff --git a/source/exhumed/src/engine.h b/source/exhumed/src/engine.h
index 1c3d804b0..6d53ec6ae 100644
--- a/source/exhumed/src/engine.h
+++ b/source/exhumed/src/engine.h
@@ -60,7 +60,6 @@ void overwritesprite(int thex, int they, short tilenum, signed char shade, char
 void precache();
 void resettiming();
 void printext(int x, int y, const char* buffer, short tilenum, char invisiblecol);
-void kensetpalette(unsigned char *vgapal);
 
 END_PS_NS
 
diff --git a/source/exhumed/src/enginesubs.cpp b/source/exhumed/src/enginesubs.cpp
index 4dccdd88e..479f5483d 100644
--- a/source/exhumed/src/enginesubs.cpp
+++ b/source/exhumed/src/enginesubs.cpp
@@ -71,30 +71,6 @@ void resettiming()
 // TODO	totalclocklock = 0L;
 }
 
-void kensetpalette(unsigned char *vgapal)
-{
-    //setbrightness(0, (char*)vgapal, 4 | 2);
-    // TODO
-    Bmemcpy(palette, vgapal, 768);
-    for (auto &i : palette)
-        i <<= 2;
-    videoSetPalette(0, 0, 0);
-#if 0
-    char vesapal[1024];
-
-    for(int i = 0; i < 256; i++)
-    {
-        vesapal[i*4+0] = vgapal[i*3+2];
-        vesapal[i*4+1] = vgapal[i*3+1];
-        vesapal[i*4+2] = vgapal[i*3+0];
-        vesapal[i*4+3] = 0;
-    }
-#ifndef __WATCOMC__
-     (0L, 256L, vesapal);
-#endif
-
-#endif
-}
 
 static int32_t xdim_to_320_16(int32_t x)
 {
diff --git a/source/exhumed/src/exhumed.cpp b/source/exhumed/src/exhumed.cpp
index fc27d0b52..3f3f55005 100644
--- a/source/exhumed/src/exhumed.cpp
+++ b/source/exhumed/src/exhumed.cpp
@@ -1469,9 +1469,6 @@ static inline int32_t calc_smoothratio(ClockTicks totalclk, ClockTicks ototalclk
     return CalcSmoothRatio(totalclk, ototalclk, 30);
 }
 
-#define COLOR_RED redcol
-#define COLOR_WHITE whitecol
-
 #define LOW_FPS ((videoGetRenderMode() == REND_CLASSIC) ? 35 : 50)
 #define SLOW_FRAME_TIME 20
 
@@ -1481,8 +1478,6 @@ static inline int32_t calc_smoothratio(ClockTicks totalclk, ClockTicks ototalclk
 # define FPS_YOFFSET 0
 #endif
 
-#define FPS_COLOR(x) ((x) ? COLOR_RED : COLOR_WHITE)
-
 FString GameInterface::statFPS()
 {
     FString out;
diff --git a/source/exhumed/src/light.cpp b/source/exhumed/src/light.cpp
index 7ac62a6a6..a3de41440 100644
--- a/source/exhumed/src/light.cpp
+++ b/source/exhumed/src/light.cpp
@@ -181,7 +181,6 @@ void GrabPalette()
 
 void BlackOut()
 {
-    g_lastpalettesum = -1;
     videoTintBlood(0, 0, 0);
 }
 
@@ -290,15 +289,7 @@ void FixPalette()
         nPalDiff = 0;
     }
 
-#ifdef USE_OPENGL
-    if (videoGetRenderMode() >= REND_POLYMOST) videoTintBlood(rtint, gtint, btint);
-    else
-#endif
-    {
-        
-        //videoUpdatePalette(0, 256);
-        g_lastpalettesum = -1;
-    }
+    videoTintBlood(rtint, gtint, btint);
 }
 
 void TintPalette(int r, int g, int b)
diff --git a/source/rr/src/common.cpp b/source/rr/src/common.cpp
index 213f8949b..863d293ed 100644
--- a/source/rr/src/common.cpp
+++ b/source/rr/src/common.cpp
@@ -11,7 +11,7 @@
 #include "gamecvars.h"
 #include "rts.h"
 #include "gamecontrol.h"
-
+#include "palettecontainer.h"
 
 #include "common.h"
 #include "common_game.h"
@@ -135,8 +135,13 @@ void G_LoadLookups(void)
         paletteSetColorTable(basepalnum, paldata, basepalnum == DREALMSPAL || basepalnum == ENDINGPAL);
     }
 
-    Bmemcpy(paldata, palette+1, 767);
-    paldata[767] = palette[767];
+    for (int i = 0; i < 256; i++)
+    {
+        // swap red and blue channels.
+        paldata[i * 3] = GPalette.BaseColors[i].b;
+        paldata[i * 3+1] = GPalette.BaseColors[i].g;
+        paldata[i * 3+2] = GPalette.BaseColors[i].r;
+    }
     paletteSetColorTable(DRUGPAL, paldata); // todo: implement this as a shader effect
 
     if (RR)
diff --git a/source/rr/src/screens.cpp b/source/rr/src/screens.cpp
index cb214db89..1b60ac870 100644
--- a/source/rr/src/screens.cpp
+++ b/source/rr/src/screens.cpp
@@ -37,9 +37,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 BEGIN_RR_NS
 
-#define COLOR_RED redcol
-#define COLOR_WHITE whitecol
-
 #define quotepulseshade (sintable[((uint32_t)totalclock<<5)&2047]>>11)
 
 int32_t g_crosshairSum = -1;
@@ -650,7 +647,6 @@ GameStats GameInterface::getStats()
 	return { p->actors_killed, p->max_actors_killed, p->secret_rooms, p->max_secret_rooms, p->player_par / REALGAMETICSPERSEC, p->frag };
 }
 
-#undef FPS_COLOR
 
 void G_DisplayRest(int32_t smoothratio)
 {
diff --git a/source/sw/src/menus.cpp b/source/sw/src/menus.cpp
index 55faaf194..f28051ed5 100644
--- a/source/sw/src/menus.cpp
+++ b/source/sw/src/menus.cpp
@@ -55,6 +55,7 @@ Prepared for public release: 03/28/2005 - Charlie Wiederhold, 3D Realms
 
 #include "colormap.h"
 #include "config.h"
+#include "palettecontainer.h"
 
 BEGIN_SW_NS
 
@@ -460,9 +461,9 @@ void SetFadeAmt(PLAYERp pp, short damage, unsigned char startcolor)
 
     pp->FadeTics = 0;
 
-    color.red = palette[pp->StartColor*3];
-    color.green = palette[pp->StartColor*3+1];
-    color.blue = palette[pp->StartColor*3+2];
+    color.red = GPalette.BaseColors[pp->StartColor].r;
+    color.green = GPalette.BaseColors[pp->StartColor].g;
+    color.blue = GPalette.BaseColors[pp->StartColor].b;
 
     // Do initial palette set
     if (pp == Player + screenpeek)
@@ -532,10 +533,10 @@ void DoPaletteFlash(PLAYERp pp)
         if (pp == Player + screenpeek)
         {
             videoFadePalette(
-                palette[pp->StartColor*3],
-                palette[pp->StartColor*3+1],
-                palette[pp->StartColor*3+2],
-				faderamp[min(31, max(0, 32 - abs(pp->FadeAmt)))]
+                GPalette.BaseColors[pp->StartColor].r,
+                GPalette.BaseColors[pp->StartColor].g,
+                GPalette.BaseColors[pp->StartColor].b,
+                faderamp[min(31, max(0, 32 - abs(pp->FadeAmt)))]
                 );
         }