Cache the optimized bitcode (note: this adds LLVMBitWriter as a dependency)

src/CMakeLists.txt

@@ -294,7 +294,7 @@ endif()
 set( LLVM_PRECOMPILED_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../llvm" )
 
 if( NOT WIN32 )
-	set( LLVM_COMPONENTS core support asmparser asmprinter bitreader codegen ipo
+	set( LLVM_COMPONENTS core support asmparser asmprinter bitreader bitwriter codegen ipo
 		irreader transformutils instrumentation profiledata runtimedyld
 		object instcombine linker analysis selectiondag scalaropts vectorize executionengine
 		mc mcdisassembler mcparser mcjit target x86asmprinter x86info x86desc x86utils x86codegen )
@@ -307,7 +307,7 @@ if( NOT WIN32 )
 	include_directories( ${LLVM_INCLUDE_DIRS} )
 	set( ZDOOM_LIBS ${ZDOOM_LIBS} ${llvm_libs} )
 else()
-	set( LLVM_COMPONENTS core support asmparser asmprinter bitreader codegen passes ipo
+	set( LLVM_COMPONENTS core support asmparser asmprinter bitreader bitwriter codegen passes ipo
 		irreader transformutils instrumentation profiledata debuginfocodeview runtimedyld
 		object instcombine linker analysis selectiondag scalaropts vectorize executionengine
 		mc mcdisassembler mcparser mcjit target x86asmprinter x86info x86desc x86utils x86codegen )

src/r_compiler/llvm_include.h

@@ -75,6 +75,7 @@
 #include <llvm/CodeGen/AsmPrinter.h>
 #include <llvm/MC/MCAsmInfo.h>
 #include <llvm/Target/TargetSubtargetInfo.h>
+#include <llvm/Bitcode/ReaderWriter.h>
 
 #if LLVM_VERSION_MAJOR < 3 || (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR < 8)
 #include <llvm/Support/FormattedStream.h>

src/r_compiler/llvmdrawers.cpp

@@ -39,6 +39,7 @@
 #include "x86.h"
 #include "c_cvars.h"
 #include "version.h"
+#include "m_misc.h"
 
 CUSTOM_CVAR(String, llvm_cpu, "auto", CVAR_ARCHIVE | CVAR_NOINITCALL)
 {
@@ -51,7 +52,9 @@ public:
 	LLVMProgram();
 
 	void CreateModule();
-	void CreateEE();
+	std::string GetTargetCPU();
+	bool LoadCachedModule(int version, std::string targetCPU);
+	void CreateEE(int version, std::string targetCPU, bool optimize);
 	std::string GenerateAssembly(std::string cpuName);
 	std::string DumpModule();
 	void StopLogFatalErrors();
@@ -64,6 +67,8 @@ public:
 	llvm::ExecutionEngine *engine() { return mEngine.get(); }
 
 private:
+	void SaveCachedModule(llvm::Module *module, int version, std::string targetCPU);
+	FString GetDrawerCacheFilename(int version, FString cpu);
 	void *PointerToFunction(const char *name);
 
 	llvm::TargetMachine *machine = nullptr;
@@ -121,6 +126,11 @@ LLVMDrawers *LLVMDrawers::Instance()
 /////////////////////////////////////////////////////////////////////////////
 
 LLVMDrawersImpl::LLVMDrawersImpl()
+{
+	int version = 1; // Increment this number if the drawer codegen is modified (forces recreation of the module).
+	std::string targetCPU = mProgram.GetTargetCPU();
+	bool loaded = mProgram.LoadCachedModule(version, targetCPU);
+	if (!loaded)
 	{
 		mProgram.CreateModule();
 
@@ -198,8 +208,9 @@ LLVMDrawersImpl::LLVMDrawersImpl()
 			CodegenDrawTriangle("TriFillSubsector32_" + std::to_string(i), TriDrawVariant::FillSubsector, (TriBlendMode)i, true);
 		}
 		CodegenDrawTriangle("TriStencil", TriDrawVariant::Stencil, TriBlendMode::Copy, false);
+	}
 
-	mProgram.CreateEE();
+	mProgram.CreateEE(version, targetCPU, !loaded);
 
 	FillColumn = mProgram.GetProcAddress<void(const DrawColumnArgs *, const WorkerThreadData *)>("FillColumn");
 	FillColumnAdd = mProgram.GetProcAddress<void(const DrawColumnArgs *, const WorkerThreadData *)>("FillColumnAdd");
@@ -603,12 +614,64 @@ void LLVMProgram::CreateModule()
 	mModule = std::make_unique<llvm::Module>("render", context());
 }
 
-void LLVMProgram::CreateEE()
+bool LLVMProgram::LoadCachedModule(int version, std::string targetCPU)
+{
+	FString filename = GetDrawerCacheFilename(version, targetCPU.c_str());
+	FILE *file = fopen(filename, "rb");
+	if (!file)
+		return false;
+
+	bool success = false;
+	std::string data;
+
+	fseek(file, 0, SEEK_END);
+	int length = ftell(file);
+	fseek(file, 0, SEEK_SET);
+	if (length > 0)
+	{
+		data.resize(length);
+		success = fread(&data[0], length, 1, file) == 1;
+	}
+
+	fclose(file);
+	if (!success)
+		return false;
+
+	auto result = llvm::parseBitcodeFile(llvm::MemoryBufferRef(data, filename.GetChars()), *mContext.get());
+	if (!result)
+		return false;
+
+	mModule = std::move(result.get());
+	return true;
+}
+
+void LLVMProgram::SaveCachedModule(llvm::Module *module, int version, std::string targetCPU)
+{
+	std::string str;
+	llvm::raw_string_ostream stream(str);
+	llvm::WriteBitcodeToFile(module, stream);
+	std::string data = stream.str();
+
+	FString filename = GetDrawerCacheFilename(version, targetCPU.c_str());
+	FILE *file = fopen(filename, "wb");
+	if (file)
+	{
+		fwrite(data.data(), data.size(), 1, file);
+		fclose(file);
+	}
+}
+
+FString LLVMProgram::GetDrawerCacheFilename(int version, FString cpu)
+{
+	FString path = M_GetCachePath(true);
+	FString filename;
+	filename.Format("%s/LLVMDrawers-%d-%s.bc", path.GetChars(), version, cpu.GetChars());
+	return filename;
+}
+
+std::string LLVMProgram::GetTargetCPU()
 {
 	using namespace llvm;
-
-	std::string errorstring;
-
 	std::string mcpu = sys::getHostCPUName();
 	if (std::string(CPU.CPUString).find("G840") != std::string::npos && mcpu == "sandybridge")
 		mcpu = "westmere"; // Pentium G840 is misdetected as a sandy bridge CPU
@@ -618,13 +681,21 @@ void LLVMProgram::CreateEE()
 		mcpu = llvm_cpu;
 		Printf("Overriding LLVM CPU target to %s\n", mcpu.c_str());
 	}
+	return mcpu;
+}
+
+void LLVMProgram::CreateEE(int version, std::string targetCPU, bool optimize)
+{
+	using namespace llvm;
+
+	std::string errorstring;
 
 	llvm::Module *module = mModule.get();
 	EngineBuilder engineBuilder(std::move(mModule));
 	engineBuilder.setErrorStr(&errorstring);
 	engineBuilder.setOptLevel(CodeGenOpt::Aggressive);
 	engineBuilder.setEngineKind(EngineKind::JIT);
-	engineBuilder.setMCPU(mcpu);
+	engineBuilder.setMCPU(targetCPU);
 	machine = engineBuilder.selectTarget();
 	if (!machine)
 		I_FatalError("Could not create LLVM target machine");
@@ -638,6 +709,10 @@ void LLVMProgram::CreateEE()
 	Printf("LLVM target triple: %s\n", targetTriple.c_str());
 	Printf("LLVM target CPU: %s\n", cpuName.c_str());
 
+	if (optimize)
+	{
+		Printf("Optimizing drawers..\n");
+
 		module->setTargetTriple(targetTriple);
 #if LLVM_VERSION_MAJOR < 3 || (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR < 8)
 		module->setDataLayout(new DataLayout(*machine->getSubtargetImpl()->getDataLayout()));
@@ -675,6 +750,11 @@ void LLVMProgram::CreateEE()
 		// Run module passes:
 		PerModulePasses.run(*module);
 
+		SaveCachedModule(module, version, targetCPU);
+	}
+
+	Printf("Compiling drawers..\n");
+
 	// Create execution engine and generate machine code
 	mEngine.reset(engineBuilder.create(machine));
 	if (!mEngine)