Embedded Cg renderprog sources into the renderer using Lua/Premake. Now the renderer can export the renderprogs to any GLSL version without having the user to copy the renderprogs/* shaders to the Doom 3 BFG installation directory. fixes #95

base/renderprogs/_manifest.lua

@@ -0,0 +1,100 @@
+return
+{
+	-- includes
+	"global.inc",
+	"skinning.inc",
+	
+	-- shaders
+	"bink.pixel",
+	"bink.vertex",
+	"bink_gui.pixel",
+	"bink_gui.vertex",
+	"blendLight.pixel",
+	"blendLight.vertex",
+	"bloodorb1_capture.pixel",
+	"bloodorb1_capture.vertex",
+	"bloodorb2_capture.pixel",
+	"bloodorb2_capture.vertex",
+	"bloodorb3_capture.pixel",
+	"bloodorb3_capture.vertex",
+	"bloodorb_draw.pixel",
+	"bloodorb_draw.vertex",
+	"bumpyenvironment.pixel",
+	"bumpyenvironment.vertex",
+	"bumpyenvironment_skinned.pixel",
+	"bumpyenvironment_skinned.vertex",
+	"color.pixel",
+	"color.vertex",
+	"colorProcess.pixel",
+	"colorProcess.vertex",
+	"depth.pixel",
+	"depth.vertex",
+	"depth_skinned.pixel",
+	"depth_skinned.vertex",
+	"environment.pixel",
+	"environment.vertex",
+	"environment_skinned.pixel",
+	"environment_skinned.vertex",
+	"enviroSuit.pixel",
+	"enviroSuit.vertex",
+	"fog.pixel",
+	"fog.vertex",
+	"fog_skinned.pixel",
+	"fog_skinned.vertex",
+	"fxaa.pixel",
+	"fxaa.vertex",
+	"global.inc",
+	"gui.pixel",
+	"gui.vertex",
+	"heathaze.pixel",
+	"heathaze.vertex",
+	"heatHazeWithMask.pixel",
+	"heatHazeWithMask.vertex",
+	"heatHazeWithMaskAndVertex.pixel",
+	"heatHazeWithMaskAndVertex.vertex",
+	"interaction.pixel",
+	"interaction.vertex",
+	"interactionAmbient.pixel",
+	"interactionAmbient.vertex",
+	"interactionAmbient_skinned.pixel",
+	"interactionAmbient_skinned.vertex",
+	"interaction_skinned.pixel",
+	"interaction_skinned.vertex",
+	"motionBlur.pixel",
+	"motionBlur.vertex",
+	"postprocess.pixel",
+	"postprocess.vertex",
+	"shadow.pixel",
+	"shadow.vertex",
+	"shadowDebug.pixel",
+	"shadowDebug.vertex",
+	"shadowDebug_skinned.pixel",
+	"shadowDebug_skinned.vertex",
+	"shadow_skinned.pixel",
+	"shadow_skinned.vertex",
+	"simpleshade.pixel",
+	"simpleshade.vertex",
+	"skinning.inc",
+	"skybox.pixel",
+	"skybox.vertex",
+	"stereoDeGhost.pixel",
+	"stereoDeGhost.vertex",
+	"stereoInterlace.pixel",
+	"stereoInterlace.vertex",
+	"stereoWarp.pixel",
+	"stereoWarp.vertex",
+	"texture.pixel",
+	"texture.vertex",
+	"texture_color.pixel",
+	"texture_color.vertex",
+	"texture_color_skinned.pixel",
+	"texture_color_skinned.vertex",
+	"texture_color_texgen.pixel",
+	"texture_color_texgen.vertex",
+	"wobblesky.pixel",
+	"wobblesky.vertex",
+	"zcullReconstruct.pixel",
+	"zcullReconstruct.vertex",
+}
+
+	

doomclassic/doom/mus2midi.cpp

@@ -31,7 +31,6 @@ If you have questions concerning this license or the applicable additional terms
 #include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include "idlib/sys/sys_defines.h"
 // mus header
 
 

neo/idlib/Parser.cpp

@@ -3,6 +3,7 @@
 
 Doom 3 BFG Edition GPL Source Code
 Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company.
+Copyright (C) 2014 Robert Beckebans
 
 This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code").
 
@@ -2597,7 +2598,9 @@ int idParser::ReadDirective()
 		{
 			if( token == "include" )
 			{
-				return idParser::Directive_include();
+				// RB lets override for embedded shaders
+				return Directive_include();
+				// RB end
 			}
 			else if( token == "define" )
 			{

neo/idlib/Parser.h

@@ -3,6 +3,7 @@
 
 Doom 3 BFG Edition GPL Source Code
 Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company.
+Copyright (C) 2014 Robert Beckebans
 
 This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code").
 
@@ -185,7 +186,9 @@ public:
 	// set the base folder to load files from
 	static void		SetBaseFolder( const char* path );
 	
-private:
+// RB: made protected to have custom #include behaviours for embedded resources
+protected:
+// RB end
 	int				loaded;						// set when a source file is loaded from file or memory
 	idStr			filename;					// file name of the script
 	idStr			includepath;				// path to include files
@@ -202,7 +205,6 @@ private:
 	
 	static define_t* globaldefines;				// list with global defines added to every source loaded
 	
-private:
 	void			PushIndent( int type, int skip );
 	void			PopIndent( int* type, int* skip );
 	void			PushScript( idLexer* script );
@@ -225,7 +227,9 @@ private:
 	static define_t* FindDefine( define_t* defines, const char* name );
 	static define_t* DefineFromString( const char* string );
 	define_t* 		CopyFirstDefine();
-	int				Directive_include();
+	// RB: allow override
+	virtual int		Directive_include();
+	// RB end
 	int				Directive_undef();
 	int				Directive_if_def( int type );
 	int				Directive_ifdef();

neo/premake4.lua

@@ -0,0 +1,13 @@
+--
+-- Use the embed action to convert all of the renderprogs scripts into C strings, which 
+-- can then be built into the executable. Always embed the scripts before creating
+-- a release build.
+--
+dofile("premake/embed.lua")
+
+newaction
+{
+	trigger     = "embed",
+	description = "Embed renderprogs and scripts in into the engine code; required before release builds",
+	execute     = doembed
+}

neo/premake4_embed.bat

@@ -0,0 +1,2 @@
+premake4 embed
+pause

neo/renderer/RenderProgs_GLSL.cpp

@@ -31,6 +31,7 @@ If you have questions concerning this license or the applicable additional terms
 #include "precompiled.h"
 
 #include "tr_local.h"
+#include "RenderProgs_embedded.h"
 
 idCVar r_skipStripDeadCode( "r_skipStripDeadCode", "0", CVAR_BOOL, "Skip stripping dead code" );
 idCVar r_useUniformArrays( "r_useUniformArrays", "1", CVAR_BOOL, "" );
@@ -315,6 +316,137 @@ static const char* GLSLParmNames[] =
 	"rpAlphaTest"
 };
 
+// RB: added embedded Cg shader resources
+static const char* FindEmbeddedSourceShader( const char* name )
+{
+	const char* embeddedSource = NULL;
+	for( int i = 0 ; cg_renderprogs[i].name ; i++ )
+	{
+		if( !idStr::Icmp( cg_renderprogs[i].name, name ) )
+		{
+			embeddedSource = cg_renderprogs[i].shaderText;
+			break;
+		}
+	}
+	
+	return embeddedSource;
+}
+
+class idParser_EmbeddedGLSL : public idParser
+{
+public:
+	idParser_EmbeddedGLSL( int flags ) : idParser( flags )
+	{
+	}
+	
+private:
+	int		Directive_include()
+	{
+		idLexer* script;
+		idToken token;
+		idStr path;
+		
+		if( !idParser::ReadSourceToken( &token ) )
+		{
+			idParser::Error( "#include without file name" );
+			return false;
+		}
+		
+		if( token.linesCrossed > 0 )
+		{
+			idParser::Error( "#include without file name" );
+			return false;
+		}
+		
+		if( token.type == TT_STRING )
+		{
+			script = new idLexer;
+			
+			// try relative to the current file
+			path = scriptstack->GetFileName();
+			path.StripFilename();
+			path += "/";
+			path += token;
+			
+			//if( !script->LoadFile( path, OSPath ) )
+			const char* embeddedSource = FindEmbeddedSourceShader( path );
+			if( embeddedSource == NULL )
+			{
+				// try absolute path
+				path = token;
+				embeddedSource = FindEmbeddedSourceShader( path );
+				if( embeddedSource == NULL )
+				{
+					// try from the include path
+					path = includepath + token;
+					embeddedSource = FindEmbeddedSourceShader( path );
+				}
+			}
+			
+			if( embeddedSource == NULL || !script->LoadMemory( embeddedSource, strlen( embeddedSource ), path ) )
+			{
+				delete script;
+				script = NULL;
+			}
+		}
+		else if( token.type == TT_PUNCTUATION && token == "<" )
+		{
+			path = idParser::includepath;
+			while( idParser::ReadSourceToken( &token ) )
+			{
+				if( token.linesCrossed > 0 )
+				{
+					idParser::UnreadSourceToken( &token );
+					break;
+				}
+				if( token.type == TT_PUNCTUATION && token == ">" )
+				{
+					break;
+				}
+				path += token;
+			}
+			if( token != ">" )
+			{
+				idParser::Warning( "#include missing trailing >" );
+			}
+			if( !path.Length() )
+			{
+				idParser::Error( "#include without file name between < >" );
+				return false;
+			}
+			if( idParser::flags & LEXFL_NOBASEINCLUDES )
+			{
+				return true;
+			}
+			script = new idLexer;
+			
+			const char* embeddedSource = FindEmbeddedSourceShader( includepath + path );
+			
+			if( embeddedSource == NULL || !script->LoadMemory( embeddedSource, strlen( embeddedSource ), path ) )
+			{
+				delete script;
+				script = NULL;
+			}
+		}
+		else
+		{
+			idParser::Error( "#include without file name" );
+			return false;
+		}
+		
+		if( !script )
+		{
+			idParser::Error( "file '%s' not found", path.c_str() );
+			return false;
+		}
+		script->SetFlags( idParser::flags );
+		script->SetPunctuations( idParser::punctuations );
+		idParser::PushScript( script );
+		return true;
+	}
+};
+// RB end
+
 /*
 ========================
 StripDeadCode
@@ -328,7 +460,7 @@ idStr StripDeadCode( const idStr& in, const char* name )
 	}
 	
 	//idLexer src( LEXFL_NOFATALERRORS );
-	idParser src( LEXFL_NOFATALERRORS );
+	idParser_EmbeddedGLSL src( LEXFL_NOFATALERRORS );
 	src.LoadMemory( in.c_str(), in.Length(), name );
 	src.AddDefine( "PC" );
 	
@@ -1365,7 +1497,6 @@ GLuint idRenderProgManager::LoadGLSLShader( GLenum target, const char* name, idL
 	
 	outFileGLSL.StripFileExtension();
 	outFileUniforms.StripFileExtension();
-	// RB end
 	
 	if( target == GL_FRAGMENT_SHADER )
 	{
@@ -1392,21 +1523,31 @@ GLuint idRenderProgManager::LoadGLSLShader( GLenum target, const char* name, idL
 	// if the glsl file doesn't exist or we have a newer HLSL file we need to recreate the glsl file.
 	idStr programGLSL;
 	idStr programUniforms;
-	if( ( glslFileLength <= 0 ) || ( hlslTimeStamp > glslTimeStamp ) || r_alwaysExportGLSL.GetBool() )
+	if( ( glslFileLength <= 0 ) || ( hlslTimeStamp != FILE_NOT_FOUND_TIMESTAMP && hlslTimeStamp > glslTimeStamp ) || r_alwaysExportGLSL.GetBool() )
 	{
+		const char* hlslFileBuffer = NULL;
+		int len = 0;
+		
 		if( hlslFileLength <= 0 )
 		{
 			// hlsl file doesn't even exist bail out
-			return false;
+			hlslFileBuffer = FindEmbeddedSourceShader( inFile.c_str() );
+			if( hlslFileBuffer == NULL )
+			{
+				return false;
+			}
+			len = strlen( hlslFileBuffer );
+		}
+		else
+		{
+			len = fileSystem->ReadFile( inFile.c_str(), ( void** ) &hlslFileBuffer );
 		}
 		
-		void* hlslFileBuffer = NULL;
-		int len = fileSystem->ReadFile( inFile.c_str(), &hlslFileBuffer );
 		if( len <= 0 )
 		{
 			return false;
 		}
-		idStr hlslCode( ( const char* ) hlslFileBuffer );
+		idStr hlslCode( hlslFileBuffer );
 		idStr programHLSL = StripDeadCode( hlslCode, inFile );
 		programGLSL = ConvertCG2GLSL( programHLSL, inFile, target == GL_VERTEX_SHADER, programUniforms );