doom3-bfg/neo/framework/Common_localize.cpp

/*
===========================================================================

Doom 3 BFG Edition GPL Source Code
Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company.

This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code").

Doom 3 BFG Edition Source Code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Doom 3 BFG Edition Source Code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Doom 3 BFG Edition Source Code.  If not, see <http://www.gnu.org/licenses/>.

In addition, the Doom 3 BFG Edition Source Code is also subject to certain additional terms. You should have received a copy of these additional terms immediately following the terms and conditions of the GNU General Public License which accompanied the Doom 3 BFG Edition Source Code.  If not, please request a copy in writing from id Software at the address below.

If you have questions concerning this license or the applicable additional terms, you may contact in writing id Software LLC, c/o ZeniMax Media Inc., Suite 120, Rockville, Maryland 20850 USA.

===========================================================================
*/

#include "../idlib/precompiled.h"
#pragma hdrstop

#include "Common_local.h"

idCVar com_product_lang_ext( "com_product_lang_ext", "1", CVAR_INTEGER | CVAR_SYSTEM | CVAR_ARCHIVE, "Extension to use when creating language files." );

/*
=================
LoadMapLocalizeData
=================
*/
typedef idHashTable<idStrList> ListHash;
void LoadMapLocalizeData( ListHash& listHash )
{

	idStr fileName = "map_localize.cfg";
	const char* buffer = NULL;
	idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT );
	
	if( fileSystem->ReadFile( fileName, ( void** )&buffer ) > 0 )
	{
		src.LoadMemory( buffer, strlen( buffer ), fileName );
		if( src.IsLoaded() )
		{
			idStr classname;
			idToken token;
			
			
			
			while( src.ReadToken( &token ) )
			{
				classname = token;
				src.ExpectTokenString( "{" );
				
				idStrList list;
				while( src.ReadToken( &token ) )
				{
					if( token == "}" )
					{
						break;
					}
					list.Append( token );
				}
				
				listHash.Set( classname, list );
			}
		}
		fileSystem->FreeFile( ( void* )buffer );
	}
	
}

void LoadGuiParmExcludeList( idStrList& list )
{

	idStr fileName = "guiparm_exclude.cfg";
	const char* buffer = NULL;
	idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT );
	
	if( fileSystem->ReadFile( fileName, ( void** )&buffer ) > 0 )
	{
		src.LoadMemory( buffer, strlen( buffer ), fileName );
		if( src.IsLoaded() )
		{
			idStr classname;
			idToken token;
			
			
			
			while( src.ReadToken( &token ) )
			{
				list.Append( token );
			}
		}
		fileSystem->FreeFile( ( void* )buffer );
	}
}

bool TestMapVal( idStr& str )
{
	//Already Localized?
	if( str.Find( "#str_" ) != -1 )
	{
		return false;
	}
	
	return true;
}

bool TestGuiParm( const char* parm, const char* value, idStrList& excludeList )
{

	idStr testVal = value;
	
	//Already Localized?
	if( testVal.Find( "#str_" ) != -1 )
	{
		return false;
	}
	
	//Numeric
	if( testVal.IsNumeric() )
	{
		return false;
	}
	
	//Contains ::
	if( testVal.Find( "::" ) != -1 )
	{
		return false;
	}
	
	//Contains /
	if( testVal.Find( "/" ) != -1 )
	{
		return false;
	}
	
	if( excludeList.Find( testVal ) )
	{
		return false;
	}
	
	return true;
}

void GetFileList( const char* dir, const char* ext, idStrList& list )
{

	//Recurse Subdirectories
	idStrList dirList;
	Sys_ListFiles( dir, "/", dirList );
	for( int i = 0; i < dirList.Num(); i++ )
	{
		if( dirList[i] == "." || dirList[i] == ".." )
		{
			continue;
		}
		idStr fullName = va( "%s/%s", dir, dirList[i].c_str() );
		GetFileList( fullName, ext, list );
	}
	
	idStrList fileList;
	Sys_ListFiles( dir, ext, fileList );
	for( int i = 0; i < fileList.Num(); i++ )
	{
		idStr fullName = va( "%s/%s", dir, fileList[i].c_str() );
		list.Append( fullName );
	}
}

int LocalizeMap( const char* mapName, idLangDict& langDict, ListHash& listHash, idStrList& excludeList, bool writeFile )
{

	common->Printf( "Localizing Map '%s'\n", mapName );
	
	int strCount = 0;
	
	idMapFile map;
	if( map.Parse( mapName, false, false ) )
	{
		int count = map.GetNumEntities();
		for( int j = 0; j < count; j++ )
		{
			idMapEntity* ent = map.GetEntity( j );
			if( ent )
			{
			
				idStr classname = ent->epairs.GetString( "classname" );
				
				//Hack: for info_location
				bool hasLocation = false;
				
				idStrList* list;
				listHash.Get( classname, &list );
				if( list )
				{
				
					for( int k = 0; k < list->Num(); k++ )
					{
					
						idStr val = ent->epairs.GetString( ( *list )[k], "" );
						
						if( val.Length() && classname == "info_location" && ( *list )[k] == "location" )
						{
							hasLocation = true;
						}
						
						if( val.Length() && TestMapVal( val ) )
						{
						
							if( !hasLocation || ( *list )[k] == "location" )
							{
								//Localize it!!!
								strCount++;
								ent->epairs.Set( ( *list )[k], langDict.AddString( val ) );
							}
						}
					}
				}
				
				listHash.Get( "all", &list );
				if( list )
				{
					for( int k = 0; k < list->Num(); k++ )
					{
						idStr val = ent->epairs.GetString( ( *list )[k], "" );
						if( val.Length() && TestMapVal( val ) )
						{
							//Localize it!!!
							strCount++;
							ent->epairs.Set( ( *list )[k], langDict.AddString( val ) );
						}
					}
				}
				
				//Localize the gui_parms
				const idKeyValue* kv = ent->epairs.MatchPrefix( "gui_parm" );
				while( kv )
				{
					if( TestGuiParm( kv->GetKey(), kv->GetValue(), excludeList ) )
					{
						//Localize It!
						strCount++;
						ent->epairs.Set( kv->GetKey(), langDict.AddString( kv->GetValue() ) );
					}
					kv = ent->epairs.MatchPrefix( "gui_parm", kv );
				}
			}
		}
		if( writeFile && strCount > 0 )
		{
			//Before we write the map file lets make a backup of the original
			idStr file =  fileSystem->RelativePathToOSPath( mapName );
			idStr bak = file.Left( file.Length() - 4 );
			bak.Append( ".bak_loc" );
			fileSystem->CopyFile( file, bak );
			
			map.Write( mapName, ".map" );
		}
	}
	
	common->Printf( "Count: %d\n", strCount );
	return strCount;
}

/*
=================
LocalizeMaps_f
=================
*/
CONSOLE_COMMAND( localizeMaps, "localize maps", NULL )
{
	if( args.Argc() < 2 )
	{
		common->Printf( "Usage: localizeMaps <count | dictupdate | all> <map>\n" );
		return;
	}
	
	int strCount = 0;
	
	bool count = false;
	bool dictUpdate = false;
	bool write = false;
	
	if( idStr::Icmp( args.Argv( 1 ), "count" ) == 0 )
	{
		count = true;
	}
	else if( idStr::Icmp( args.Argv( 1 ), "dictupdate" ) == 0 )
	{
		count = true;
		dictUpdate = true;
	}
	else if( idStr::Icmp( args.Argv( 1 ), "all" ) == 0 )
	{
		count = true;
		dictUpdate = true;
		write = true;
	}
	else
	{
		common->Printf( "Invalid Command\n" );
		common->Printf( "Usage: localizeMaps <count | dictupdate | all>\n" );
		return;
		
	}
	
	idLangDict strTable;
	idStr filename = va( "strings/english%.3i.lang", com_product_lang_ext.GetInteger() );
	
	{
		// I think this is equivalent...
		const byte* buffer = NULL;
		int len = fileSystem->ReadFile( filename, ( void** )&buffer );
		if( verify( len > 0 ) )
		{
			strTable.Load( buffer, len, filename );
		}
		fileSystem->FreeFile( ( void* )buffer );
		
		// ... to this
		//if ( strTable.Load( filename ) == false) {
		//	//This is a new file so set the base index
		//	strTable.SetBaseID(com_product_lang_ext.GetInteger()*100000);
		//}
	}
	
	common->SetRefreshOnPrint( true );
	
	ListHash listHash;
	LoadMapLocalizeData( listHash );
	
	idStrList excludeList;
	LoadGuiParmExcludeList( excludeList );
	
	if( args.Argc() == 3 )
	{
		strCount += LocalizeMap( args.Argv( 2 ), strTable, listHash, excludeList, write );
	}
	else
	{
		idStrList files;
		GetFileList( "z:/d3xp/d3xp/maps/game", "*.map", files );
		for( int i = 0; i < files.Num(); i++ )
		{
			idStr file =  fileSystem->OSPathToRelativePath( files[i] );
			strCount += LocalizeMap( file, strTable, listHash, excludeList, write );
		}
	}
	
	if( count )
	{
		common->Printf( "Localize String Count: %d\n", strCount );
	}
	
	common->SetRefreshOnPrint( false );
	
	if( dictUpdate )
	{
		strTable.Save( filename );
	}
}

/*
=================
LocalizeGuis_f
=================
*/
CONSOLE_COMMAND( localizeGuis, "localize guis", NULL )
{

	if( args.Argc() != 2 )
	{
		common->Printf( "Usage: localizeGuis <all | gui>\n" );
		return;
	}
	
	idLangDict strTable;
	
	idStr filename = va( "strings/english%.3i.lang", com_product_lang_ext.GetInteger() );
	
	{
		// I think this is equivalent...
		const byte* buffer = NULL;
		int len = fileSystem->ReadFile( filename, ( void** )&buffer );
		if( verify( len > 0 ) )
		{
			strTable.Load( buffer, len, filename );
		}
		fileSystem->FreeFile( ( void* )buffer );
		
		// ... to this
		//if(strTable.Load( filename ) == false) {
		//	//This is a new file so set the base index
		//	strTable.SetBaseID(com_product_lang_ext.GetInteger()*100000);
		//}
	}
	
	idFileList* files;
	if( idStr::Icmp( args.Argv( 1 ), "all" ) == 0 )
	{
		idStr game = cvarSystem->GetCVarString( "game_expansion" );
		if( game.Length() )
		{
			files = fileSystem->ListFilesTree( "guis", "*.gui", true, game );
		}
		else
		{
			files = fileSystem->ListFilesTree( "guis", "*.gui", true );
		}
		for( int i = 0; i < files->GetNumFiles(); i++ )
		{
			commonLocal.LocalizeGui( files->GetFile( i ), strTable );
		}
		fileSystem->FreeFileList( files );
		
		if( game.Length() )
		{
			files = fileSystem->ListFilesTree( "guis", "*.pd", true, game );
		}
		else
		{
			files = fileSystem->ListFilesTree( "guis", "*.pd", true, "d3xp" );
		}
		
		for( int i = 0; i < files->GetNumFiles(); i++ )
		{
			commonLocal.LocalizeGui( files->GetFile( i ), strTable );
		}
		fileSystem->FreeFileList( files );
		
	}
	else
	{
		commonLocal.LocalizeGui( args.Argv( 1 ), strTable );
	}
	strTable.Save( filename );
}

CONSOLE_COMMAND( localizeGuiParmsTest, "Create test files that show gui parms localized and ignored.", NULL )
{

	common->SetRefreshOnPrint( true );
	
	idFile* localizeFile = fileSystem->OpenFileWrite( "gui_parm_localize.csv" );
	idFile* noLocalizeFile = fileSystem->OpenFileWrite( "gui_parm_nolocalize.csv" );
	
	idStrList excludeList;
	LoadGuiParmExcludeList( excludeList );
	
	idStrList files;
	GetFileList( "z:/d3xp/d3xp/maps/game", "*.map", files );
	
	for( int i = 0; i < files.Num(); i++ )
	{
	
		common->Printf( "Testing Map '%s'\n", files[i].c_str() );
		idMapFile map;
		
		idStr file =  fileSystem->OSPathToRelativePath( files[i] );
		if( map.Parse( file, false, false ) )
		{
			int count = map.GetNumEntities();
			for( int j = 0; j < count; j++ )
			{
				idMapEntity* ent = map.GetEntity( j );
				if( ent )
				{
					const idKeyValue* kv = ent->epairs.MatchPrefix( "gui_parm" );
					while( kv )
					{
						if( TestGuiParm( kv->GetKey(), kv->GetValue(), excludeList ) )
						{
							idStr out = va( "%s,%s,%s\r\n", kv->GetValue().c_str(), kv->GetKey().c_str(), file.c_str() );
							localizeFile->Write( out.c_str(), out.Length() );
						}
						else
						{
							idStr out = va( "%s,%s,%s\r\n", kv->GetValue().c_str(), kv->GetKey().c_str(), file.c_str() );
							noLocalizeFile->Write( out.c_str(), out.Length() );
						}
						kv = ent->epairs.MatchPrefix( "gui_parm", kv );
					}
				}
			}
		}
	}
	
	fileSystem->CloseFile( localizeFile );
	fileSystem->CloseFile( noLocalizeFile );
	
	common->SetRefreshOnPrint( false );
}


CONSOLE_COMMAND( localizeMapsTest, "Create test files that shows which strings will be localized.", NULL )
{

	ListHash listHash;
	LoadMapLocalizeData( listHash );
	
	
	common->SetRefreshOnPrint( true );
	
	idFile* localizeFile = fileSystem->OpenFileWrite( "map_localize.csv" );
	
	idStrList files;
	GetFileList( "z:/d3xp/d3xp/maps/game", "*.map", files );
	
	for( int i = 0; i < files.Num(); i++ )
	{
	
		common->Printf( "Testing Map '%s'\n", files[i].c_str() );
		idMapFile map;
		
		idStr file =  fileSystem->OSPathToRelativePath( files[i] );
		if( map.Parse( file, false, false ) )
		{
			int count = map.GetNumEntities();
			for( int j = 0; j < count; j++ )
			{
				idMapEntity* ent = map.GetEntity( j );
				if( ent )
				{
				
					//Temp code to get a list of all entity key value pairs
					/*idStr classname = ent->epairs.GetString("classname");
					if(classname == "worldspawn" || classname == "func_static" || classname == "light" || classname == "speaker" || classname.Left(8) == "trigger_") {
						continue;
					}
					for( int i = 0; i < ent->epairs.GetNumKeyVals(); i++) {
						const idKeyValue* kv = ent->epairs.GetKeyVal(i);
						idStr out = va("%s,%s,%s,%s\r\n", classname.c_str(), kv->GetKey().c_str(), kv->GetValue().c_str(), file.c_str());
						localizeFile->Write( out.c_str(), out.Length() );
					}*/
					
					idStr classname = ent->epairs.GetString( "classname" );
					
					//Hack: for info_location
					bool hasLocation = false;
					
					idStrList* list;
					listHash.Get( classname, &list );
					if( list )
					{
					
						for( int k = 0; k < list->Num(); k++ )
						{
						
							idStr val = ent->epairs.GetString( ( *list )[k], "" );
							
							if( classname == "info_location" && ( *list )[k] == "location" )
							{
								hasLocation = true;
							}
							
							if( val.Length() && TestMapVal( val ) )
							{
							
								if( !hasLocation || ( *list )[k] == "location" )
								{
									idStr out = va( "%s,%s,%s\r\n", val.c_str(), ( *list )[k].c_str(), file.c_str() );
									localizeFile->Write( out.c_str(), out.Length() );
								}
							}
						}
					}
					
					listHash.Get( "all", &list );
					if( list )
					{
						for( int k = 0; k < list->Num(); k++ )
						{
							idStr val = ent->epairs.GetString( ( *list )[k], "" );
							if( val.Length() && TestMapVal( val ) )
							{
								idStr out = va( "%s,%s,%s\r\n", val.c_str(), ( *list )[k].c_str(), file.c_str() );
								localizeFile->Write( out.c_str(), out.Length() );
							}
						}
					}
				}
			}
		}
	}
	
	fileSystem->CloseFile( localizeFile );
	
	common->SetRefreshOnPrint( false );
}

/*
===============
idCommonLocal::LocalizeSpecificMapData
===============
*/
void idCommonLocal::LocalizeSpecificMapData( const char* fileName, idLangDict& langDict, const idLangDict& replaceArgs )
{
	idStr out, ws, work;
	
	idMapFile map;
	if( map.Parse( fileName, false, false ) )
	{
		int count = map.GetNumEntities();
		for( int i = 0; i < count; i++ )
		{
			idMapEntity* ent = map.GetEntity( i );
			if( ent )
			{
				for( int j = 0; j < replaceArgs.GetNumKeyVals(); j++ )
				{
					const idLangKeyValue* kv = replaceArgs.GetKeyVal( j );
					const char* temp = ent->epairs.GetString( kv->key );
					if( ( temp != NULL ) && *temp )
					{
						idStr val = kv->value;
						if( val == temp )
						{
							ent->epairs.Set( kv->key, langDict.AddString( temp ) );
						}
					}
				}
			}
		}
		map.Write( fileName, ".map" );
	}
}

/*
===============
idCommonLocal::LocalizeMapData
===============
*/
void idCommonLocal::LocalizeMapData( const char* fileName, idLangDict& langDict )
{
	const char* buffer = NULL;
	idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT );
	
	common->SetRefreshOnPrint( true );
	
	if( fileSystem->ReadFile( fileName, ( void** )&buffer ) > 0 )
	{
		src.LoadMemory( buffer, strlen( buffer ), fileName );
		if( src.IsLoaded() )
		{
			common->Printf( "Processing %s\n", fileName );
			idStr mapFileName;
			idToken token, token2;
			idLangDict replaceArgs;
			while( src.ReadToken( &token ) )
			{
				mapFileName = token;
				replaceArgs.Clear();
				src.ExpectTokenString( "{" );
				while( src.ReadToken( &token ) )
				{
					if( token == "}" )
					{
						break;
					}
					if( src.ReadToken( &token2 ) )
					{
						if( token2 == "}" )
						{
							break;
						}
						replaceArgs.AddKeyVal( token, token2 );
					}
				}
				common->Printf( "  localizing map %s...\n", mapFileName.c_str() );
				LocalizeSpecificMapData( mapFileName, langDict, replaceArgs );
			}
		}
		fileSystem->FreeFile( ( void* )buffer );
	}
	
	common->SetRefreshOnPrint( false );
}

/*
===============
idCommonLocal::LocalizeGui
===============
*/
void idCommonLocal::LocalizeGui( const char* fileName, idLangDict& langDict )
{
	idStr out, ws, work;
	const char* buffer = NULL;
	out.Empty();
	int k;
	char ch;
	char slash = '\\';
	char tab = 't';
	char nl = 'n';
	idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT );
	if( fileSystem->ReadFile( fileName, ( void** )&buffer ) > 0 )
	{
		src.LoadMemory( buffer, strlen( buffer ), fileName );
		if( src.IsLoaded() )
		{
			idFile* outFile = fileSystem->OpenFileWrite( fileName );
			common->Printf( "Processing %s\n", fileName );
			
			const bool captureToImage = false;
			UpdateScreen( captureToImage );
			idToken token;
			while( src.ReadToken( &token ) )
			{
				src.GetLastWhiteSpace( ws );
				out += ws;
				if( token.type == TT_STRING )
				{
					out += va( "\"%s\"", token.c_str() );
				}
				else
				{
					out += token;
				}
				if( out.Length() > 200000 )
				{
					outFile->Write( out.c_str(), out.Length() );
					out = "";
				}
				work = token.Right( 6 );
				if( token.Icmp( "text" ) == 0 || work.Icmp( "::text" ) == 0 || token.Icmp( "choices" ) == 0 )
				{
					if( src.ReadToken( &token ) )
					{
						// see if already exists, if so save that id to this position in this file
						// otherwise add this to the list and save the id to this position in this file
						src.GetLastWhiteSpace( ws );
						out += ws;
						token = langDict.AddString( token );
						out += "\"";
						for( k = 0; k < token.Length(); k++ )
						{
							ch = token[k];
							if( ch == '\t' )
							{
								out += slash;
								out += tab;
							}
							else if( ch == '\n' || ch == '\r' )
							{
								out += slash;
								out += nl;
							}
							else
							{
								out += ch;
							}
						}
						out += "\"";
					}
				}
				else if( token.Icmp( "comment" ) == 0 )
				{
					if( src.ReadToken( &token ) )
					{
						// need to write these out by hand to preserve any \n's
						// see if already exists, if so save that id to this position in this file
						// otherwise add this to the list and save the id to this position in this file
						src.GetLastWhiteSpace( ws );
						out += ws;
						out += "\"";
						for( k = 0; k < token.Length(); k++ )
						{
							ch = token[k];
							if( ch == '\t' )
							{
								out += slash;
								out += tab;
							}
							else if( ch == '\n' || ch == '\r' )
							{
								out += slash;
								out += nl;
							}
							else
							{
								out += ch;
							}
						}
						out += "\"";
					}
				}
			}
			outFile->Write( out.c_str(), out.Length() );
			fileSystem->CloseFile( outFile );
		}
		fileSystem->FreeFile( ( void* )buffer );
	}
}