doom3-bfg/neo/idlib/LangDict.cpp

/*
===========================================================================

Doom 3 BFG Edition GPL Source Code
Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company.

This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code").

Doom 3 BFG Edition Source Code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Doom 3 BFG Edition Source Code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Doom 3 BFG Edition Source Code.  If not, see <http://www.gnu.org/licenses/>.

In addition, the Doom 3 BFG Edition Source Code is also subject to certain additional terms. You should have received a copy of these additional terms immediately following the terms and conditions of the GNU General Public License which accompanied the Doom 3 BFG Edition Source Code.  If not, please request a copy in writing from id Software at the address below.

If you have questions concerning this license or the applicable additional terms, you may contact in writing id Software LLC, c/o ZeniMax Media Inc., Suite 120, Rockville, Maryland 20850 USA.

===========================================================================
*/
#pragma hdrstop
#include "precompiled.h"

// This is the default language dict that the entire system uses, but you can instantiate your own idLangDict classes to manipulate a language dictionary in a tool
idLangDict	idLocalization::languageDict;

idCVar lang_maskLocalizedStrings( "lang_maskLocalizedStrings", "0", CVAR_BOOL, "Masks all localized strings to help debugging.  When set will replace strings with an equal length of W's and ending in an X.  Note: The masking occurs at string table load time." );

/*
========================
idLocalization::ClearDictionary
========================
*/
void idLocalization::ClearDictionary()
{
	languageDict.Clear();
}

/*
========================
idLocalization::LoadDictionary
========================
*/
bool idLocalization::LoadDictionary( const byte* data, int dataLen, const char* fileName )
{
	return languageDict.Load( data, dataLen, fileName );
}

/*
========================
idLocalization::GetString
========================
*/
const char* idLocalization::GetString( const char* inString )
{
	return languageDict.GetString( inString );
}

/*
========================
idLocalization::FindString
========================
*/
const char* idLocalization::FindString( const char* inString )
{
	return languageDict.FindString( inString );
}

/*
========================
idLocalization::VerifyUTF8
========================
*/
utf8Encoding_t idLocalization::VerifyUTF8( const uint8* buffer, const int bufferLen, const char* name )
{
	utf8Encoding_t encoding;
	idStr::IsValidUTF8( buffer, bufferLen, encoding );
	if( encoding == UTF8_INVALID )
	{
		idLib::FatalError( "Language file %s is not valid UTF-8 or plain ASCII.", name );
	}
	else if( encoding == UTF8_INVALID_BOM )
	{
		idLib::FatalError( "Language file %s is marked as UTF-8 but has invalid encoding.", name );
	}
	else if( encoding == UTF8_ENCODED_NO_BOM )
	{
		idLib::FatalError( "Language file %s has no byte order marker. Fix this or roll back to a version that has the marker.", name );
	}
	else if( encoding != UTF8_ENCODED_BOM && encoding != UTF8_PURE_ASCII )
	{
		idLib::FatalError( "Language file %s has unknown utf8Encoding_t.", name );
	}
	return encoding;
}

// string entries can refer to other string entries,
// recursing up to this many times before we decided someone did something stupid
const char* idLangDict::KEY_PREFIX = "#str_";	// all keys should be prefixed with this for redirection to work
const int idLangDict::KEY_PREFIX_LEN = idStr::Length( KEY_PREFIX );

/*
========================
idLangDict::idLangDict
========================
*/
idLangDict::idLangDict() : keyIndex( 4096, 4096 )
{
}

/*
========================
idLangDict::~idLangDict
========================
*/
idLangDict::~idLangDict()
{
	Clear();
}

/*
========================
idLangDict::Clear
========================
*/
void idLangDict::Clear()
{
	//mem.PushHeap();
	for( int i = 0; i < keyVals.Num(); i++ )
	{
		if( keyVals[i].value == NULL )
		{
			continue;
		}
		blockAlloc.Free( keyVals[i].value );
		keyVals[i].value = NULL;
	}
	//mem.PopHeap();
}

/*
========================
idLangDict::Load
========================
*/
bool idLangDict::Load( const byte* buffer, const int bufferLen, const char* name )
{

	if( buffer == NULL || bufferLen <= 0 )
	{
		// let whoever called us deal with the failure (so sys_lang can be reset)
		return false;
	}
	
	idLib::Printf( "Reading %s", name );
	
	bool utf8 = false;
	
	// in all but retail builds, ensure that the byte-order mark is NOT MISSING so that
	// we can avoid debugging UTF-8 code
#ifndef ID_RETAIL
	utf8Encoding_t encoding = idLocalization::VerifyUTF8( buffer, bufferLen, name );
	if( encoding == UTF8_ENCODED_BOM )
	{
		utf8 = true;
	}
	else if( encoding == UTF8_PURE_ASCII )
	{
		utf8 = false;
	}
	else
	{
		assert( false );	// this should have been handled in VerifyUTF8 with a FatalError
		return false;
	}
#else
	// in release we just check the BOM so we're not scanning the lang file twice on startup
	if( bufferLen > 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF )
	{
		utf8 = true;
	}
#endif
	
	if( utf8 )
	{
		idLib::Printf( " as UTF-8\n" );
	}
	else
	{
		idLib::Printf( " as ASCII\n" );
	}
	
	idStr tempKey;
	idStr tempVal;
	
	int line = 0;
	int numStrings = 0;
	
	int i = 0;
	while( i < bufferLen )
	{
		uint32 c = buffer[i++];
		if( c == '/' )    // comment, read until new line
		{
			while( i < bufferLen )
			{
				c = buffer[i++];
				if( c == '\n' )
				{
					line++;
					break;
				}
			}
		}
		else if( c == '}' )
		{
			break;
		}
		else if( c == '\n' )
		{
			line++;
		}
		else if( c == '\"' )
		{
			int keyStart = i;
			int keyEnd = -1;
			while( i < bufferLen )
			{
				c = buffer[i++];
				if( c == '\"' )
				{
					keyEnd = i - 1;
					break;
				}
			}
			if( keyEnd < keyStart )
			{
				idLib::FatalError( "%s File ended while reading key at line %d", name, line );
			}
			tempKey.CopyRange( ( char* )buffer, keyStart, keyEnd );
			
			int valStart = -1;
			while( i < bufferLen )
			{
				c = buffer[i++];
				if( c == '\"' )
				{
					valStart = i;
					break;
				}
			}
			if( valStart < 0 )
			{
				idLib::FatalError( "%s File ended while reading value at line %d", name, line );
			}
			int valEnd = -1;
			tempVal.CapLength( 0 );
			while( i < bufferLen )
			{
				c = utf8 ? idStr::UTF8Char( buffer, i ) : buffer[i++];
				if( !utf8 && c >= 0x80 )
				{
					// this is a serious error and we must check this to avoid accidentally shipping a file where someone squased UTF-8 encodings
					idLib::FatalError( "Language file %s is supposed to be plain ASCII, but has byte values > 127!", name );
				}
				if( c == '\"' )
				{
					valEnd = i - 1;
					continue;
				}
				if( c == '\n' )
				{
					line++;
					break;
				}
				if( c == '\r' )
				{
					continue;
				}
				if( c == '\\' )
				{
					c = utf8 ? idStr::UTF8Char( buffer, i ) : buffer[i++];
					if( c == 'n' )
					{
						c = '\n';
					}
					else if( c == 't' )
					{
						c = '\t';
					}
					else if( c == '\"' )
					{
						c = '\"';
					}
					else if( c == '\\' )
					{
						c = '\\';
					}
					else
					{
						idLib::Warning( "Unknown escape sequence %x at line %d", c, line );
					}
				}
				tempVal.AppendUTF8Char( c );
			}
			if( valEnd < valStart )
			{
				idLib::FatalError( "%s File ended while reading value at line %d", name, line );
			}
			if( lang_maskLocalizedStrings.GetBool() && tempVal.Length() > 0 && tempKey.Find( "#font_" ) == -1 )
			{
				int len = tempVal.Length();
				if( len > 0 )
				{
					tempVal.Fill( 'W', len - 1 );
				}
				else
				{
					tempVal.Empty();
				}
				tempVal.Append( 'X' );
			}
			AddKeyVal( tempKey, tempVal );
			numStrings++;
		}
	}
	
	idLib::Printf( "%i strings read\n", numStrings );
	
	// get rid of any waste due to geometric list growth
	//mem.PushHeap();
	keyVals.Condense();
	//mem.PopHeap();
	
	return true;
}

/*
========================
idLangDict::Save
========================
*/
bool idLangDict::Save( const char* fileName )
{
	idFile* outFile = fileSystem->OpenFileWrite( fileName );
	if( outFile == NULL )
	{
		idLib::Warning( "Error saving: %s", fileName );
		return false;
	}
	byte bof[3] = { 0xEF, 0xBB, 0xBF };
	outFile->Write( bof, 3 );
	outFile->WriteFloatString( "// string table\n//\n\n{\n" );
	for( int j = 0; j < keyVals.Num(); j++ )
	{
		const idLangKeyValue& kvp = keyVals[j];
		if( kvp.value == NULL )
		{
			continue;
		}
		outFile->WriteFloatString( "\t\"%s\"\t\"", kvp.key );
		for( int k = 0; kvp.value[k] != 0; k++ )
		{
			char ch = kvp.value[k];
			if( ch == '\t' )
			{
				outFile->Write( "\\t", 2 );
			}
			else if( ch == '\n' || ch == '\r' )
			{
				outFile->Write( "\\n", 2 );
			}
			else if( ch == '"' )
			{
				outFile->Write( "\\\"", 2 );
			}
			else if( ch == '\\' )
			{
				outFile->Write( "\\\\", 2 );
			}
			else
			{
				outFile->Write( &ch, 1 );
			}
		}
		outFile->WriteFloatString( "\"\n" );
	}
	outFile->WriteFloatString( "\n}\n" );
	delete outFile;
	return true;
}

/*
========================
idLangDict::GetString
========================
*/
const char* idLangDict::GetString( const char* str ) const
{
	const char* localized = FindString( str );
	if( localized == NULL )
	{
		return str;
	}
	return localized;
}

/*
========================
idLangDict::FindStringIndex
========================
*/
int idLangDict::FindStringIndex( const char* str ) const
{
	if( str == NULL )
	{
		return -1;
	}
	int hash = idStr::IHash( str );
	for( int i = keyIndex.GetFirst( hash ); i >= 0; i = keyIndex.GetNext( i ) )
	{
		if( idStr::Icmp( str, keyVals[i].key ) == 0 )
		{
			return i;
		}
	}
	return -1;
}

/*
========================
idLangDict::FindString_r
========================
*/
const char* idLangDict::FindString_r( const char* str, int& depth ) const
{
	depth++;
	if( depth > MAX_REDIRECTION_DEPTH )
	{
		// This isn't an error because we assume the error will be obvious somewhere in a GUI or something,
		// and the whole point of tracking the depth is to avoid a crash.
		idLib::Warning( "String '%s', indirection depth > %d", str, MAX_REDIRECTION_DEPTH );
		return NULL;
	}
	
	if( str == NULL || str[0] == '\0' )
	{
		return NULL;
	}
	
	int index = FindStringIndex( str );
	if( index < 0 )
	{
		return NULL;
	}
	const char* value = keyVals[index].value;
	if( value == NULL )
	{
		return NULL;
	}
	if( IsStringId( value ) )
	{
		// this string is re-directed to another entry
		return FindString_r( value, depth );
	}
	return value;
}

/*
========================
idLangDict::FindString
========================
*/
const char* idLangDict::FindString( const char* str ) const
{
	int depth = 0;
	return FindString_r( str, depth );
}

/*
========================
idLangDict::DeleteString
========================
*/
bool idLangDict::DeleteString( const char* key )
{
	return DeleteString( FindStringIndex( key ) );
}

/*
========================
idLangDict::DeleteString
========================
*/
bool idLangDict::DeleteString( const int idx )
{
	if( idx < 0 || idx >= keyVals.Num() )
	{
		return false;
	}
	
	//mem.PushHeap();
	blockAlloc.Free( keyVals[idx].value );
	keyVals[idx].value = NULL;
	//mem.PopHeap();
	
	return true;
}

/*
========================
idLangDict::RenameStringKey
========================
*/
bool idLangDict::RenameStringKey( const char* oldKey, const char* newKey )
{
	int index = FindStringIndex( oldKey );
	if( index < 0 )
	{
		return false;
	}
	//mem.PushHeap();
	blockAlloc.Free( keyVals[index].key );
	int newKeyLen = idStr::Length( newKey );
	keyVals[index].key = blockAlloc.Alloc( newKeyLen + 1 );
	idStr::Copynz( keyVals[index].key, newKey, newKeyLen + 1 );
	int oldHash = idStr::IHash( oldKey );
	int newHash = idStr::IHash( newKey );
	if( oldHash != newHash )
	{
		keyIndex.Remove( oldHash, index );
		keyIndex.Add( newHash, index );
	}
	//mem.PopHeap();
	
	return true;
}

/*
========================
idLangDict::SetString
========================
*/
bool idLangDict::SetString( const char* key, const char* val )
{
	int index = FindStringIndex( key );
	if( index < 0 )
	{
		return false;
	}
	//mem.PushHeap();
	if( keyVals[index].value != NULL )
	{
		blockAlloc.Free( keyVals[index].value );
	}
	int valLen = idStr::Length( val );
	keyVals[index].value = blockAlloc.Alloc( valLen + 1 );
	idStr::Copynz( keyVals[index].value, val, valLen + 1 );
	//mem.PopHeap();
	return true;
}

/*
========================
idLangDict::AddKeyVal
========================
*/
void idLangDict::AddKeyVal( const char* key, const char* val )
{
	if( SetString( key, val ) )
	{
		return;
	}
	//mem.PushHeap();
	int keyLen = idStr::Length( key );
	char* k = blockAlloc.Alloc( keyLen + 1 );
	idStr::Copynz( k, key, keyLen + 1 );
	char* v = NULL;
	if( val != NULL )
	{
		int valLen = idStr::Length( val );
		v = blockAlloc.Alloc( valLen + 1 );
		idStr::Copynz( v, val, valLen + 1 );
	}
	int index = keyVals.Append( idLangKeyValue( k, v ) );
	int hash = idStr::IHash( key );
	keyIndex.Add( hash, index );
	//mem.PopHeap();
}

/*
========================
idLangDict::AddString
========================
*/
const char* idLangDict::AddString( const char* val )
{
	int i = Sys_Milliseconds();
	idStr key;
	sprintf( key, "#str_%06d", ( i++ % 1000000 ) );
	while( FindStringIndex( key ) > 0 )
	{
		sprintf( key, "#str_%06d", ( i++ % 1000000 ) );
	}
	AddKeyVal( key, val );
	int index = FindStringIndex( key );
	return keyVals[index].key;
}

/*
========================
idLangDict::GetNumKeyVals
========================
*/
int idLangDict::GetNumKeyVals() const
{
	return keyVals.Num();
}

/*
========================
idLangDict::GetKeyVal
========================
*/
const idLangKeyValue* idLangDict::GetKeyVal( int i ) const
{
	return &keyVals[i];
}

/*
========================
idLangDict::IsStringId
========================
*/
bool idLangDict::IsStringId( const char* str )
{
	return idStr::Icmpn( str, KEY_PREFIX, KEY_PREFIX_LEN ) == 0;
}

/*
========================
idLangDict::GetLocalizedString
========================
*/
const char* idLangDict::GetLocalizedString( const idStrId& strId ) const
{
	if( strId.GetIndex() >= 0 && strId.GetIndex() < keyVals.Num() )
	{
		if( keyVals[ strId.GetIndex() ].value == NULL )
		{
			return keyVals[ strId.GetIndex() ].key;
		}
		else
		{
			return keyVals[ strId.GetIndex() ].value;
		}
	}
	return "";
}

/*
================================================================================================
idStrId
================================================================================================
*/

/*
========================
idStrId::Set
========================
*/
void idStrId::Set( const char* key )
{
	if( key == NULL || key[0] == 0 )
	{
		index = -1;
	}
	else
	{
		index = idLocalization::languageDict.FindStringIndex( key );
		if( index < 0 )
		{
			// don't allow setting of string ID's to an unknown ID... this should only be allowed from
			// the string table tool because additions from anywhere else are not guaranteed to be
			// saved to the .lang file.
			idLib::Warning( "Attempted to set unknown string ID '%s'", key );
		}
	}
}

/*
========================
idStrId::GetKey
========================
*/
const char* idStrId::GetKey() const
{
	if( index >= 0 && index < idLocalization::languageDict.keyVals.Num() )
	{
		return idLocalization::languageDict.keyVals[index].key;
	}
	return "";
}

/*
========================
idStrId::GetLocalizedString
========================
*/
const char* idStrId::GetLocalizedString() const
{
	return idLocalization::languageDict.GetLocalizedString( *this );
}