#include "precompiled.h" #pragma hdrstop #ifdef LEXER_READ_AHEAD namespace { #define IO_BUFFER_CHUNK_SIZE (1024 * 64) enum { IOJobPending = 0, IOJobQuit, IOJobComplete, IOJobCount }; static HANDLE mIOJobs[IOJobCount]; static DWORD mThreadId = 0; static HANDLE mThreadHandle = 0; struct IOJobData { idFile *mFile; int mAmtToRead; byte mBuffer[IO_BUFFER_CHUNK_SIZE]; }; IOJobData gIOJobData; void SetPendingJob(idFile *f, int size) { gIOJobData.mFile = f; gIOJobData.mAmtToRead = size; if ( mThreadHandle ) { SetEvent( mIOJobs[IOJobPending] ); } } IOJobData *GetPendingJob() { return &gIOJobData; } void GetPendingJobResults(byte *dest, int &size) { if ( mThreadHandle ) { if ( WAIT_OBJECT_0 == WaitForSingleObject( mIOJobs[IOJobComplete], INFINITE ) ) { //ResetEvent( mIOJobs[IOJobComplete] ); size = gIOJobData.mAmtToRead; memcpy(dest, gIOJobData.mBuffer, size ); } else { assert( 0 ); } } else { gIOJobData.mFile->Read( dest, gIOJobData.mAmtToRead ); size = gIOJobData.mAmtToRead; } } static LONG IOThread( void * ) { while ( 1 ) { DWORD res = WaitForMultipleObjects( 2, mIOJobs, FALSE, INFINITE ); if ( res == WAIT_OBJECT_0 ) { //ResetEvent( mIOJobs[IOJobPending] ); IOJobData *j = GetPendingJob(); j->mFile->Read( j->mBuffer, j->mAmtToRead ); SetEvent( mIOJobs[IOJobComplete] ); } else if ( res == (WAIT_OBJECT_0 + 1) ) { ExitThread(0); } else { assert( 0 ); } } ExitThread(0); } void InitIO() { for ( int i = 0; i < IOJobCount; ++i ) { //mIOJobs[i] = CreateEvent( 0, TRUE, FALSE, 0 ); mIOJobs[i] = CreateEvent( 0, FALSE, FALSE, 0 ); } mThreadHandle = CreateThread( NULL, 1024 * 512, ( LPTHREAD_START_ROUTINE )IOThread, 0, CREATE_SUSPENDED, &mThreadId ); ResumeThread( mThreadHandle ); } void ShutdownIO() { SetEvent( mIOJobs[IOJobQuit] ); DWORD ecode = 0; while ( !GetExitCodeThread( mThreadHandle, &ecode ) ) { Sleep( 1 ); } CloseHandle( mThreadHandle ); for ( int i = 0; i < IOJobCount; ++i ) { CloseHandle( mIOJobs[i] ); } mThreadHandle = 0; } } namespace ChunkBuffer { idFile *mFile; int mFileSize; int mBytesRead; int mFileBytesRead; byte mChunkMem[IO_BUFFER_CHUNK_SIZE]; int mCurrentChunkBytes; int mCurrentChunkBytesRead = 0; /* ================ ChunkBuffer::Length ================ */ int Length() { return mFileSize; } /* ================ ChunkBuffer::BytesRead ================ */ int BytesRead() { return mBytesRead; } /* ================ ChunkBuffer::Begin ================ */ void Begin( idFile *f ) { mFile = f; mFileSize = f->Length(); mBytesRead = 0; int initialRead = (mFileSize < IO_BUFFER_CHUNK_SIZE) ? mFileSize : IO_BUFFER_CHUNK_SIZE; mFile->Read( mChunkMem, initialRead ); mFileBytesRead = initialRead; mCurrentChunkBytesRead = 0; mCurrentChunkBytes = initialRead; int bytesRemaining = mFileSize - mFileBytesRead; if ( bytesRemaining > 0 ) { SetPendingJob( mFile, (bytesRemaining > IO_BUFFER_CHUNK_SIZE) ? IO_BUFFER_CHUNK_SIZE : bytesRemaining ); } } /* ================ ChunkBuffer::End ================ */ void End() { } /* ================ ChunkBuffer::Advance ================ */ int Advance( byte *chunkMem ) { int readAmt = 0; GetPendingJobResults( chunkMem, readAmt ); mFileBytesRead+=readAmt; int bytesRemaining = mFileSize - mFileBytesRead; if ( bytesRemaining > 0 ) { SetPendingJob( mFile, (bytesRemaining > IO_BUFFER_CHUNK_SIZE) ? IO_BUFFER_CHUNK_SIZE : bytesRemaining ); } return readAmt; } /* ================ ChunkBuffer::Read ================ */ int Read( byte *dest, int size ) { int bytesRem = mFileSize - mBytesRead; if ( size > bytesRem ) { size = bytesRem; } if ( !size ) { return 0; } byte *writePos = dest; int cnt = size; while ( cnt ) { if ( mCurrentChunkBytesRead >= mCurrentChunkBytes ) { mCurrentChunkBytes = Advance( mChunkMem ); mCurrentChunkBytesRead = 0; } int chunkBytesRemaining = mCurrentChunkBytes - mCurrentChunkBytesRead; int toRead = (cnt > chunkBytesRemaining) ? chunkBytesRemaining : cnt; byte *readPos = &mChunkMem[mCurrentChunkBytesRead]; memcpy( writePos, readPos, toRead ); cnt-=toRead; mCurrentChunkBytesRead+=toRead; writePos+=toRead; } mBytesRead+=size; return size; } } /* ================ LexerIOWrapper::LexerIOWrapper ================ */ LexerIOWrapper::LexerIOWrapper() { file = 0; offset = 0; size = 0; memory = 0; } /* ================ LexerIOWrapper::InitFromMemory ================ */ void LexerIOWrapper::InitFromMemory(char const * const ptr, int length) { file = 0; offset = 0; size = length; memory = (char *)ptr; } /* ================ LexerIOWrapper::OpenFile ================ */ bool LexerIOWrapper::OpenFile(char const *filename, bool OSPath) { if ( OSPath ) { file = idLib::fileSystem->OpenExplicitFileRead( filename ); } else { file = idLib::fileSystem->OpenFileRead( filename ); } if ( !file ) { return false; } ChunkBuffer::Begin( file ); return true; } /* ================ LexerIOWrapper::Length ================ */ int LexerIOWrapper::Length() { if ( file ) { return ChunkBuffer::Length(); } return size; } /* ================ LexerIOWrapper::Tell ================ */ int LexerIOWrapper::Tell() { if ( file ) { return ChunkBuffer::BytesRead(); } return offset; } /* ================ LexerIOWrapper::Seek ================ */ void LexerIOWrapper::Seek( int loc, int mode ) { if ( file ) { if ( loc != 0 || mode != FS_SEEK_SET ) { common->Error( "lol\n" ); } file->Seek( 0, FS_SEEK_SET ); ChunkBuffer::Begin( file ); return; } if ( mode == FS_SEEK_SET ) { offset = loc; } else { common->Error( "lol" ); } } /* ================ LexerIOWrapper::Read ================ */ void LexerIOWrapper::Read( void *dest, int s ) { if ( file ) { ChunkBuffer::Read( (byte*)dest, s ); return; } char *d = (char *)dest; char *dEnd = d + (((s + offset) > size) ? (size - offset) : (s)); while ( d < dEnd ) { *d = memory[offset]; ++d; ++offset; } } /* ================ LexerIOWrapper::Close ================ */ void LexerIOWrapper::Close() { if ( file ) { ChunkBuffer::End(); idLib::fileSystem->CloseFile(file); return; } } /* ================ LexerIOWrapper::IsLoaded ================ */ int LexerIOWrapper::IsLoaded() { if ( file || memory ) { return 1; } return 0; } /* ================ Lexer::BeginLevelLoad ================ */ void Lexer::BeginLevelLoad() { InitIO(); } /* ================ Lexer::EndLevelLoad ================ */ void Lexer::EndLevelLoad() { ShutdownIO(); } /* ================ ReadValue ================ */ template inline type ReadValue(LexerIOWrapper *in) { type ret; in->Read(&ret, sizeof(type)); return ret; } /* ================ ReadValue specialization read for idStr's ================ */ template <> inline idStr ReadValue(LexerIOWrapper *in) { char c; idStr str; in->Read(&c, 1); while(c != '\0') { str.Append(c); in->Read(&c, 1); } str.Append(c); return str; } #endif #define PUNCTABLE //longer punctuations first punctuation_t default_punctuations[] = { //binary operators {">>=",P_RSHIFT_ASSIGN}, {"<<=",P_LSHIFT_ASSIGN}, // {"...",P_PARMS}, //define merge operator {"##",P_PRECOMPMERGE}, // pre-compiler //logic operators {"&&",P_LOGIC_AND}, // pre-compiler {"||",P_LOGIC_OR}, // pre-compiler {">=",P_LOGIC_GEQ}, // pre-compiler {"<=",P_LOGIC_LEQ}, // pre-compiler {"==",P_LOGIC_EQ}, // pre-compiler {"!=",P_LOGIC_UNEQ}, // pre-compiler //arithmatic operators {"*=",P_MUL_ASSIGN}, {"/=",P_DIV_ASSIGN}, {"%=",P_MOD_ASSIGN}, {"+=",P_ADD_ASSIGN}, {"-=",P_SUB_ASSIGN}, {"++",P_INC}, {"--",P_DEC}, //binary operators {"&=",P_BIN_AND_ASSIGN}, {"|=",P_BIN_OR_ASSIGN}, {"^=",P_BIN_XOR_ASSIGN}, {">>",P_RSHIFT}, // pre-compiler {"<<",P_LSHIFT}, // pre-compiler //reference operators {"->",P_POINTERREF}, //C++ {"::",P_CPP1}, {".*",P_CPP2}, //arithmatic operators {"*",P_MUL}, // pre-compiler {"/",P_DIV}, // pre-compiler {"%",P_MOD}, // pre-compiler {"+",P_ADD}, // pre-compiler {"-",P_SUB}, // pre-compiler {"=",P_ASSIGN}, //binary operators {"&",P_BIN_AND}, // pre-compiler {"|",P_BIN_OR}, // pre-compiler {"^",P_BIN_XOR}, // pre-compiler {"~",P_BIN_NOT}, // pre-compiler //logic operators {"!",P_LOGIC_NOT}, // pre-compiler {">",P_LOGIC_GREATER}, // pre-compiler {"<",P_LOGIC_LESS}, // pre-compiler //reference operator {".",P_REF}, //seperators {",",P_COMMA}, // pre-compiler {";",P_SEMICOLON}, //label indication {":",P_COLON}, // pre-compiler //if statement {"?",P_QUESTIONMARK}, // pre-compiler //embracements {"(",P_PARENTHESESOPEN}, // pre-compiler {")",P_PARENTHESESCLOSE}, // pre-compiler {"{",P_BRACEOPEN}, // pre-compiler {"}",P_BRACECLOSE}, // pre-compiler {"[",P_SQBRACKETOPEN}, {"]",P_SQBRACKETCLOSE}, // {"\\",P_BACKSLASH}, //precompiler operator {"#",P_PRECOMP}, // pre-compiler {"$",P_DOLLAR}, // RAVEN BEGIN {"¡",P_INVERTED_PLING}, {"¿",P_INVERTED_QUERY}, // RAVEN END {NULL, 0} }; int default_punctuationtable[256]; int default_nextpunctuation[sizeof(default_punctuations) / sizeof(punctuation_t)]; int default_setup; // RAVEN BEGIN // jsinger: changed to be Lexer instead of idLexer so that we have the ability to read binary files char Lexer::baseFolder[ 256 ]; // Added this to allow easy changing of the suffix that signifies a binary file idStr const Lexer::sCompiledFileSuffix("c"); // RAVEN END /* ================ idLexer::CreatePunctuationTable ================ */ void idLexer::CreatePunctuationTable( const punctuation_t *punctuations ) { int i, n, lastp; const punctuation_t *p, *newp; //get memory for the table if ( punctuations == default_punctuations ) { idLexer::punctuationtable = default_punctuationtable; idLexer::nextpunctuation = default_nextpunctuation; if ( default_setup ) { return; } default_setup = true; i = sizeof(default_punctuations) / sizeof(punctuation_t); } else { if ( !idLexer::punctuationtable || idLexer::punctuationtable == default_punctuationtable ) { //RAVEN BEGIN //amccarthy: Added memory allocation tag idLexer::punctuationtable = (int *) Mem_Alloc(256 * sizeof(int),MA_LEXER); //RAVEN END } if ( idLexer::nextpunctuation && idLexer::nextpunctuation != default_nextpunctuation ) { Mem_Free( idLexer::nextpunctuation ); } for (i = 0; punctuations[i].p; i++) { } //RAVEN BEGIN //amccarthy: Added memory allocation tag idLexer::nextpunctuation = (int *) Mem_Alloc(i * sizeof(int),MA_LEXER); //RAVEN END } memset(idLexer::punctuationtable, 0xFF, 256 * sizeof(int)); memset(idLexer::nextpunctuation, 0xFF, i * sizeof(int)); //add the punctuations in the list to the punctuation table for (i = 0; punctuations[i].p; i++) { newp = &punctuations[i]; lastp = -1; //sort the punctuations in this table entry on length (longer punctuations first) for (n = idLexer::punctuationtable[(unsigned int) newp->p[0]]; n >= 0; n = idLexer::nextpunctuation[n] ) { p = &punctuations[n]; if (strlen(p->p) < strlen(newp->p)) { idLexer::nextpunctuation[i] = n; if (lastp >= 0) { idLexer::nextpunctuation[lastp] = i; } else { idLexer::punctuationtable[(unsigned int) newp->p[0]] = i; } break; } lastp = n; } if (n < 0) { idLexer::nextpunctuation[i] = -1; if (lastp >= 0) { idLexer::nextpunctuation[lastp] = i; } else { idLexer::punctuationtable[(unsigned int) newp->p[0]] = i; } } } } /* ================ idLexer::GetPunctuationFromId ================ */ const char *idLexer::GetPunctuationFromId( int id ) { int i; for (i = 0; idLexer::punctuations[i].p; i++) { if ( idLexer::punctuations[i].n == id ) { return idLexer::punctuations[i].p; } } return "unkown punctuation"; } /* ================ idLexer::GetPunctuationId ================ */ int idLexer::GetPunctuationId( const char *p ) { int i; for (i = 0; idLexer::punctuations[i].p; i++) { if ( !idStr::Cmp(idLexer::punctuations[i].p, p) ) { return idLexer::punctuations[i].n; } } return 0; } /* ================ idLexer::Error ================ */ void idLexer::Error( const char *str, ... ) { char text[MAX_STRING_CHARS]; va_list ap; hadError = true; if ( idLexer::flags & LEXFL_NOERRORS ) { return; } va_start(ap, str); vsprintf(text, str, ap); va_end(ap); if ( idLexer::flags & LEXFL_NOFATALERRORS ) { idLib::common->Warning( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text ); } else { idLib::common->Error( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text ); } } /* ================ idLexer::Warning ================ */ void idLexer::Warning( const char *str, ... ) { char text[MAX_STRING_CHARS]; va_list ap; if ( idLexer::flags & LEXFL_NOWARNINGS ) { return; } va_start( ap, str ); vsprintf( text, str, ap ); va_end( ap ); idLib::common->Warning( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text ); } /* ================ idLexer::SetPunctuations ================ */ void idLexer::SetPunctuations( const punctuation_t *p ) { #ifdef PUNCTABLE if (p) { idLexer::CreatePunctuationTable( p ); } else { idLexer::CreatePunctuationTable( default_punctuations ); } #endif //PUNCTABLE if (p) { idLexer::punctuations = p; } else { idLexer::punctuations = default_punctuations; } } /* ================ idLexer::ReadWhiteSpace Reads spaces, tabs, C-like comments etc. When a newline character is found the scripts line counter is increased. ================ */ int idLexer::ReadWhiteSpace( void ) { while(1) { // skip white space // RAVEN BEGIN while(( byte )*idLexer::script_p <= ' ') { // RAVEN END if (!*idLexer::script_p) { return 0; } if (*idLexer::script_p == '\n') { idLexer::line++; } idLexer::script_p++; } // skip comments if (*idLexer::script_p == '/') { // comments // if (*(idLexer::script_p+1) == '/') { idLexer::script_p++; do { idLexer::script_p++; if ( !*idLexer::script_p ) { return 0; } } while( *idLexer::script_p != '\n' ); idLexer::line++; idLexer::script_p++; if ( !*idLexer::script_p ) { return 0; } continue; } // comments /* */ else if (*(idLexer::script_p+1) == '*') { idLexer::script_p++; while( 1 ) { idLexer::script_p++; if ( !*idLexer::script_p ) { return 0; } if ( *idLexer::script_p == '\n' ) { idLexer::line++; } else if ( *idLexer::script_p == '/' ) { if ( *(idLexer::script_p-1) == '*' ) { break; } if ( *(idLexer::script_p+1) == '*' ) { idLexer::Warning( "nested comment" ); } } } idLexer::script_p++; if ( !*idLexer::script_p ) { return 0; } idLexer::script_p++; if ( !*idLexer::script_p ) { return 0; } continue; } } break; } return 1; } /* ================ idLexer::ReadEscapeCharacter ================ */ int idLexer::ReadEscapeCharacter( char *ch ) { int c, val, i; // step over the leading '\\' idLexer::script_p++; // determine the escape character switch(*idLexer::script_p) { case '\\': c = '\\'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\v'; break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'a': c = '\a'; break; case '\'': c = '\''; break; case '\"': c = '\"'; break; case '\?': c = '\?'; break; case 'x': { idLexer::script_p++; for (i = 0, val = 0; ; i++, idLexer::script_p++) { c = *idLexer::script_p; if (c >= '0' && c <= '9') c = c - '0'; else if (c >= 'A' && c <= 'Z') c = c - 'A' + 10; else if (c >= 'a' && c <= 'z') c = c - 'a' + 10; else break; val = (val << 4) + c; } idLexer::script_p--; if (val > 0xFF) { idLexer::Warning( "too large value in escape character" ); val = 0xFF; } c = val; break; } default: //NOTE: decimal ASCII code, NOT octal { if (*idLexer::script_p < '0' || *idLexer::script_p > '9') { idLexer::Error("unknown escape char"); } for (i = 0, val = 0; ; i++, idLexer::script_p++) { c = *idLexer::script_p; if (c >= '0' && c <= '9') c = c - '0'; else break; val = val * 10 + c; } idLexer::script_p--; if (val > 0xFF) { idLexer::Warning( "too large value in escape character" ); val = 0xFF; } c = val; break; } } // step over the escape character or the last digit of the number idLexer::script_p++; // store the escape character *ch = c; // succesfully read escape character return 1; } /* ================ idLexer::ReadString Escape characters are interpretted. Reads two strings with only a white space between them as one string. ================ */ int idLexer::ReadString( idToken *token, int quote ) { int tmpline; const char *tmpscript_p; char ch; if ( quote == '\"' ) { token->type = TT_STRING; } else { token->type = TT_LITERAL; } // leading quote idLexer::script_p++; while(1) { // if there is an escape character and escape characters are allowed if (*idLexer::script_p == '\\' && !(idLexer::flags & LEXFL_NOSTRINGESCAPECHARS)) { if ( !idLexer::ReadEscapeCharacter( &ch ) ) { return 0; } token->AppendDirty( ch ); } // if a trailing quote else if (*idLexer::script_p == quote) { // step over the quote idLexer::script_p++; // if consecutive strings should not be concatenated if ( (idLexer::flags & LEXFL_NOSTRINGCONCAT) && (!(idLexer::flags & LEXFL_ALLOWBACKSLASHSTRINGCONCAT) || (quote != '\"')) ) { break; } tmpscript_p = idLexer::script_p; tmpline = idLexer::line; // read white space between possible two consecutive strings if ( !idLexer::ReadWhiteSpace() ) { idLexer::script_p = tmpscript_p; idLexer::line = tmpline; break; } if ( idLexer::flags & LEXFL_NOSTRINGCONCAT ) { if ( *idLexer::script_p != '\\' ) { idLexer::script_p = tmpscript_p; idLexer::line = tmpline; break; } // step over the '\\' idLexer::script_p++; if ( !idLexer::ReadWhiteSpace() || ( *idLexer::script_p != quote ) ) { idLexer::Error( "expecting string after '\' terminated line" ); return 0; } } // if there's no leading qoute if ( *idLexer::script_p != quote ) { idLexer::script_p = tmpscript_p; idLexer::line = tmpline; break; } // step over the new leading quote idLexer::script_p++; } else { if (*idLexer::script_p == '\0') { idLexer::Error( "missing trailing quote" ); return 0; } if (*idLexer::script_p == '\n') { idLexer::Error( "newline inside string" ); return 0; } token->AppendDirty( *idLexer::script_p++ ); } } token->data[token->len] = '\0'; if ( token->type == TT_LITERAL ) { if ( !(idLexer::flags & LEXFL_ALLOWMULTICHARLITERALS) ) { if ( token->Length() != 1 ) { idLexer::Warning( "literal is not one character long" ); } } token->subtype = (*token)[0]; } else { // the sub type is the length of the string token->subtype = token->Length(); } return 1; } /* ================ idLexer::ReadName ================ */ int idLexer::ReadName( idToken *token ) { char c; token->type = TT_NAME; do { token->AppendDirty( *idLexer::script_p++ ); c = *idLexer::script_p; // RAVEN BEGIN } while ( idStr::CharIsAlpha( c ) || idStr::CharIsNumeric( c ) || c == '_' || // RAVEN EBD // if treating all tokens as strings, don't parse '-' as a seperate token ((idLexer::flags & LEXFL_ONLYSTRINGS) && (c == '-')) || // if special path name characters are allowed ((idLexer::flags & LEXFL_ALLOWPATHNAMES) && (c == '/' || c == '\\' || c == ':' || c == '.')) ); token->data[token->len] = '\0'; //the sub type is the length of the name token->subtype = token->Length(); return 1; } /* ================ idLexer::CheckString ================ */ ID_INLINE int idLexer::CheckString( const char *str ) const { int i; for ( i = 0; str[i]; i++ ) { if ( idLexer::script_p[i] != str[i] ) { return false; } } return true; } /* ================ idLexer::ReadNumber ================ */ int idLexer::ReadNumber( idToken *token ) { int i; int dot; char c, c2; token->type = TT_NUMBER; token->subtype = 0; token->intvalue = 0; token->floatvalue = 0; c = *idLexer::script_p; c2 = *(idLexer::script_p + 1); if ( c == '0' && c2 != '.' ) { // check for a hexadecimal number if ( c2 == 'x' || c2 == 'X' ) { token->AppendDirty( *idLexer::script_p++ ); token->AppendDirty( *idLexer::script_p++ ); c = *idLexer::script_p; while((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { token->AppendDirty( c ); c = *(++idLexer::script_p); } token->subtype = TT_HEX | TT_INTEGER; } // check for a binary number else if ( c2 == 'b' || c2 == 'B' ) { token->AppendDirty( *idLexer::script_p++ ); token->AppendDirty( *idLexer::script_p++ ); c = *idLexer::script_p; while( c == '0' || c == '1' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } token->subtype = TT_BINARY | TT_INTEGER; } // its an octal number else { token->AppendDirty( *idLexer::script_p++ ); c = *idLexer::script_p; while( c >= '0' && c <= '7' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } token->subtype = TT_OCTAL | TT_INTEGER; } } else { // decimal integer or floating point number or ip address dot = 0; while( 1 ) { if ( c >= '0' && c <= '9' ) { } else if ( c == '.' ) { dot++; } else { break; } token->AppendDirty( c ); c = *(++idLexer::script_p); } if( c == 'e' && dot == 0) { //We have scientific notation without a decimal point dot++; } // if a floating point number if ( dot == 1 ) { token->subtype = TT_DECIMAL | TT_FLOAT; // check for floating point exponent if ( c == 'e' ) { //Append the e so that GetFloatValue code works token->AppendDirty( c ); c = *(++idLexer::script_p); if ( c == '-' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } else if ( c == '+' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } while( c >= '0' && c <= '9' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } } // check for floating point exception infinite 1.#INF or indefinite 1.#IND or NaN else if ( c == '#' ) { c2 = 4; if ( CheckString( "INF" ) ) { token->subtype |= TT_INFINITE; } else if ( CheckString( "IND" ) ) { token->subtype |= TT_INDEFINITE; } else if ( CheckString( "NAN" ) ) { token->subtype |= TT_NAN; } else if ( CheckString( "QNAN" ) ) { token->subtype |= TT_NAN; c2++; } else if ( CheckString( "SNAN" ) ) { token->subtype |= TT_NAN; c2++; } for ( i = 0; i < c2; i++ ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } while( c >= '0' && c <= '9' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } if ( !(idLexer::flags & LEXFL_ALLOWFLOATEXCEPTIONS) ) { token->AppendDirty( 0 ); // zero terminate for c_str idLexer::Error( "parsed %s", token->c_str() ); } } } else if ( dot > 1 ) { if ( !( idLexer::flags & LEXFL_ALLOWIPADDRESSES ) ) { idLexer::Error( "more than one dot in number" ); return 0; } if ( dot != 3 ) { idLexer::Error( "ip address should have three dots" ); return 0; } token->subtype = TT_IPADDRESS; } else { token->subtype = TT_DECIMAL | TT_INTEGER; } } if ( token->subtype & TT_FLOAT ) { if ( c > ' ' ) { // single-precision: float if ( c == 'f' || c == 'F' ) { token->subtype |= TT_SINGLE_PRECISION; idLexer::script_p++; } // extended-precision: long double else if ( c == 'l' || c == 'L' ) { token->subtype |= TT_EXTENDED_PRECISION; idLexer::script_p++; } // default is double-precision: double else { token->subtype |= TT_DOUBLE_PRECISION; } } else { token->subtype |= TT_DOUBLE_PRECISION; } } else if ( token->subtype & TT_INTEGER ) { if ( c > ' ' ) { // default: signed long for ( i = 0; i < 2; i++ ) { // long integer if ( c == 'l' || c == 'L' ) { token->subtype |= TT_LONG; } // unsigned integer else if ( c == 'u' || c == 'U' ) { token->subtype |= TT_UNSIGNED; } else { break; } c = *(++idLexer::script_p); } } } else if ( token->subtype & TT_IPADDRESS ) { if ( c == ':' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); while( c >= '0' && c <= '9' ) { token->AppendDirty( c ); c = *(++idLexer::script_p); } token->subtype |= TT_IPPORT; } } token->data[token->len] = '\0'; return 1; } /* ================ idLexer::ReadPunctuation ================ */ int idLexer::ReadPunctuation( idToken *token ) { int l, n, i; char *p; const punctuation_t *punc; #ifdef PUNCTABLE for (n = idLexer::punctuationtable[(unsigned int)*(idLexer::script_p)]; n >= 0; n = idLexer::nextpunctuation[n]) { punc = &(idLexer::punctuations[n]); #else int i; for (i = 0; idLexer::punctuations[i].p; i++) { punc = &idLexer::punctuations[i]; #endif p = punc->p; // check for this punctuation in the script for ( l = 0; p[l] && idLexer::script_p[l]; l++ ) { if ( idLexer::script_p[l] != p[l] ) { break; } } if ( !p[l] ) { // token->EnsureAlloced( l+1, false ); for ( i = 0; i <= l; i++ ) { token->data[i] = p[i]; } token->len = l; // idLexer::script_p += l; token->type = TT_PUNCTUATION; // sub type is the punctuation id token->subtype = punc->n; return 1; } } return 0; } /* ================ idLexer::ReadToken ================ */ int idLexer::ReadToken( idToken *token ) { int c; if ( !loaded ) { // RAVEN BEGIN #ifndef _XENON // nrausch: should not be an error on xenon since it prevents the precache stuff from working idLib::common->Error( "idLexer::ReadToken: no file loaded" ); #else idLib::common->Warning( "idLexer::ReadToken: no file loaded" ); #endif // RAVEN END return 0; } // if there is a token available (from unreadToken) if ( tokenavailable ) { tokenavailable = 0; *token = idLexer::token; return 1; } // save script pointer lastScript_p = script_p; // save line counter lastline = line; // clear the token stuff token->data[0] = '\0'; token->len = 0; // start of the white space whiteSpaceStart_p = script_p; token->whiteSpaceStart_p = script_p; // read white space before token if ( !ReadWhiteSpace() ) { return 0; } // end of the white space idLexer::whiteSpaceEnd_p = script_p; token->whiteSpaceEnd_p = script_p; // line the token is on token->line = line; // number of lines crossed before token token->linesCrossed = line - lastline; // clear token flags token->flags = 0; c = *idLexer::script_p; // if we're keeping everything as whitespace deliminated strings if ( idLexer::flags & LEXFL_ONLYSTRINGS ) { // if there is a leading quote if ( c == '\"' || c == '\'' ) { if (!idLexer::ReadString( token, c )) { return 0; } } else if ( !idLexer::ReadName( token ) ) { return 0; } } // if there is a number else if ( (c >= '0' && c <= '9') || (c == '.' && (*(idLexer::script_p + 1) >= '0' && *(idLexer::script_p + 1) <= '9')) ) { if ( !idLexer::ReadNumber( token ) ) { return 0; } // if names are allowed to start with a number if ( idLexer::flags & LEXFL_ALLOWNUMBERNAMES ) { c = *idLexer::script_p; if ( idStr::CharIsAlpha( c ) || idStr::CharIsNumeric( c ) || c == '_' ) { if ( !idLexer::ReadName( token ) ) { return 0; } } } } // if there is a leading quote else if ( c == '\"' || c == '\'' ) { if (!idLexer::ReadString( token, c )) { return 0; } } // if there is a name // RAVEN BEGIN else if ( idStr::CharIsAlpha( c ) || idStr::CharIsNumeric( c ) || c == '_' ) { // RAVEN END if ( !idLexer::ReadName( token ) ) { return 0; } } // names may also start with a slash when pathnames are allowed else if ( ( idLexer::flags & LEXFL_ALLOWPATHNAMES ) && ( (c == '/' || c == '\\') || c == '.' ) ) { if ( !idLexer::ReadName( token ) ) { return 0; } } // check for punctuations else if ( !idLexer::ReadPunctuation( token ) ) { idLexer::Error( "unknown punctuation %c", c ); return 0; } // RAVEN BEGIN // jsinger: added to write out the binary version of this token when binary writes have been turned on WriteBinaryToken(token); // RAVEN END // succesfully read a token return 1; } /* ================ idLexer::ExpectTokenString ================ */ int idLexer::ExpectTokenString( const char *string ) { idToken token; if (!idLexer::ReadToken( &token )) { idLexer::Error( "couldn't find expected '%s'", string ); return 0; } if ( token != string ) { idLexer::Error( "expected '%s' but found '%s'", string, token.c_str() ); return 0; } return 1; } /* ================ idLexer::ExpectTokenType ================ */ int idLexer::ExpectTokenType( int type, int subtype, idToken *token ) { idStr str; if ( !idLexer::ReadToken( token ) ) { idLexer::Error( "couldn't read expected token" ); return 0; } if ( token->type != type ) { switch( type ) { case TT_STRING: str = "string"; break; case TT_LITERAL: str = "literal"; break; case TT_NUMBER: str = "number"; break; case TT_NAME: str = "name"; break; case TT_PUNCTUATION: str = "punctuation"; break; default: str = "unknown type"; break; } idLexer::Error( "expected a %s but found '%s'", str.c_str(), token->c_str() ); return 0; } if ( token->type == TT_NUMBER ) { if ( (token->subtype & subtype) != subtype ) { str.Clear(); if ( subtype & TT_DECIMAL ) str = "decimal "; if ( subtype & TT_HEX ) str = "hex "; if ( subtype & TT_OCTAL ) str = "octal "; if ( subtype & TT_BINARY ) str = "binary "; if ( subtype & TT_UNSIGNED ) str += "unsigned "; if ( subtype & TT_LONG ) str += "long "; if ( subtype & TT_FLOAT ) str += "float "; if ( subtype & TT_INTEGER ) str += "integer "; str.StripTrailing( ' ' ); idLexer::Error( "expected %s but found '%s'", str.c_str(), token->c_str() ); return 0; } } else if ( token->type == TT_PUNCTUATION ) { if ( subtype < 0 ) { idLexer::Error( "BUG: wrong punctuation subtype" ); return 0; } if ( token->subtype != subtype ) { idLexer::Error( "expected '%s' but found '%s'", GetPunctuationFromId( subtype ), token->c_str() ); return 0; } } return 1; } /* ================ idLexer::ExpectAnyToken ================ */ int idLexer::ExpectAnyToken( idToken *token ) { if (!idLexer::ReadToken( token )) { idLexer::Error( "couldn't read expected token" ); return 0; } else { return 1; } } /* ================ idLexer::CheckTokenString ================ */ int idLexer::CheckTokenString( const char *string ) { idToken tok; if (!idLexer::ReadToken( &tok )) { return 0; } // if the token is available if ( tok == string ) { return 1; } // token not available UnreadToken( &tok ); return 0; } /* ================ idLexer::CheckTokenType ================ */ int idLexer::CheckTokenType( int type, int subtype, idToken *token ) { idToken tok; if (!idLexer::ReadToken( &tok )) { return 0; } // if the type matches if (tok.type == type && (tok.subtype & subtype) == subtype) { *token = tok; return 1; } // token is not available idLexer::script_p = lastScript_p; idLexer::line = lastline; return 0; } /* ================ idLexer::SkipUntilString ================ */ int idLexer::SkipUntilString( const char *string ) { idToken token; while(idLexer::ReadToken( &token )) { if ( token == string ) { return 1; } } return 0; } /* ================ idLexer::SkipRestOfLine ================ */ int idLexer::SkipRestOfLine( void ) { idToken token; while(idLexer::ReadToken( &token )) { if ( token.linesCrossed ) { idLexer::script_p = lastScript_p; idLexer::line = lastline; return 1; } } return 0; } /* ================= idLexer::SkipBracedSection Skips until a matching close brace is found. Internal brace depths are properly skipped. ================= */ int idLexer::SkipBracedSection( bool parseFirstBrace ) { idToken token; int depth; depth = parseFirstBrace ? 0 : 1; do { if ( !ReadToken( &token ) ) { return false; } if ( token.type == TT_PUNCTUATION ) { if ( token == "{" ) { depth++; } else if ( token == "}" ) { depth--; } } } while( depth ); return true; } /* ================ idLexer::UnreadToken ================ */ void idLexer::UnreadToken( const idToken *token ) { if ( idLexer::tokenavailable ) { idLib::common->FatalError( "idLexer::unreadToken, unread token twice\n" ); } idLexer::token = *token; idLexer::tokenavailable = 1; } /* ================ idLexer::ReadTokenOnLine ================ */ int idLexer::ReadTokenOnLine( idToken *token ) { idToken tok; if (!idLexer::ReadToken( &tok )) { idLexer::script_p = lastScript_p; idLexer::line = lastline; return false; } // if no lines were crossed before this token if ( !tok.linesCrossed ) { *token = tok; return true; } // restore our position idLexer::script_p = lastScript_p; idLexer::line = lastline; token->Clear(); return false; } /* ================ idLexer::ReadRestOfLine ================ */ const char* idLexer::ReadRestOfLine(idStr& out) { while(1) { if(*idLexer::script_p == '\n') { idLexer::line++; break; } if(!*idLexer::script_p) { break; } if(*idLexer::script_p <= ' ') { out += " "; } else { out += *idLexer::script_p; } idLexer::script_p++; } out.Strip(' '); return out.c_str(); } /* ================ idLexer::ParseInt ================ */ int idLexer::ParseInt( void ) { idToken token; if ( !idLexer::ReadToken( &token ) ) { idLexer::Error( "couldn't read expected integer" ); return 0; } if ( token.type == TT_PUNCTUATION && token == "-" ) { idLexer::ExpectTokenType( TT_NUMBER, TT_INTEGER, &token ); return -((signed int) token.GetIntValue()); } else if ( token.type != TT_NUMBER || token.subtype == TT_FLOAT ) { idLexer::Error( "expected integer value, found '%s'", token.c_str() ); } return token.GetIntValue(); } /* ================ idLexer::ParseBool ================ */ bool idLexer::ParseBool( void ) { idToken token; if ( !idLexer::ExpectTokenType( TT_NUMBER, 0, &token ) ) { idLexer::Error( "couldn't read expected boolean" ); return false; } return ( token.GetIntValue() != 0 ); } /* ================ idLexer::ParseFloat ================ */ float idLexer::ParseFloat( bool *errorFlag ) { idToken token; if ( errorFlag ) { *errorFlag = false; } if ( !idLexer::ReadToken( &token ) ) { if ( errorFlag ) { idLexer::Warning( "couldn't read expected floating point number" ); *errorFlag = true; } else { idLexer::Error( "couldn't read expected floating point number" ); } return 0; } if ( token.type == TT_PUNCTUATION && token == "-" ) { idLexer::ExpectTokenType( TT_NUMBER, 0, &token ); return -token.GetFloatValue(); } else if ( token.type != TT_NUMBER ) { if ( errorFlag ) { idLexer::Warning( "expected float value, found '%s'", token.c_str() ); *errorFlag = true; } else { idLexer::Error( "expected float value, found '%s'", token.c_str() ); } } return token.GetFloatValue(); } /* ================ idLexer::Parse1DMatrix ================ */ int idLexer::Parse1DMatrix( int x, float *m ) { int i; if ( !idLexer::ExpectTokenString( "(" ) ) { return false; } for ( i = 0; i < x; i++ ) { m[i] = idLexer::ParseFloat(); } if ( !idLexer::ExpectTokenString( ")" ) ) { return false; } return true; } // RAVEN BEGIN // rjohnson: added vertex color support to proc files. assume a default RGBA of 0x000000ff /* ================ idLexer::Parse1DMatrixOpenEnded ================ */ int idLexer::Parse1DMatrixOpenEnded( int MaxCount, float *m ) { int i; if ( !idLexer::ExpectTokenString( "(" ) ) { return 0; } for ( i = 0; i < MaxCount; i++ ) { idToken tok; if (!idLexer::ReadToken( &tok )) { return 0; } if ( tok == ")" ) { return i; } idLexer::UnreadToken( &tok ); m[i] = idLexer::ParseFloat(); } if ( !idLexer::ExpectTokenString( ")" ) ) { return 0; } return i; } // RAVEN END /* ================ idLexer::Parse2DMatrix ================ */ int idLexer::Parse2DMatrix( int y, int x, float *m ) { int i; if ( !idLexer::ExpectTokenString( "(" ) ) { return false; } for ( i = 0; i < y; i++ ) { if ( !idLexer::Parse1DMatrix( x, m + i * x ) ) { return false; } } if ( !idLexer::ExpectTokenString( ")" ) ) { return false; } return true; } /* ================ idLexer::Parse3DMatrix ================ */ int idLexer::Parse3DMatrix( int z, int y, int x, float *m ) { int i; if ( !idLexer::ExpectTokenString( "(" ) ) { return false; } for ( i = 0 ; i < z; i++ ) { if ( !idLexer::Parse2DMatrix( y, x, m + i * x*y ) ) { return false; } } if ( !idLexer::ExpectTokenString( ")" ) ) { return false; } return true; } /* ================ idLexer::ParseNumericStructArray ================ */ void idLexer::ParseNumericStructArray( int numStructElements, int tokenSubTypeStructElements[], int arrayCount, byte *arrayStorage ) { int arrayOffset, curElement; for ( arrayOffset = 0; arrayOffset < arrayCount; arrayOffset++ ) { for ( curElement = 0; curElement < numStructElements; curElement++ ) { if ( tokenSubTypeStructElements[curElement] & TT_FLOAT ) { *(float*)arrayStorage = idLexer::ParseFloat(); arrayStorage += sizeof(float); } else { *(int*)arrayStorage = idLexer::ParseInt(); arrayStorage += sizeof(int); } } } } /* ================= idParser::ParseBracedSection The next token should be an open brace. Parses until a matching close brace is found. Maintains exact characters between braces. FIXME: this should use ReadToken and replace the token white space with correct indents and newlines ================= */ const char *idLexer::ParseBracedSectionExact( idStr &out, int tabs ) { int depth; bool doTabs; bool skipWhite; out.Empty(); if ( !idLexer::ExpectTokenString( "{" ) ) { return out.c_str( ); } out = "{"; depth = 1; skipWhite = false; doTabs = tabs >= 0; while( depth && *idLexer::script_p ) { char c = *(idLexer::script_p++); switch ( c ) { case '\t': case ' ': { if ( skipWhite ) { continue; } break; } case '\n': { // RAVEN BEGIN // jscott: now gives correct line number in error reports line++; // RAVEN END if ( doTabs ) { skipWhite = true; out += c; continue; } break; } case '{': { depth++; tabs++; break; } case '}': { depth--; tabs--; break; } } if ( skipWhite ) { int i = tabs; if ( c == '{' ) { i--; } skipWhite = false; for ( ; i > 0; i-- ) { out += '\t'; } } out += c; } return out.c_str(); } /* ================= idLexer::ParseBracedSection The next token should be an open brace. Parses until a matching close brace is found. Internal brace depths are properly skipped. ================= */ const char *idLexer::ParseBracedSection( idStr &out ) { idToken token; int i, depth; out.Empty(); if ( !idLexer::ExpectTokenString( "{" ) ) { return out.c_str(); } out = "{"; depth = 1; do { if ( !idLexer::ReadToken( &token ) ) { Error( "missing closing brace" ); return out.c_str(); } // if the token is on a new line for ( i = 0; i < token.linesCrossed; i++ ) { out += "\r\n"; } if ( token.type == TT_PUNCTUATION ) { if ( token[0] == '{' ) { depth++; } else if ( token[0] == '}' ) { depth--; } } if ( token.type == TT_STRING ) { out += "\"" + token + "\""; } else { out += token; } out += " "; } while( depth ); return out.c_str(); } /* ================= idLexer::ParseRestOfLine parse the rest of the line ================= */ const char *idLexer::ParseRestOfLine( idStr &out ) { idToken token; out.Empty(); while(idLexer::ReadToken( &token )) { if ( token.linesCrossed ) { idLexer::script_p = lastScript_p; idLexer::line = lastline; break; } if ( out.Length() ) { out += " "; } out += token; } return out.c_str(); } /* ================ idLexer::GetLastWhiteSpace ================ */ int idLexer::GetLastWhiteSpace( idStr &whiteSpace ) const { whiteSpace.Clear(); for ( const char *p = whiteSpaceStart_p; p < whiteSpaceEnd_p; p++ ) { whiteSpace.Append( *p ); } return whiteSpace.Length(); } /* ================ idLexer::GetLastWhiteSpaceStart ================ */ int idLexer::GetLastWhiteSpaceStart( void ) const { return whiteSpaceStart_p - buffer; } /* ================ idLexer::GetLastWhiteSpaceEnd ================ */ int idLexer::GetLastWhiteSpaceEnd( void ) const { return whiteSpaceEnd_p - buffer; } /* ================ idLexer::Reset ================ */ void idLexer::Reset( void ) { // pointer in script buffer idLexer::script_p = idLexer::buffer; // pointer in script buffer before reading token idLexer::lastScript_p = idLexer::buffer; // begin of white space idLexer::whiteSpaceStart_p = NULL; // end of white space idLexer::whiteSpaceEnd_p = NULL; // set if there's a token available in idLexer::token idLexer::tokenavailable = 0; idLexer::line = 1; idLexer::lastline = 1; // clear the saved token idLexer::token = ""; } /* ================ idLexer::EndOfFile ================ */ int idLexer::EndOfFile( void ) { return idLexer::script_p >= idLexer::end_p; } /* ================ idLexer::NumLinesCrossed ================ */ int idLexer::NumLinesCrossed( void ) { return idLexer::line - idLexer::lastline; } /* ================ idLexer::LoadFile ================ */ int idLexer::LoadFile( const char *filename, bool OSPath ) { idFile *fp; idStr pathname; int length; char *buf; if ( idLexer::loaded ) { idLib::common->Error("idLexer::LoadFile: another script already loaded"); return false; } if ( !OSPath && ( baseFolder[0] != '\0' ) ) { pathname = va( "%s/%s", baseFolder, filename ); } else { pathname = filename; } if ( OSPath ) { fp = idLib::fileSystem->OpenExplicitFileRead( pathname ); } else { fp = idLib::fileSystem->OpenFileRead( pathname ); } if ( !fp ) { return false; } length = fp->Length(); // RAVEN BEGIN // amccarthy: Added memory allocation tag buf = (char *) Mem_Alloc( length + 1, MA_LEXER ); if( !buf ) { common->FatalError( "Memory system failure : out of memory" ); } // RAVEN END buf[length] = '\0'; fp->Read( buf, length ); idLexer::fileTime = fp->Timestamp(); idLexer::filename = fp->GetFullPath(); idLib::fileSystem->CloseFile( fp ); idLexer::buffer = buf; idLexer::length = length; // pointer in script buffer idLexer::script_p = idLexer::buffer; // pointer in script buffer before reading token idLexer::lastScript_p = idLexer::buffer; // pointer to end of script buffer idLexer::end_p = &(idLexer::buffer[length]); idLexer::tokenavailable = 0; idLexer::line = 1; idLexer::lastline = 1; idLexer::allocated = true; idLexer::loaded = true; // RAVEN BEGIN // jsinger: initialize compiled file if(flags & LEXFL_WRITEBINARY) { pathname.Append(Lexer::sCompiledFileSuffix); if ( OSPath ) { mBinaryFile = idLib::fileSystem->OpenExplicitFileWrite( pathname ); } else { mBinaryFile = idLib::fileSystem->OpenFileWrite( pathname ); } } // RAVEN END return true; } /* ================ idLexer::LoadMemory ================ */ int idLexer::LoadMemory( const char *ptr, int length, const char *name, int startLine ) { if ( idLexer::loaded ) { idLib::common->Error("idLexer::LoadMemory: another script already loaded"); return false; } idLexer::filename = name; idLexer::buffer = ptr; idLexer::fileTime = 0; idLexer::length = length; // pointer in script buffer idLexer::script_p = idLexer::buffer; // pointer in script buffer before reading token idLexer::lastScript_p = idLexer::buffer; // pointer to end of script buffer idLexer::end_p = &(idLexer::buffer[length]); idLexer::tokenavailable = 0; idLexer::line = startLine; idLexer::lastline = startLine; idLexer::allocated = false; idLexer::loaded = true; return true; } /* ================ idLexer::FreeSource ================ */ void idLexer::FreeSource( void ) { #ifdef PUNCTABLE if ( idLexer::punctuationtable && idLexer::punctuationtable != default_punctuationtable ) { Mem_Free( (void *) idLexer::punctuationtable ); idLexer::punctuationtable = NULL; } if ( idLexer::nextpunctuation && idLexer::nextpunctuation != default_nextpunctuation ) { Mem_Free( (void *) idLexer::nextpunctuation ); idLexer::nextpunctuation = NULL; } #endif //PUNCTABLE if ( idLexer::allocated ) { Mem_Free( (void *) idLexer::buffer ); idLexer::buffer = NULL; idLexer::allocated = false; } idLexer::tokenavailable = 0; idLexer::token = ""; idLexer::loaded = false; // RAVEN BEGIN // jsinger: close compile file if it exists if(flags & LEXFL_WRITEBINARY) { if(mBinaryFile) { idLib::fileSystem->CloseFile(mBinaryFile); mBinaryFile = NULL; } } // RAVEN END } /* ================ idLexer::idLexer ================ */ idLexer::idLexer( void ) { idLexer::loaded = false; idLexer::filename = ""; idLexer::flags = 0; idLexer::SetPunctuations( NULL ); idLexer::allocated = false; idLexer::fileTime = 0; idLexer::length = 0; idLexer::line = 0; idLexer::lastline = 0; idLexer::tokenavailable = 0; idLexer::token = ""; idLexer::next = NULL; idLexer::hadError = false; // RAVEN BEGIN // jsinger: initialize compiled file idLexer::mBinaryFile = NULL; // RAVEN END } /* ================ idLexer::idLexer ================ */ idLexer::idLexer( int flags ) { idLexer::loaded = false; idLexer::filename = ""; idLexer::flags = flags; idLexer::SetPunctuations( NULL ); idLexer::allocated = false; idLexer::fileTime = 0; idLexer::length = 0; idLexer::line = 0; idLexer::lastline = 0; idLexer::tokenavailable = 0; idLexer::token = ""; idLexer::next = NULL; idLexer::hadError = false; // RAVEN BEGIN // jsinger: initialize compiled file idLexer::mBinaryFile = NULL; // RAVEN END } /* ================ idLexer::idLexer ================ */ idLexer::idLexer( const char *filename, int flags, bool OSPath ) { idLexer::loaded = false; idLexer::flags = flags; idLexer::SetPunctuations( NULL ); idLexer::allocated = false; idLexer::token = ""; idLexer::next = NULL; idLexer::hadError = false; // RAVEN BEGIN // jsinger: initialize compiled file idLexer::mBinaryFile = NULL; // RAVEN END idLexer::LoadFile( filename, OSPath ); } /* ================ idLexer::idLexer ================ */ idLexer::idLexer( const char *ptr, int length, const char *name, int flags ) { idLexer::loaded = false; idLexer::flags = flags; idLexer::SetPunctuations( NULL ); idLexer::allocated = false; idLexer::token = ""; idLexer::next = NULL; idLexer::hadError = false; // RAVEN BEGIN // jsinger: initialize compiled file idLexer::mBinaryFile = NULL; // RAVEN END idLexer::LoadMemory( ptr, length, name ); } /* ================ idLexer::~idLexer ================ */ idLexer::~idLexer( void ) { idLexer::FreeSource(); } // RAVEN BEGIN // jsinger: SetBaseFolder was moved to the Lexer base class to unify its functionality across all // derived classes /* ================ idLexer::SetBaseFolder ================ void idLexer::SetBaseFolder( const char *path ) { idStr::Copynz( baseFolder, path, sizeof( baseFolder ) ); } */ // RAVEN END /* ================ idLexer::HadError ================ */ bool idLexer::HadError( void ) const { return hadError; } // RAVEN BEGIN // jsinger: This method can write out a binary representation of a token in a format // suitable to be read by the Lexer /* ================ idLexer::WriteBinaryToken Writes a binary representation of the token value to the compiled file when compiling is turned on ================ */ void idLexer::WriteBinaryToken(idToken *tok) { bool swapBytes = (flags & LEXFL_BYTESWAP) == LEXFL_BYTESWAP; if(flags & LEXFL_WRITEBINARY) { unsigned char prefix; switch(tok->type) { case TT_NUMBER: if(tok->subtype & TT_INTEGER) { if(tok->subtype & TT_UNSIGNED) { unsigned long val = tok->GetUnsignedLongValue(); unsigned char byteVal = (unsigned char)val; unsigned short shortVal = (unsigned short)val; if(byteVal == val) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_UNSIGNEDINT, BTT_STORED_1BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(byteVal, mBinaryFile, swapBytes); } else if(shortVal == val) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_UNSIGNEDINT, BTT_STORED_2BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(shortVal, mBinaryFile, swapBytes); } else { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_UNSIGNEDINT, BTT_STORED_4BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(val, mBinaryFile, swapBytes); } } else { long val = tok->GetIntValue(); char byteVal = (char)val; short shortVal = (short)val; if(byteVal == val) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_INT, BTT_STORED_1BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(byteVal, mBinaryFile, swapBytes); } else if(shortVal == val) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_INT, BTT_STORED_2BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(shortVal, mBinaryFile, swapBytes); } else { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_INT, BTT_STORED_4BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(val, mBinaryFile, swapBytes); } } } else if(tok->subtype & TT_FLOAT) { if(tok->subtype & TT_SINGLE_PRECISION) { float val = tok->GetFloatValue(); int intval = tok->GetIntValue(); if(((float)intval) == val) // integral check { char byteVal = (char)intval; short shortVal = (short)intval; if(byteVal == intval) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_1BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(byteVal, mBinaryFile, swapBytes); } else if(shortVal == intval) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_2BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(shortVal, mBinaryFile, swapBytes); } else { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_4BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(val, mBinaryFile, swapBytes); } } else { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_4BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(val, mBinaryFile, swapBytes); } } else if(tok->subtype & TT_DOUBLE_PRECISION) { // we can write out doubles as floats because the text file doesn't have double precision anyway float val = tok->GetDoubleValue(); int intval = tok->GetIntValue(); if(((float)intval) == val) // integral check { char byteVal = (char)intval; short shortVal = (short)intval; if(byteVal == intval) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_1BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(byteVal, mBinaryFile, swapBytes); } else if(shortVal == intval) { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_2BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(shortVal, mBinaryFile, swapBytes); } else { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_DOUBLE, BTT_STORED_4BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(val, mBinaryFile, swapBytes); } } else { prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_DOUBLE, BTT_STORED_4BYTE); TextCompiler::WriteValue(prefix, mBinaryFile, swapBytes); TextCompiler::WriteValue(val, mBinaryFile, swapBytes); } } } break; case TT_STRING: TextCompiler::WriteValue(BTT_MAKESTRING_PREFIX(BTT_STRING, tok->Length()), mBinaryFile, swapBytes); TextCompiler::WriteValue(tok, mBinaryFile, swapBytes); break; case TT_LITERAL: TextCompiler::WriteValue(BTT_MAKESTRING_PREFIX(BTT_LITERAL, tok->Length()), mBinaryFile, swapBytes); TextCompiler::WriteValue(tok, mBinaryFile, swapBytes); break; case TT_NAME: TextCompiler::WriteValue(BTT_MAKESTRING_PREFIX(BTT_NAME, tok->Length()), mBinaryFile, swapBytes); TextCompiler::WriteValue(tok, mBinaryFile, swapBytes); break; case TT_PUNCTUATION: if(*tok == "&&") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LOGICALAND), mBinaryFile, swapBytes); } else if (*tok == "&") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_AMPERSAND), mBinaryFile, swapBytes); } else if(*tok == "=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_EQUAL), mBinaryFile, swapBytes); } else if(*tok == "==") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_EQUALEQUAL), mBinaryFile, swapBytes); } else if(*tok == "!=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_NOTEQUAL), mBinaryFile, swapBytes); } else if(*tok == "!") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_EXCLAMATION), mBinaryFile, swapBytes); } else if(*tok == "<") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LESSTHAN), mBinaryFile, swapBytes); } else if(*tok == "<=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LESSOREQUAL), mBinaryFile, swapBytes); } else if(*tok == "<<") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_SHIFTLEFT), mBinaryFile, swapBytes); } else if(*tok == ">") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_GREATERTHAN), mBinaryFile, swapBytes); } else if(*tok == ">=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_GREATEROREQUAL), mBinaryFile, swapBytes); } else if(*tok == ">>") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_SHIFTRIGHT), mBinaryFile, swapBytes); } else if(*tok == "%") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PERCENT), mBinaryFile, swapBytes); } else if(*tok == "[") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LEFTBRACKET), mBinaryFile, swapBytes); } else if(*tok == "]") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_RIGHTBRACKET), mBinaryFile, swapBytes); } else if(*tok == "-") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_MINUS), mBinaryFile, swapBytes); } else if(*tok == "--") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_MINUSMINUS), mBinaryFile, swapBytes); } else if(*tok == "-=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_MINUSEQUAL), mBinaryFile, swapBytes); } else if(*tok == "+") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PLUS), mBinaryFile, swapBytes); } else if(*tok == "+=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PLUSEQUAL), mBinaryFile, swapBytes); } else if(*tok == "++") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PLUSPLUS), mBinaryFile, swapBytes); } else if(*tok == "(") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LEFTPAREN), mBinaryFile, swapBytes); } else if(*tok == ")") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_RIGHTPAREN), mBinaryFile, swapBytes); } else if(*tok == "{") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LEFTBRACE), mBinaryFile, swapBytes); } else if(*tok == "}") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_RIGHTBRACE), mBinaryFile, swapBytes); } else if(*tok == ",") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_COMMA), mBinaryFile, swapBytes); } else if(*tok == "::") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOUBLECOLON), mBinaryFile, swapBytes); } else if(*tok == "#") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_HASH), mBinaryFile, swapBytes); } else if(*tok == "##") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOUBLEHASH), mBinaryFile, swapBytes); } else if(*tok == "/") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_FORWARDSLASH), mBinaryFile, swapBytes); } else if(*tok == "\\") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_BACKSLASH), mBinaryFile, swapBytes); } else if(*tok == ";") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_SEMICOLON), mBinaryFile, swapBytes); } else if(*tok == ".") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PERIOD), mBinaryFile, swapBytes); } else if(*tok == "$") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOLLARSIGN), mBinaryFile, swapBytes); } else if(*tok == "~") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_TILDE), mBinaryFile, swapBytes); } else if(*tok == "|") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PIPE), mBinaryFile, swapBytes); } else if(*tok == "||") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOUBLEPIPE), mBinaryFile, swapBytes); } else if(*tok == "*") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_ASTERISK), mBinaryFile, swapBytes); } else if(*tok == "*=") { TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_TIMESEQUAL), mBinaryFile, swapBytes); } else if(*tok == "¡") { TextCompiler::WriteValue(BTT_PUNC_INVERTEDPLING, mBinaryFile, swapBytes); } else if(*tok == "¿") { TextCompiler::WriteValue(BTT_PUNC_INVERTEDQUERY, mBinaryFile, swapBytes); } else { common->Warning("Unsupported punctuation: '%s'", tok->c_str()); TextCompiler::WriteValue(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_ASNULLTERMINATED), mBinaryFile, swapBytes); TextCompiler::WriteValue(tok, mBinaryFile, swapBytes); } break; default: // wtf? assert(false); } } } // jsinger: this method takes an existing file and causes a binary tokenized // version of the file to be generated void idLexer::WriteBinaryFile(char const * const filename) { if(!cvarSystem->GetCVarBool("com_BinaryRead")) { unsigned int swap=0; switch(cvarSystem->GetCVarInteger("com_BinaryWrite")) { case 1: swap = 0; break; case 2: swap = LEXFL_BYTESWAP; break; } idLexer src(filename, LEXFL_WRITEBINARY | swap); idToken token; if(src.IsLoaded()) { while(src.ReadToken(&token)) { // we don't need to do anything, because just reading the file // causes it to write a tokenized version } } } } // jsinger: implementation for a class that can load binary tokenized files using namespace TextCompiler; Lexer::Lexer(int flags) { #ifndef LEXER_READ_AHEAD mFile = NULL; #endif offset = 0; tokenAvailable = false; mFlags = flags; if(flags & LEXFL_READBINARY) { mDelegate = NULL; } else { mDelegate = new idLexer(flags); } } Lexer::Lexer(char const * const ptr, int length, char const * const name, int flags) { if(flags & LEXFL_READBINARY) { #ifdef LEXER_READ_AHEAD mLexerIOWrapper.InitFromMemory(ptr, length); #else assert(false); #endif mDelegate = NULL; } else { mDelegate = new idLexer(ptr, length, name, flags); } } Lexer::Lexer(char const * const filename, int flags, bool OSPath) { #ifndef LEXER_READ_AHEAD mFile = NULL; #endif offset = 0; tokenAvailable = false; mFlags = flags; if(flags & LEXFL_READBINARY) { mDelegate = NULL; LoadFile(filename, OSPath); } else { mDelegate = new idLexer(filename, flags, OSPath); } } Lexer::~Lexer() { FreeSource(); } char const * Lexer::GetFileName() { if(mDelegate) { return mDelegate->GetFileName(); } else { assert(false); return ""; } } bool Lexer::HadError() const { if(mDelegate) { return mDelegate->HadError(); } else { assert(false); return false; } } void Lexer::Warning(char const *str, ...) { char text[MAX_STRING_CHARS]; va_list ap; va_start( ap, str ); vsprintf( text, str, ap ); va_end( ap ); idLib::common->Warning( "Lexer: '%s'", text ); } void Lexer::Error(char const *str, ...) { char text[MAX_STRING_CHARS]; va_list ap; va_start( ap, str ); vsprintf( text, str, ap ); va_end( ap ); assert(false); idLib::common->Error( "Lexer: offset: %d '%s'", offset, text ); } int const Lexer::GetLineNum() { if(mDelegate) { return mDelegate->GetLineNum(); } else { return 1; } } unsigned int const Lexer::GetFileTime() { if(mDelegate) { return mDelegate->GetFileTime(); } else { return 0; } } int const Lexer::GetFileOffset() { if(mDelegate) { return mDelegate->GetFileOffset(); } { assert(false); return 0; } } int Lexer::EndOfFile() { if(mDelegate) { return mDelegate->EndOfFile(); } else { #ifdef LEXER_READ_AHEAD return mLexerIOWrapper.Tell()>= mLexerIOWrapper.Length(); #else return mFile->Tell()>= mFile->Length(); #endif } } void Lexer::Reset() { if(mDelegate) { mDelegate->Reset(); } else { #ifdef LEXER_READ_AHEAD mLexerIOWrapper.Seek(0, FS_SEEK_SET); #else mFile->Seek(0, FS_SEEK_SET); #endif offset = 0; tokenAvailable = false; } } int Lexer::GetFlags() { if(mDelegate) { return mDelegate->GetFlags(); } else { return mFlags; } } void Lexer::SetFlags(int flags) { if(mDelegate) { mDelegate->SetFlags(flags); } else { mFlags = flags; } } int Lexer::GetPunctuationId(char const *str) { if(mDelegate) { return mDelegate->GetPunctuationId(str); } else { // this method is not supported assert(false); return 0; } } void Lexer::SetPunctuations(struct punctuation_s const *punc) { if(mDelegate) { mDelegate->SetPunctuations(punc); } else { // this method is not supported assert(false); } } int Lexer::GetLastWhiteSpaceEnd() const { if(mDelegate) { return mDelegate->GetLastWhiteSpaceEnd(); } else { return 0; } } char const *Lexer::GetPunctuationFromId(int val) { if(mDelegate) { return mDelegate->GetPunctuationFromId(val); } else { // this method is not supported assert(false); return ""; } } int Lexer::GetLastWhiteSpaceStart() const { if(mDelegate) { return mDelegate->GetLastWhiteSpaceStart(); } else { return 0; } } int Lexer::GetLastWhiteSpace(idStr &str) const { if(mDelegate) { return mDelegate->GetLastWhiteSpace(str); } else { return 0; } } char const *Lexer::ParseRestOfLine(idStr &str) { if(mDelegate) { return mDelegate->ParseRestOfLine(str); } else { // this method is not supported assert(false); return ""; } } char const *Lexer::ParseBracedSectionExact(idStr &str, int val) { if(mDelegate) { return mDelegate->ParseBracedSectionExact(str, val); } else { // this method is not supported assert(false); return ""; } } char const *Lexer::ParseBracedSection(idStr &str) { if(mDelegate) { return mDelegate->ParseBracedSection(str); } else { // this method is not supported assert(false); return ""; } } int Lexer::Parse3DMatrix(int z, int y, int x, float *m) { if(mDelegate) { return mDelegate->Parse3DMatrix(z, y, x, m); } else { int i; if ( !ExpectTokenString( "(" ) ) { return false; } for ( i = 0 ; i < z; i++ ) { if ( !Parse2DMatrix( y, x, m + i * x*y ) ) { return false; } } if ( !ExpectTokenString( ")" ) ) { return false; } return true; } } int Lexer::Parse2DMatrix(int y, int x, float *m) { if(mDelegate) { return mDelegate->Parse2DMatrix(y, x, m); } else { int i; if ( !ExpectTokenString( "(" ) ) { return false; } for ( i = 0; i < y; i++ ) { if ( !Parse1DMatrix( x, m + i * x ) ) { return false; } } if ( !ExpectTokenString( ")" ) ) { return false; } return true; } } int Lexer::Parse1DMatrixOpenEnded(int MaxCount, float *m) { if(mDelegate) { return mDelegate->Parse1DMatrixOpenEnded(MaxCount, m); } else { int i; if ( !ExpectTokenString( "(" ) ) { return 0; } for ( i = 0; i < MaxCount; i++ ) { idToken tok; if (!ReadToken( &tok )) { return 0; } if ( tok == ")" ) { return i; } UnreadToken( &tok ); m[i] = ParseFloat(); } if ( !ExpectTokenString( ")" ) ) { return 0; } return i; } } int Lexer::Parse1DMatrix(int x, float *m) { if(mDelegate) { return mDelegate->Parse1DMatrix(x, m); } else { int i; if ( !ExpectTokenString( "(" ) ) { return false; } for ( i = 0; i < x; i++ ) { m[i] = ParseFloat(); } if ( !ExpectTokenString( ")" ) ) { return false; } return true; } } float Lexer::ParseFloat(bool *errorFlag) { if(mDelegate) { return mDelegate->ParseFloat(errorFlag); } else { idToken token; if(EndOfFile()) { return 0; } if ( !ReadToken( &token ) ) { Warning( "couldn't read expected floating point number" ); return 0; } if ( token.type == TT_PUNCTUATION && token == "-" ) { ExpectTokenType( TT_NUMBER, 0, &token ); return -token.GetFloatValue(); } else if ( token.type != TT_NUMBER ) { Warning( "expected float value, found '%s'", token.c_str() ); } return token.GetFloatValue(); } } bool Lexer::ParseBool() { if(mDelegate) { return mDelegate->ParseBool(); } else { idToken token; if ( !ExpectTokenType( TT_NUMBER, 0, &token ) ) { Error( "couldn't read expected boolean" ); return false; } return ( token.GetIntValue() != 0 ); } } int Lexer::ParseInt() { if(mDelegate) { return mDelegate->ParseInt(); } else { idToken token; if ( !ReadToken( &token ) ) { Error( "couldn't read expected integer" ); return 0; } if ( token.type == TT_PUNCTUATION && token == "-" ) { ExpectTokenType( TT_NUMBER, TT_INTEGER, &token ); return -((signed int) token.GetIntValue()); } else if ( token.type != TT_NUMBER || token.subtype == TT_FLOAT ) { Error( "expected integer value, found '%s'", token.c_str() ); } return token.GetIntValue(); } } int Lexer::ReadTokenOnLine(idToken *token) { if(mDelegate) { return mDelegate->ReadTokenOnLine(token); } else { // lines are meaningless in the binary format return ReadToken(token); } } void Lexer::UnreadToken(idToken const *token) { if(mDelegate) { mDelegate->UnreadToken(token); } else { if ( tokenAvailable ) { idLib::common->FatalError( "Lexer::unreadToken, unread token twice\n" ); } unreadToken = *token; tokenAvailable = true; offset -= unreadSize+1; // the +1 is for the prefix } } int Lexer::SkipBracedSection(bool parseFirstBrace) { if(mDelegate) { return mDelegate->SkipBracedSection(parseFirstBrace); } else { idToken token; int depth; depth = parseFirstBrace ? 0 : 1; do { if ( !ReadToken( &token ) ) { return false; } if ( token.type == TT_PUNCTUATION ) { if ( token == "{" ) { depth++; } else if ( token == "}" ) { depth--; } } } while( depth ); return true; } } int Lexer::SkipRestOfLine() { if(mDelegate) { return mDelegate->SkipRestOfLine(); } else { // this method is not supported assert(false); return 0; } } int Lexer::SkipUntilString(char const *str) { if(mDelegate) { return mDelegate->SkipUntilString(str); } else { // this method is not supported assert(false); return 0; } } int Lexer::CheckTokenType(int type, int subtype, idToken *token) { if(mDelegate) { return mDelegate->CheckTokenType(type, subtype, token); } else { idToken tok; if (!ReadToken( &tok )) { return 0; } // if the type matches if (tok.type == type && (tok.subtype & subtype) == subtype) { *token = tok; return 1; } return 0; } } int Lexer::CheckTokenString(char const *string) { if(mDelegate) { return mDelegate->CheckTokenString(string); } else { idToken tok; if (!ReadToken( &tok )) { return 0; } // if the token is available if ( tok == string ) { return 1; } UnreadToken( &tok ); return 0; } } int Lexer::ExpectAnyToken(idToken *token) { if(mDelegate) { return mDelegate->ExpectAnyToken(token); } else { if (!ReadToken( token )) { Error( "couldn't read expected token" ); return 0; } else { return 1; } } } int Lexer::ExpectTokenType(int type, int subtype, idToken *token) { if(mDelegate) { return mDelegate->ExpectTokenType(type, subtype, token); } else { idStr str; if ( !ReadToken( token ) ) { Error( "couldn't read expected token" ); return 0; } if ( token->type != type ) { switch( type ) { case TT_STRING: str = "string"; break; case TT_LITERAL: str = "literal"; break; case TT_NUMBER: str = "number"; break; case TT_NAME: str = "name"; break; case TT_PUNCTUATION: str = "punctuation"; break; default: str = "unknown type"; break; } Error( "expected a %s but found '%s'", str.c_str(), token->c_str() ); return 0; } if ( token->type == TT_NUMBER ) { if ( (token->subtype & subtype) != subtype ) { str.Clear(); if ( subtype & TT_DECIMAL ) str = "decimal "; if ( subtype & TT_HEX ) str = "hex "; if ( subtype & TT_OCTAL ) str = "octal "; if ( subtype & TT_BINARY ) str = "binary "; if ( subtype & TT_UNSIGNED ) str += "unsigned "; if ( subtype & TT_LONG ) str += "long "; if ( subtype & TT_FLOAT ) str += "float "; if ( subtype & TT_INTEGER ) str += "integer "; str.StripTrailing( ' ' ); Error( "expected %s but found '%s'", str.c_str(), token->c_str() ); return 0; } } else if ( token->type == TT_PUNCTUATION ) { if ( subtype < 0 ) { Error( "BUG: wrong punctuation subtype" ); return 0; } if ( token->subtype != subtype ) { Error( "expected '%s' but found '%s'", GetPunctuationFromId( subtype ), token->c_str() ); return 0; } } return 1; } } int Lexer::ExpectTokenString(char const *string) { if(mDelegate) { return mDelegate->ExpectTokenString(string); } else { idToken token; if(!ReadToken( &token )) { Error( "couldn't find expected '%s'", string ); return 0; } if( token != string ) { Error( "expected '%s' but found '%s'", string, token.c_str() ); return 0; } return 1; } } int Lexer::ReadToken(idToken *token) { #ifdef LEXER_READ_AHEAD #define OBJ &mLexerIOWrapper #else #define OBJ mFile #endif if(mDelegate) { return mDelegate->ReadToken(token); } else { // if there are any unread tokens, use them first if(tokenAvailable) { *token = unreadToken; tokenAvailable = false; } else { #ifndef LEXER_READ_AHEAD assert(mFile); #endif if(EndOfFile()) return 0; unsigned char prefix = ReadValue(OBJ); token->Clear(); // BTT types are equivalent to the TT types except they are one less, so we add one to get the token type if(BTT_GET_TYPE(prefix) == BTT_PUNCTUATION2) token->type = TT_PUNCTUATION; else token->type = BTT_GET_TYPE(prefix)+1; // this is used only to determine if the punctuation was encoded in the prefix or came afterwards as a null terminated string bool encoded = true; switch(token->type) { case TT_STRING: case TT_NAME: case TT_LITERAL: if(BTT_GET_STRING_LENGTH(prefix) != 0) { // short string int length = BTT_GET_STRING_LENGTH(prefix); if ( length == 0 ) { assert( false ); } int i; for( i = 0; i < length; i++ ) { char c = ReadValue(OBJ); token->Append(c); } // short strings have no null, so we must add one //token->Append('\0'); token->subtype = token->Length(); unreadSize = token->Length(); } else { // null terminated string *token = ReadValue(OBJ); token->subtype = token->Length(); unreadSize = token->Length()+1; } break; case TT_PUNCTUATION: switch(BTT_GET_PUNCTUATION(prefix)) { case BTT_PUNC_ASNULLTERMINATED: // null terminated string *token = ReadValue(OBJ); token->subtype = token->Length(); unreadSize = token->Length()+1; encoded = false; break; case BTT_PUNC_RIGHTPAREN: *token = ")"; break; case BTT_PUNC_LEFTBRACE: *token = "{"; break; case BTT_PUNC_RIGHTBRACE: *token = "}"; break; case BTT_PUNC_MINUS: *token = "-"; break; case BTT_PUNC_PLUS: *token = "+"; break; case BTT_PUNC_COMMA: *token = ","; break; case BTT_PUNC_PLUSPLUS: *token = "++"; break; case BTT_PUNC_LEFTBRACKET: *token = "["; break; case BTT_PUNC_RIGHTBRACKET: *token = "]"; break; case BTT_PUNC_EQUAL: *token = "="; break; case BTT_PUNC_EQUALEQUAL: *token = "=="; break; case BTT_PUNC_NOTEQUAL: *token = "!="; break; case BTT_PUNC_PERCENT: *token = "%"; break; case BTT_PUNC_LESSTHAN: *token = "<"; break; case BTT_PUNC_GREATERTHAN: *token = ">"; break; case BTT_PUNC_LOGICALAND: *token = "&&"; break; case BTT_PUNC_AMPERSAND: *token = "&"; break; case BTT_PUNC_MINUSMINUS: *token = "--"; break; case BTT_PUNC_HASH: *token = "#"; break; case BTT_PUNC_LESSOREQUAL: *token = "<="; break; case BTT_PUNC_GREATEROREQUAL: *token = ">="; break; case BTT_PUNC_FORWARDSLASH: *token = "/"; break; case BTT_PUNC_SHIFTLEFT: *token = "<<"; break; case BTT_PUNC_SHIFTRIGHT: *token = ">>"; break; case BTT_PUNC_LEFTPAREN: *token = "("; break; case BTT_PUNC_SEMICOLON: *token = ";"; break; case BTT_PUNC_ASTERISK: *token = "*"; break; case BTT_PUNC_PERIOD: *token = "."; break; case BTT_PUNC_DOLLARSIGN: *token = "$"; break; case BTT_PUNC_PLUSEQUAL: *token = "+="; break; case BTT_PUNC_MINUSEQUAL: *token = "-="; break; case BTT_PUNC_TILDE: *token = "~"; break; case BTT_PUNC_EXCLAMATION: *token = "!"; break; case BTT_PUNC_PIPE: *token = "|"; break; case BTT_PUNC_BACKSLASH: *token = "\\"; break; case BTT_PUNC_DOUBLEHASH: *token = "##"; break; case BTT_PUNC_TIMESEQUAL: *token = "*="; break; case BTT_PUNC_DOUBLEPIPE: *token = "||"; break; case BTT_PUNC_INVERTEDPLING: *token = "¡"; break; case BTT_PUNC_INVERTEDQUERY: *token = "¿"; break; default: assert(false); // unrecognized punctuation } if(encoded) unreadSize = 0; // punctuation encoded in prefix break; case TT_NUMBER: { // number of bytes actually in the stream used to represent this value unsigned int size = BTT_GET_STORED_SIZE(prefix); const int buffersize = 100; char buffer[buffersize]; // big enough buffer for the int to string conversion routines switch(BTT_GET_SUBTYPE(prefix)) { case BTT_SUBTYPE_INT: switch(size) { case BTT_STORED_1BYTE: token->intvalue = ReadValue(OBJ); unreadSize = sizeof(char); break; case BTT_STORED_2BYTE: token->intvalue = ReadValue(OBJ); unreadSize = sizeof(short); break; case BTT_STORED_4BYTE: token->intvalue = ReadValue(OBJ); unreadSize = sizeof(int); break; default: // invalid stored size for an integer assert(false); } token->floatvalue = token->intvalue; // I hate the fact that I have to copy into the string, but there's no way // of easily knowing whether it is used later or not // This conversion to a string assumes that long and int are the same assert(sizeof(long) == sizeof(int)); //ltoa(token->intvalue, buffer, 10); idStr::snPrintf( buffer, buffersize, "%ld", token->intvalue ); assert(token->intvalue == atol(buffer)); *token = buffer; token->subtype = TT_INTEGER | TT_DECIMAL | TT_VALUESVALID; break; case BTT_SUBTYPE_UNSIGNEDINT: switch(size) { case BTT_STORED_1BYTE: token->intvalue = ReadValue(OBJ); unreadSize = sizeof(unsigned char); break; case BTT_STORED_2BYTE: token->intvalue = ReadValue(OBJ); unreadSize = sizeof(unsigned short); break; case BTT_STORED_4BYTE: token->intvalue = ReadValue(OBJ); unreadSize = sizeof(unsigned int); break; default: // invalid stored size for an unsigned integer assert(false); } token->floatvalue = token->intvalue; // I hate the fact that I have to copy into the string, but there's no way // of easily knowing whether it is used later or not // This conversion to a string assumes that long and int are the same assert(sizeof(long) == sizeof(int)); //ultoa(token->intvalue, buffer, 10); idStr::snPrintf( buffer, buffersize, "%lu", token->intvalue ); assert(token->intvalue == ((unsigned long)atol(buffer))); *token = buffer; token->subtype = TT_INTEGER | TT_UNSIGNED | TT_DECIMAL | TT_VALUESVALID; break; case BTT_SUBTYPE_FLOAT: // invalid stored size for a float assert(sizeof(float) == 4); switch(size) { case BTT_STORED_4BYTE: token->floatvalue = ReadValue(OBJ); token->intvalue = token->floatvalue; unreadSize = sizeof(float); break; case BTT_STORED_2BYTE: // requested a float, but it was integral, so it was saved that way token->floatvalue = ReadValue(OBJ); token->intvalue = token->floatvalue; unreadSize = sizeof(short); break; case BTT_STORED_1BYTE: // requested a float, but it was integral, so it was saved that way token->floatvalue = ReadValue(OBJ); token->intvalue = token->floatvalue; unreadSize = sizeof(char); break; default: assert(false); } // I hate the fact that I have to copy into the string, but there's no way // of easily knowing whether it is used later or not idStr::snPrintf( buffer, buffersize, "%f", token->floatvalue ); *token = buffer; token->subtype = TT_FLOAT | TT_DECIMAL | TT_SINGLE_PRECISION | TT_VALUESVALID; unreadSize = sizeof(float); break; case BTT_SUBTYPE_DOUBLE: // invalid stored size for a double assert(sizeof(double) == 8); // doubles are stored as floats since the original text file never uses full double precision anyway //assert(size == BTT_STORED_4BYTE); switch(size) { case BTT_STORED_4BYTE: token->floatvalue = ReadValue(OBJ); token->intvalue = token->floatvalue; unreadSize = sizeof(float); break; case BTT_STORED_2BYTE: // requested a float, but it was integral, so it was saved that way token->floatvalue = ReadValue(OBJ); token->intvalue = token->floatvalue; unreadSize = sizeof(short); break; case BTT_STORED_1BYTE: // requested a float, but it was integral, so it was saved that way token->floatvalue = ReadValue(OBJ); token->intvalue = token->floatvalue; unreadSize = sizeof(char); break; default: assert(false); } // I hate the fact that I have to copy into the string, but there's no way // of easily knowing whether it is used later or not idStr::snPrintf( buffer, buffersize, "%f", token->floatvalue ); *token = buffer; token->subtype = TT_FLOAT | TT_DECIMAL | TT_DOUBLE_PRECISION | TT_VALUESVALID; break; } } break; default: // unsupported binary type assert(false); } } //common->Warning("Read Token: '%s (int: %d)(float: %f)'\n", token->c_str(), token->GetIntValue(), token->GetFloatValue()); //common->Printf("Read Token: %s\n", token->c_str()); offset += unreadSize+1; // the +1 is for the prefix return 1; } } int Lexer::IsLoaded() { if(mDelegate) { return mDelegate->IsLoaded(); } else { #ifdef LEXER_READ_AHEAD return mLexerIOWrapper.IsLoaded(); #else return mFile != NULL; #endif } } void Lexer::FreeSource() { #ifdef LEXER_READ_AHEAD mLexerIOWrapper.Close(); #else if(mFile) { idLib::fileSystem->CloseFile(mFile); mFile = NULL; } #endif if(mDelegate) { delete mDelegate; mDelegate = NULL; } } int Lexer::LoadMemory(char const *ptr, int length, char const *name, int startLine) { if(mDelegate) { return mDelegate->LoadMemory(ptr, length, name, startLine); } else { // this is essentially a no op return true; } } int Lexer::LoadFile(char const *filename, bool OSPath) { if(mDelegate) { return mDelegate->LoadFile(filename, OSPath); } else { idStr fullName; FreeSource(); fullName = filename; fullName.Append(sCompiledFileSuffix); bool binaryFound = OpenFile(fullName, OSPath); if(binaryFound) { return binaryFound; } else { mDelegate = new idLexer(filename, mFlags, OSPath); int isLoaded = mDelegate->IsLoaded(); if(isLoaded) { // don't do this right now until I clean up the Lexer class if(mFlags & LEXFL_READBINARY) { Warning("%s not found, loading ascii version", fullName.c_str()); } } else { // didn't find the ascii version either, make sure and clean up in case this lexer is reused delete mDelegate; mDelegate = NULL; } return isLoaded; } } } void Lexer::WriteBinaryToken(idToken *tok) { if(mDelegate) { mDelegate->WriteBinaryToken(tok); } else { // can't write out an already binary file assert(false); } } bool Lexer::OpenFile(char const *filename, bool OSPath) { idStr pathname; if ( !OSPath && ( baseFolder[0] != '\0' ) ) { pathname = va( "%s/%s", baseFolder, filename ); } else { pathname = filename; } #ifdef LEXER_READ_AHEAD if ( !mLexerIOWrapper.OpenFile( pathname, OSPath ) ) { return false; } #else if ( OSPath ) { mFile = idLib::fileSystem->OpenExplicitFileRead( pathname ); } else { mFile = idLib::fileSystem->OpenFileRead( pathname ); } if ( !mFile ) { return false; } #endif return true; } void Lexer::SetBaseFolder( const char *path ) { idStr::Copynz( baseFolder, path, sizeof( baseFolder ) ); } // RAVEN BEGIN // dluetscher: added method to parse a structure array that is made up of numerics (floats, ints), and stores them in the given storage void Lexer::ParseNumericStructArray( int numStructElements, int tokenSubTypeStructElements[], int arrayCount, byte *arrayStorage ) { if(mDelegate) { mDelegate->ParseNumericStructArray( numStructElements, tokenSubTypeStructElements, arrayCount, arrayStorage ); } else { int arrayOffset, curElement; for ( arrayOffset = 0; arrayOffset < arrayCount; arrayOffset++ ) { for ( curElement = 0; curElement < numStructElements; curElement++ ) { if ( tokenSubTypeStructElements[curElement] & TT_FLOAT ) { *(float*)arrayStorage = Lexer::ParseFloat(); arrayStorage += sizeof(float); } else { *(int*)arrayStorage = Lexer::ParseInt(); arrayStorage += sizeof(int); } } } } } // RAVEN END char idLexer::baseFolder[256]; // RAVEN END