3945 lines
84 KiB
C++
3945 lines
84 KiB
C++
|
||
#include "precompiled.h"
|
||
#pragma hdrstop
|
||
|
||
#ifdef LEXER_READ_AHEAD
|
||
|
||
namespace {
|
||
|
||
#define IO_BUFFER_CHUNK_SIZE (1024 * 64)
|
||
|
||
enum {
|
||
IOJobPending = 0,
|
||
IOJobQuit,
|
||
IOJobComplete,
|
||
IOJobCount
|
||
};
|
||
static HANDLE mIOJobs[IOJobCount];
|
||
|
||
static DWORD mThreadId = 0;
|
||
static HANDLE mThreadHandle = 0;
|
||
|
||
|
||
struct IOJobData {
|
||
idFile *mFile;
|
||
int mAmtToRead;
|
||
byte mBuffer[IO_BUFFER_CHUNK_SIZE];
|
||
};
|
||
|
||
IOJobData gIOJobData;
|
||
|
||
void SetPendingJob(idFile *f, int size) {
|
||
gIOJobData.mFile = f;
|
||
gIOJobData.mAmtToRead = size;
|
||
|
||
if ( mThreadHandle ) {
|
||
SetEvent( mIOJobs[IOJobPending] );
|
||
}
|
||
}
|
||
|
||
IOJobData *GetPendingJob() {
|
||
return &gIOJobData;
|
||
}
|
||
|
||
void GetPendingJobResults(byte *dest, int &size) {
|
||
if ( mThreadHandle ) {
|
||
if ( WAIT_OBJECT_0 == WaitForSingleObject( mIOJobs[IOJobComplete], INFINITE ) ) {
|
||
//ResetEvent( mIOJobs[IOJobComplete] );
|
||
size = gIOJobData.mAmtToRead;
|
||
memcpy(dest, gIOJobData.mBuffer, size );
|
||
} else {
|
||
assert( 0 );
|
||
}
|
||
} else {
|
||
gIOJobData.mFile->Read( dest, gIOJobData.mAmtToRead );
|
||
size = gIOJobData.mAmtToRead;
|
||
}
|
||
}
|
||
|
||
static LONG IOThread( void * ) {
|
||
while ( 1 ) {
|
||
DWORD res = WaitForMultipleObjects( 2, mIOJobs, FALSE, INFINITE );
|
||
if ( res == WAIT_OBJECT_0 ) {
|
||
//ResetEvent( mIOJobs[IOJobPending] );
|
||
IOJobData *j = GetPendingJob();
|
||
j->mFile->Read( j->mBuffer, j->mAmtToRead );
|
||
SetEvent( mIOJobs[IOJobComplete] );
|
||
} else if ( res == (WAIT_OBJECT_0 + 1) ) {
|
||
ExitThread(0);
|
||
} else {
|
||
assert( 0 );
|
||
}
|
||
}
|
||
ExitThread(0);
|
||
}
|
||
|
||
void InitIO() {
|
||
for ( int i = 0; i < IOJobCount; ++i ) {
|
||
//mIOJobs[i] = CreateEvent( 0, TRUE, FALSE, 0 );
|
||
mIOJobs[i] = CreateEvent( 0, FALSE, FALSE, 0 );
|
||
}
|
||
mThreadHandle = CreateThread( NULL, 1024 * 512, ( LPTHREAD_START_ROUTINE )IOThread, 0, CREATE_SUSPENDED, &mThreadId );
|
||
ResumeThread( mThreadHandle );
|
||
}
|
||
|
||
void ShutdownIO() {
|
||
|
||
SetEvent( mIOJobs[IOJobQuit] );
|
||
|
||
DWORD ecode = 0;
|
||
while ( !GetExitCodeThread( mThreadHandle, &ecode ) ) {
|
||
Sleep( 1 );
|
||
}
|
||
CloseHandle( mThreadHandle );
|
||
|
||
for ( int i = 0; i < IOJobCount; ++i ) {
|
||
CloseHandle( mIOJobs[i] );
|
||
}
|
||
|
||
mThreadHandle = 0;
|
||
}
|
||
|
||
}
|
||
|
||
namespace ChunkBuffer {
|
||
|
||
idFile *mFile;
|
||
int mFileSize;
|
||
int mBytesRead;
|
||
|
||
int mFileBytesRead;
|
||
|
||
byte mChunkMem[IO_BUFFER_CHUNK_SIZE];
|
||
int mCurrentChunkBytes;
|
||
int mCurrentChunkBytesRead = 0;
|
||
|
||
/*
|
||
================
|
||
ChunkBuffer::Length
|
||
================
|
||
*/
|
||
int Length() {
|
||
return mFileSize;
|
||
}
|
||
|
||
/*
|
||
================
|
||
ChunkBuffer::BytesRead
|
||
================
|
||
*/
|
||
int BytesRead() {
|
||
return mBytesRead;
|
||
}
|
||
|
||
/*
|
||
================
|
||
ChunkBuffer::Begin
|
||
================
|
||
*/
|
||
void Begin( idFile *f ) {
|
||
mFile = f;
|
||
mFileSize = f->Length();
|
||
mBytesRead = 0;
|
||
|
||
int initialRead = (mFileSize < IO_BUFFER_CHUNK_SIZE) ? mFileSize : IO_BUFFER_CHUNK_SIZE;
|
||
mFile->Read( mChunkMem, initialRead );
|
||
mFileBytesRead = initialRead;
|
||
mCurrentChunkBytesRead = 0;
|
||
mCurrentChunkBytes = initialRead;
|
||
|
||
int bytesRemaining = mFileSize - mFileBytesRead;
|
||
if ( bytesRemaining > 0 ) {
|
||
SetPendingJob( mFile, (bytesRemaining > IO_BUFFER_CHUNK_SIZE) ? IO_BUFFER_CHUNK_SIZE : bytesRemaining );
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
ChunkBuffer::End
|
||
================
|
||
*/
|
||
void End() {
|
||
}
|
||
|
||
/*
|
||
================
|
||
ChunkBuffer::Advance
|
||
================
|
||
*/
|
||
int Advance( byte *chunkMem ) {
|
||
int readAmt = 0;
|
||
GetPendingJobResults( chunkMem, readAmt );
|
||
mFileBytesRead+=readAmt;
|
||
|
||
int bytesRemaining = mFileSize - mFileBytesRead;
|
||
if ( bytesRemaining > 0 ) {
|
||
SetPendingJob( mFile, (bytesRemaining > IO_BUFFER_CHUNK_SIZE) ? IO_BUFFER_CHUNK_SIZE : bytesRemaining );
|
||
}
|
||
|
||
return readAmt;
|
||
}
|
||
|
||
/*
|
||
================
|
||
ChunkBuffer::Read
|
||
================
|
||
*/
|
||
int Read( byte *dest, int size ) {
|
||
int bytesRem = mFileSize - mBytesRead;
|
||
|
||
if ( size > bytesRem ) {
|
||
size = bytesRem;
|
||
}
|
||
|
||
if ( !size ) {
|
||
return 0;
|
||
}
|
||
|
||
byte *writePos = dest;
|
||
int cnt = size;
|
||
|
||
while ( cnt ) {
|
||
if ( mCurrentChunkBytesRead >= mCurrentChunkBytes ) {
|
||
mCurrentChunkBytes = Advance( mChunkMem );
|
||
mCurrentChunkBytesRead = 0;
|
||
}
|
||
|
||
int chunkBytesRemaining = mCurrentChunkBytes - mCurrentChunkBytesRead;
|
||
int toRead = (cnt > chunkBytesRemaining) ? chunkBytesRemaining : cnt;
|
||
|
||
byte *readPos = &mChunkMem[mCurrentChunkBytesRead];
|
||
|
||
memcpy( writePos, readPos, toRead );
|
||
|
||
cnt-=toRead;
|
||
mCurrentChunkBytesRead+=toRead;
|
||
writePos+=toRead;
|
||
}
|
||
|
||
mBytesRead+=size;
|
||
return size;
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::LexerIOWrapper
|
||
================
|
||
*/
|
||
LexerIOWrapper::LexerIOWrapper() {
|
||
file = 0;
|
||
offset = 0;
|
||
size = 0;
|
||
memory = 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::InitFromMemory
|
||
================
|
||
*/
|
||
void LexerIOWrapper::InitFromMemory(char const * const ptr, int length) {
|
||
file = 0;
|
||
offset = 0;
|
||
size = length;
|
||
memory = (char *)ptr;
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::OpenFile
|
||
================
|
||
*/
|
||
bool LexerIOWrapper::OpenFile(char const *filename, bool OSPath) {
|
||
if ( OSPath ) {
|
||
file = idLib::fileSystem->OpenExplicitFileRead( filename );
|
||
} else {
|
||
file = idLib::fileSystem->OpenFileRead( filename );
|
||
}
|
||
|
||
if ( !file ) {
|
||
return false;
|
||
}
|
||
|
||
ChunkBuffer::Begin( file );
|
||
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::Length
|
||
================
|
||
*/
|
||
int LexerIOWrapper::Length() {
|
||
if ( file ) {
|
||
return ChunkBuffer::Length();
|
||
}
|
||
return size;
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::Tell
|
||
================
|
||
*/
|
||
int LexerIOWrapper::Tell() {
|
||
if ( file ) {
|
||
return ChunkBuffer::BytesRead();
|
||
}
|
||
return offset;
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::Seek
|
||
================
|
||
*/
|
||
void LexerIOWrapper::Seek( int loc, int mode ) {
|
||
if ( file ) {
|
||
if ( loc != 0 || mode != FS_SEEK_SET ) {
|
||
common->Error( "lol\n" );
|
||
}
|
||
file->Seek( 0, FS_SEEK_SET );
|
||
ChunkBuffer::Begin( file );
|
||
return;
|
||
}
|
||
if ( mode == FS_SEEK_SET ) {
|
||
offset = loc;
|
||
} else {
|
||
common->Error( "lol" );
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::Read
|
||
================
|
||
*/
|
||
void LexerIOWrapper::Read( void *dest, int s ) {
|
||
if ( file ) {
|
||
ChunkBuffer::Read( (byte*)dest, s );
|
||
return;
|
||
}
|
||
char *d = (char *)dest;
|
||
char *dEnd = d + (((s + offset) > size) ? (size - offset) : (s));
|
||
while ( d < dEnd ) {
|
||
*d = memory[offset];
|
||
++d;
|
||
++offset;
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::Close
|
||
================
|
||
*/
|
||
void LexerIOWrapper::Close() {
|
||
if ( file ) {
|
||
ChunkBuffer::End();
|
||
idLib::fileSystem->CloseFile(file);
|
||
return;
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
LexerIOWrapper::IsLoaded
|
||
================
|
||
*/
|
||
int LexerIOWrapper::IsLoaded() {
|
||
if ( file || memory ) {
|
||
return 1;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
Lexer::BeginLevelLoad
|
||
================
|
||
*/
|
||
void Lexer::BeginLevelLoad() {
|
||
InitIO();
|
||
}
|
||
|
||
/*
|
||
================
|
||
Lexer::EndLevelLoad
|
||
================
|
||
*/
|
||
void Lexer::EndLevelLoad() {
|
||
ShutdownIO();
|
||
}
|
||
|
||
/*
|
||
================
|
||
ReadValue
|
||
================
|
||
*/
|
||
template <typename type> inline type ReadValue(LexerIOWrapper *in)
|
||
{
|
||
type ret;
|
||
|
||
in->Read(&ret, sizeof(type));
|
||
return ret;
|
||
}
|
||
|
||
/*
|
||
================
|
||
ReadValue
|
||
specialization read for idStr's
|
||
================
|
||
*/
|
||
template <> inline idStr ReadValue(LexerIOWrapper *in)
|
||
{
|
||
char c;
|
||
idStr str;
|
||
|
||
in->Read(&c, 1);
|
||
while(c != '\0')
|
||
{
|
||
str.Append(c);
|
||
in->Read(&c, 1);
|
||
}
|
||
|
||
str.Append(c);
|
||
|
||
return str;
|
||
}
|
||
|
||
#endif
|
||
|
||
#define PUNCTABLE
|
||
|
||
//longer punctuations first
|
||
punctuation_t default_punctuations[] = {
|
||
//binary operators
|
||
{">>=",P_RSHIFT_ASSIGN},
|
||
{"<<=",P_LSHIFT_ASSIGN},
|
||
//
|
||
{"...",P_PARMS},
|
||
//define merge operator
|
||
{"##",P_PRECOMPMERGE}, // pre-compiler
|
||
//logic operators
|
||
{"&&",P_LOGIC_AND}, // pre-compiler
|
||
{"||",P_LOGIC_OR}, // pre-compiler
|
||
{">=",P_LOGIC_GEQ}, // pre-compiler
|
||
{"<=",P_LOGIC_LEQ}, // pre-compiler
|
||
{"==",P_LOGIC_EQ}, // pre-compiler
|
||
{"!=",P_LOGIC_UNEQ}, // pre-compiler
|
||
//arithmatic operators
|
||
{"*=",P_MUL_ASSIGN},
|
||
{"/=",P_DIV_ASSIGN},
|
||
{"%=",P_MOD_ASSIGN},
|
||
{"+=",P_ADD_ASSIGN},
|
||
{"-=",P_SUB_ASSIGN},
|
||
{"++",P_INC},
|
||
{"--",P_DEC},
|
||
//binary operators
|
||
{"&=",P_BIN_AND_ASSIGN},
|
||
{"|=",P_BIN_OR_ASSIGN},
|
||
{"^=",P_BIN_XOR_ASSIGN},
|
||
{">>",P_RSHIFT}, // pre-compiler
|
||
{"<<",P_LSHIFT}, // pre-compiler
|
||
//reference operators
|
||
{"->",P_POINTERREF},
|
||
//C++
|
||
{"::",P_CPP1},
|
||
{".*",P_CPP2},
|
||
//arithmatic operators
|
||
{"*",P_MUL}, // pre-compiler
|
||
{"/",P_DIV}, // pre-compiler
|
||
{"%",P_MOD}, // pre-compiler
|
||
{"+",P_ADD}, // pre-compiler
|
||
{"-",P_SUB}, // pre-compiler
|
||
{"=",P_ASSIGN},
|
||
//binary operators
|
||
{"&",P_BIN_AND}, // pre-compiler
|
||
{"|",P_BIN_OR}, // pre-compiler
|
||
{"^",P_BIN_XOR}, // pre-compiler
|
||
{"~",P_BIN_NOT}, // pre-compiler
|
||
//logic operators
|
||
{"!",P_LOGIC_NOT}, // pre-compiler
|
||
{">",P_LOGIC_GREATER}, // pre-compiler
|
||
{"<",P_LOGIC_LESS}, // pre-compiler
|
||
//reference operator
|
||
{".",P_REF},
|
||
//seperators
|
||
{",",P_COMMA}, // pre-compiler
|
||
{";",P_SEMICOLON},
|
||
//label indication
|
||
{":",P_COLON}, // pre-compiler
|
||
//if statement
|
||
{"?",P_QUESTIONMARK}, // pre-compiler
|
||
//embracements
|
||
{"(",P_PARENTHESESOPEN}, // pre-compiler
|
||
{")",P_PARENTHESESCLOSE}, // pre-compiler
|
||
{"{",P_BRACEOPEN}, // pre-compiler
|
||
{"}",P_BRACECLOSE}, // pre-compiler
|
||
{"[",P_SQBRACKETOPEN},
|
||
{"]",P_SQBRACKETCLOSE},
|
||
//
|
||
{"\\",P_BACKSLASH},
|
||
//precompiler operator
|
||
{"#",P_PRECOMP}, // pre-compiler
|
||
{"$",P_DOLLAR},
|
||
// RAVEN BEGIN
|
||
{"<EFBFBD>",P_INVERTED_PLING},
|
||
{"<EFBFBD>",P_INVERTED_QUERY},
|
||
// RAVEN END
|
||
{NULL, 0}
|
||
};
|
||
|
||
int default_punctuationtable[256];
|
||
int default_nextpunctuation[sizeof(default_punctuations) / sizeof(punctuation_t)];
|
||
int default_setup;
|
||
|
||
// RAVEN BEGIN
|
||
// jsinger: changed to be Lexer instead of idLexer so that we have the ability to read binary files
|
||
char Lexer::baseFolder[ 256 ];
|
||
|
||
// Added this to allow easy changing of the suffix that signifies a binary file
|
||
idStr const Lexer::sCompiledFileSuffix("c");
|
||
// RAVEN END
|
||
|
||
/*
|
||
================
|
||
idLexer::CreatePunctuationTable
|
||
================
|
||
*/
|
||
void idLexer::CreatePunctuationTable( const punctuation_t *punctuations ) {
|
||
int i, n, lastp;
|
||
const punctuation_t *p, *newp;
|
||
|
||
//get memory for the table
|
||
if ( punctuations == default_punctuations ) {
|
||
idLexer::punctuationtable = default_punctuationtable;
|
||
idLexer::nextpunctuation = default_nextpunctuation;
|
||
if ( default_setup ) {
|
||
return;
|
||
}
|
||
default_setup = true;
|
||
i = sizeof(default_punctuations) / sizeof(punctuation_t);
|
||
}
|
||
else {
|
||
if ( !idLexer::punctuationtable || idLexer::punctuationtable == default_punctuationtable ) {
|
||
//RAVEN BEGIN
|
||
//amccarthy: Added memory allocation tag
|
||
idLexer::punctuationtable = (int *) Mem_Alloc(256 * sizeof(int),MA_LEXER);
|
||
//RAVEN END
|
||
}
|
||
if ( idLexer::nextpunctuation && idLexer::nextpunctuation != default_nextpunctuation ) {
|
||
Mem_Free( idLexer::nextpunctuation );
|
||
}
|
||
for (i = 0; punctuations[i].p; i++) {
|
||
}
|
||
//RAVEN BEGIN
|
||
//amccarthy: Added memory allocation tag
|
||
idLexer::nextpunctuation = (int *) Mem_Alloc(i * sizeof(int),MA_LEXER);
|
||
//RAVEN END
|
||
}
|
||
memset(idLexer::punctuationtable, 0xFF, 256 * sizeof(int));
|
||
memset(idLexer::nextpunctuation, 0xFF, i * sizeof(int));
|
||
//add the punctuations in the list to the punctuation table
|
||
for (i = 0; punctuations[i].p; i++) {
|
||
newp = &punctuations[i];
|
||
lastp = -1;
|
||
//sort the punctuations in this table entry on length (longer punctuations first)
|
||
for (n = idLexer::punctuationtable[(unsigned int) newp->p[0]]; n >= 0; n = idLexer::nextpunctuation[n] ) {
|
||
p = &punctuations[n];
|
||
if (strlen(p->p) < strlen(newp->p)) {
|
||
idLexer::nextpunctuation[i] = n;
|
||
if (lastp >= 0) {
|
||
idLexer::nextpunctuation[lastp] = i;
|
||
}
|
||
else {
|
||
idLexer::punctuationtable[(unsigned int) newp->p[0]] = i;
|
||
}
|
||
break;
|
||
}
|
||
lastp = n;
|
||
}
|
||
if (n < 0) {
|
||
idLexer::nextpunctuation[i] = -1;
|
||
if (lastp >= 0) {
|
||
idLexer::nextpunctuation[lastp] = i;
|
||
}
|
||
else {
|
||
idLexer::punctuationtable[(unsigned int) newp->p[0]] = i;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::GetPunctuationFromId
|
||
================
|
||
*/
|
||
const char *idLexer::GetPunctuationFromId( int id ) {
|
||
int i;
|
||
|
||
for (i = 0; idLexer::punctuations[i].p; i++) {
|
||
if ( idLexer::punctuations[i].n == id ) {
|
||
return idLexer::punctuations[i].p;
|
||
}
|
||
}
|
||
return "unkown punctuation";
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::GetPunctuationId
|
||
================
|
||
*/
|
||
int idLexer::GetPunctuationId( const char *p ) {
|
||
int i;
|
||
|
||
for (i = 0; idLexer::punctuations[i].p; i++) {
|
||
if ( !idStr::Cmp(idLexer::punctuations[i].p, p) ) {
|
||
return idLexer::punctuations[i].n;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::Error
|
||
================
|
||
*/
|
||
|
||
void idLexer::Error( const char *str, ... ) {
|
||
char text[MAX_STRING_CHARS];
|
||
va_list ap;
|
||
|
||
hadError = true;
|
||
|
||
if ( idLexer::flags & LEXFL_NOERRORS ) {
|
||
return;
|
||
}
|
||
|
||
va_start(ap, str);
|
||
vsprintf(text, str, ap);
|
||
va_end(ap);
|
||
|
||
if ( idLexer::flags & LEXFL_NOFATALERRORS ) {
|
||
idLib::common->Warning( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text );
|
||
} else {
|
||
idLib::common->Error( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text );
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::Warning
|
||
================
|
||
*/
|
||
void idLexer::Warning( const char *str, ... ) {
|
||
char text[MAX_STRING_CHARS];
|
||
va_list ap;
|
||
|
||
if ( idLexer::flags & LEXFL_NOWARNINGS ) {
|
||
return;
|
||
}
|
||
|
||
va_start( ap, str );
|
||
vsprintf( text, str, ap );
|
||
va_end( ap );
|
||
idLib::common->Warning( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text );
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::SetPunctuations
|
||
================
|
||
*/
|
||
void idLexer::SetPunctuations( const punctuation_t *p ) {
|
||
#ifdef PUNCTABLE
|
||
if (p) {
|
||
idLexer::CreatePunctuationTable( p );
|
||
}
|
||
else {
|
||
idLexer::CreatePunctuationTable( default_punctuations );
|
||
}
|
||
#endif //PUNCTABLE
|
||
if (p) {
|
||
idLexer::punctuations = p;
|
||
}
|
||
else {
|
||
idLexer::punctuations = default_punctuations;
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadWhiteSpace
|
||
|
||
Reads spaces, tabs, C-like comments etc.
|
||
When a newline character is found the scripts line counter is increased.
|
||
================
|
||
*/
|
||
int idLexer::ReadWhiteSpace( void ) {
|
||
while(1) {
|
||
// skip white space
|
||
// RAVEN BEGIN
|
||
while(( byte )*idLexer::script_p <= ' ') {
|
||
// RAVEN END
|
||
if (!*idLexer::script_p) {
|
||
return 0;
|
||
}
|
||
if (*idLexer::script_p == '\n') {
|
||
idLexer::line++;
|
||
}
|
||
idLexer::script_p++;
|
||
}
|
||
// skip comments
|
||
if (*idLexer::script_p == '/') {
|
||
// comments //
|
||
if (*(idLexer::script_p+1) == '/') {
|
||
idLexer::script_p++;
|
||
do {
|
||
idLexer::script_p++;
|
||
if ( !*idLexer::script_p ) {
|
||
return 0;
|
||
}
|
||
}
|
||
while( *idLexer::script_p != '\n' );
|
||
idLexer::line++;
|
||
idLexer::script_p++;
|
||
if ( !*idLexer::script_p ) {
|
||
return 0;
|
||
}
|
||
continue;
|
||
}
|
||
// comments /* */
|
||
else if (*(idLexer::script_p+1) == '*') {
|
||
idLexer::script_p++;
|
||
while( 1 ) {
|
||
idLexer::script_p++;
|
||
if ( !*idLexer::script_p ) {
|
||
return 0;
|
||
}
|
||
if ( *idLexer::script_p == '\n' ) {
|
||
idLexer::line++;
|
||
}
|
||
else if ( *idLexer::script_p == '/' ) {
|
||
if ( *(idLexer::script_p-1) == '*' ) {
|
||
break;
|
||
}
|
||
if ( *(idLexer::script_p+1) == '*' ) {
|
||
idLexer::Warning( "nested comment" );
|
||
}
|
||
}
|
||
}
|
||
idLexer::script_p++;
|
||
if ( !*idLexer::script_p ) {
|
||
return 0;
|
||
}
|
||
idLexer::script_p++;
|
||
if ( !*idLexer::script_p ) {
|
||
return 0;
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadEscapeCharacter
|
||
================
|
||
*/
|
||
int idLexer::ReadEscapeCharacter( char *ch ) {
|
||
int c, val, i;
|
||
|
||
// step over the leading '\\'
|
||
idLexer::script_p++;
|
||
// determine the escape character
|
||
switch(*idLexer::script_p) {
|
||
case '\\': c = '\\'; break;
|
||
case 'n': c = '\n'; break;
|
||
case 'r': c = '\r'; break;
|
||
case 't': c = '\t'; break;
|
||
case 'v': c = '\v'; break;
|
||
case 'b': c = '\b'; break;
|
||
case 'f': c = '\f'; break;
|
||
case 'a': c = '\a'; break;
|
||
case '\'': c = '\''; break;
|
||
case '\"': c = '\"'; break;
|
||
case '\?': c = '\?'; break;
|
||
case 'x':
|
||
{
|
||
idLexer::script_p++;
|
||
for (i = 0, val = 0; ; i++, idLexer::script_p++) {
|
||
c = *idLexer::script_p;
|
||
if (c >= '0' && c <= '9')
|
||
c = c - '0';
|
||
else if (c >= 'A' && c <= 'Z')
|
||
c = c - 'A' + 10;
|
||
else if (c >= 'a' && c <= 'z')
|
||
c = c - 'a' + 10;
|
||
else
|
||
break;
|
||
val = (val << 4) + c;
|
||
}
|
||
idLexer::script_p--;
|
||
if (val > 0xFF) {
|
||
idLexer::Warning( "too large value in escape character" );
|
||
val = 0xFF;
|
||
}
|
||
c = val;
|
||
break;
|
||
}
|
||
default: //NOTE: decimal ASCII code, NOT octal
|
||
{
|
||
if (*idLexer::script_p < '0' || *idLexer::script_p > '9') {
|
||
idLexer::Error("unknown escape char");
|
||
}
|
||
for (i = 0, val = 0; ; i++, idLexer::script_p++) {
|
||
c = *idLexer::script_p;
|
||
if (c >= '0' && c <= '9')
|
||
c = c - '0';
|
||
else
|
||
break;
|
||
val = val * 10 + c;
|
||
}
|
||
idLexer::script_p--;
|
||
if (val > 0xFF) {
|
||
idLexer::Warning( "too large value in escape character" );
|
||
val = 0xFF;
|
||
}
|
||
c = val;
|
||
break;
|
||
}
|
||
}
|
||
// step over the escape character or the last digit of the number
|
||
idLexer::script_p++;
|
||
// store the escape character
|
||
*ch = c;
|
||
// succesfully read escape character
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadString
|
||
|
||
Escape characters are interpretted.
|
||
Reads two strings with only a white space between them as one string.
|
||
================
|
||
*/
|
||
int idLexer::ReadString( idToken *token, int quote ) {
|
||
int tmpline;
|
||
const char *tmpscript_p;
|
||
char ch;
|
||
|
||
if ( quote == '\"' ) {
|
||
token->type = TT_STRING;
|
||
} else {
|
||
token->type = TT_LITERAL;
|
||
}
|
||
|
||
// leading quote
|
||
idLexer::script_p++;
|
||
|
||
while(1) {
|
||
// if there is an escape character and escape characters are allowed
|
||
if (*idLexer::script_p == '\\' && !(idLexer::flags & LEXFL_NOSTRINGESCAPECHARS)) {
|
||
if ( !idLexer::ReadEscapeCharacter( &ch ) ) {
|
||
return 0;
|
||
}
|
||
token->AppendDirty( ch );
|
||
}
|
||
// if a trailing quote
|
||
else if (*idLexer::script_p == quote) {
|
||
// step over the quote
|
||
idLexer::script_p++;
|
||
// if consecutive strings should not be concatenated
|
||
if ( (idLexer::flags & LEXFL_NOSTRINGCONCAT) &&
|
||
(!(idLexer::flags & LEXFL_ALLOWBACKSLASHSTRINGCONCAT) || (quote != '\"')) ) {
|
||
break;
|
||
}
|
||
|
||
tmpscript_p = idLexer::script_p;
|
||
tmpline = idLexer::line;
|
||
// read white space between possible two consecutive strings
|
||
if ( !idLexer::ReadWhiteSpace() ) {
|
||
idLexer::script_p = tmpscript_p;
|
||
idLexer::line = tmpline;
|
||
break;
|
||
}
|
||
|
||
if ( idLexer::flags & LEXFL_NOSTRINGCONCAT ) {
|
||
if ( *idLexer::script_p != '\\' ) {
|
||
idLexer::script_p = tmpscript_p;
|
||
idLexer::line = tmpline;
|
||
break;
|
||
}
|
||
// step over the '\\'
|
||
idLexer::script_p++;
|
||
if ( !idLexer::ReadWhiteSpace() || ( *idLexer::script_p != quote ) ) {
|
||
idLexer::Error( "expecting string after '\' terminated line" );
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
// if there's no leading qoute
|
||
if ( *idLexer::script_p != quote ) {
|
||
idLexer::script_p = tmpscript_p;
|
||
idLexer::line = tmpline;
|
||
break;
|
||
}
|
||
// step over the new leading quote
|
||
idLexer::script_p++;
|
||
}
|
||
else {
|
||
if (*idLexer::script_p == '\0') {
|
||
idLexer::Error( "missing trailing quote" );
|
||
return 0;
|
||
}
|
||
if (*idLexer::script_p == '\n') {
|
||
idLexer::Error( "newline inside string" );
|
||
return 0;
|
||
}
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
}
|
||
}
|
||
token->data[token->len] = '\0';
|
||
|
||
if ( token->type == TT_LITERAL ) {
|
||
if ( !(idLexer::flags & LEXFL_ALLOWMULTICHARLITERALS) ) {
|
||
if ( token->Length() != 1 ) {
|
||
idLexer::Warning( "literal is not one character long" );
|
||
}
|
||
}
|
||
token->subtype = (*token)[0];
|
||
}
|
||
else {
|
||
// the sub type is the length of the string
|
||
token->subtype = token->Length();
|
||
}
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadName
|
||
================
|
||
*/
|
||
int idLexer::ReadName( idToken *token ) {
|
||
char c;
|
||
|
||
token->type = TT_NAME;
|
||
do {
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
c = *idLexer::script_p;
|
||
// RAVEN BEGIN
|
||
} while ( idStr::CharIsAlpha( c ) || idStr::CharIsNumeric( c ) || c == '_' ||
|
||
// RAVEN EBD
|
||
// if treating all tokens as strings, don't parse '-' as a seperate token
|
||
((idLexer::flags & LEXFL_ONLYSTRINGS) && (c == '-')) ||
|
||
// if special path name characters are allowed
|
||
((idLexer::flags & LEXFL_ALLOWPATHNAMES) && (c == '/' || c == '\\' || c == ':' || c == '.')) );
|
||
token->data[token->len] = '\0';
|
||
//the sub type is the length of the name
|
||
token->subtype = token->Length();
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::CheckString
|
||
================
|
||
*/
|
||
ID_INLINE int idLexer::CheckString( const char *str ) const {
|
||
int i;
|
||
|
||
for ( i = 0; str[i]; i++ ) {
|
||
if ( idLexer::script_p[i] != str[i] ) {
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadNumber
|
||
================
|
||
*/
|
||
int idLexer::ReadNumber( idToken *token ) {
|
||
int i;
|
||
int dot;
|
||
char c, c2;
|
||
|
||
token->type = TT_NUMBER;
|
||
token->subtype = 0;
|
||
token->intvalue = 0;
|
||
token->floatvalue = 0;
|
||
|
||
c = *idLexer::script_p;
|
||
c2 = *(idLexer::script_p + 1);
|
||
|
||
if ( c == '0' && c2 != '.' ) {
|
||
// check for a hexadecimal number
|
||
if ( c2 == 'x' || c2 == 'X' ) {
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
c = *idLexer::script_p;
|
||
while((c >= '0' && c <= '9') ||
|
||
(c >= 'a' && c <= 'f') ||
|
||
(c >= 'A' && c <= 'F')) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
token->subtype = TT_HEX | TT_INTEGER;
|
||
}
|
||
// check for a binary number
|
||
else if ( c2 == 'b' || c2 == 'B' ) {
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
c = *idLexer::script_p;
|
||
while( c == '0' || c == '1' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
token->subtype = TT_BINARY | TT_INTEGER;
|
||
}
|
||
// its an octal number
|
||
else {
|
||
token->AppendDirty( *idLexer::script_p++ );
|
||
c = *idLexer::script_p;
|
||
while( c >= '0' && c <= '7' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
token->subtype = TT_OCTAL | TT_INTEGER;
|
||
}
|
||
}
|
||
else {
|
||
// decimal integer or floating point number or ip address
|
||
dot = 0;
|
||
while( 1 ) {
|
||
if ( c >= '0' && c <= '9' ) {
|
||
}
|
||
else if ( c == '.' ) {
|
||
dot++;
|
||
}
|
||
else {
|
||
break;
|
||
}
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
if( c == 'e' && dot == 0) {
|
||
//We have scientific notation without a decimal point
|
||
dot++;
|
||
}
|
||
// if a floating point number
|
||
if ( dot == 1 ) {
|
||
token->subtype = TT_DECIMAL | TT_FLOAT;
|
||
// check for floating point exponent
|
||
if ( c == 'e' ) {
|
||
//Append the e so that GetFloatValue code works
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
if ( c == '-' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
else if ( c == '+' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
while( c >= '0' && c <= '9' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
}
|
||
// check for floating point exception infinite 1.#INF or indefinite 1.#IND or NaN
|
||
else if ( c == '#' ) {
|
||
c2 = 4;
|
||
if ( CheckString( "INF" ) ) {
|
||
token->subtype |= TT_INFINITE;
|
||
}
|
||
else if ( CheckString( "IND" ) ) {
|
||
token->subtype |= TT_INDEFINITE;
|
||
}
|
||
else if ( CheckString( "NAN" ) ) {
|
||
token->subtype |= TT_NAN;
|
||
}
|
||
else if ( CheckString( "QNAN" ) ) {
|
||
token->subtype |= TT_NAN;
|
||
c2++;
|
||
}
|
||
else if ( CheckString( "SNAN" ) ) {
|
||
token->subtype |= TT_NAN;
|
||
c2++;
|
||
}
|
||
for ( i = 0; i < c2; i++ ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
while( c >= '0' && c <= '9' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
if ( !(idLexer::flags & LEXFL_ALLOWFLOATEXCEPTIONS) ) {
|
||
token->AppendDirty( 0 ); // zero terminate for c_str
|
||
idLexer::Error( "parsed %s", token->c_str() );
|
||
}
|
||
}
|
||
}
|
||
else if ( dot > 1 ) {
|
||
if ( !( idLexer::flags & LEXFL_ALLOWIPADDRESSES ) ) {
|
||
idLexer::Error( "more than one dot in number" );
|
||
return 0;
|
||
}
|
||
if ( dot != 3 ) {
|
||
idLexer::Error( "ip address should have three dots" );
|
||
return 0;
|
||
}
|
||
token->subtype = TT_IPADDRESS;
|
||
}
|
||
else {
|
||
token->subtype = TT_DECIMAL | TT_INTEGER;
|
||
}
|
||
}
|
||
|
||
if ( token->subtype & TT_FLOAT ) {
|
||
if ( c > ' ' ) {
|
||
// single-precision: float
|
||
if ( c == 'f' || c == 'F' ) {
|
||
token->subtype |= TT_SINGLE_PRECISION;
|
||
idLexer::script_p++;
|
||
}
|
||
// extended-precision: long double
|
||
else if ( c == 'l' || c == 'L' ) {
|
||
token->subtype |= TT_EXTENDED_PRECISION;
|
||
idLexer::script_p++;
|
||
}
|
||
// default is double-precision: double
|
||
else {
|
||
token->subtype |= TT_DOUBLE_PRECISION;
|
||
}
|
||
}
|
||
else {
|
||
token->subtype |= TT_DOUBLE_PRECISION;
|
||
}
|
||
}
|
||
else if ( token->subtype & TT_INTEGER ) {
|
||
if ( c > ' ' ) {
|
||
// default: signed long
|
||
for ( i = 0; i < 2; i++ ) {
|
||
// long integer
|
||
if ( c == 'l' || c == 'L' ) {
|
||
token->subtype |= TT_LONG;
|
||
}
|
||
// unsigned integer
|
||
else if ( c == 'u' || c == 'U' ) {
|
||
token->subtype |= TT_UNSIGNED;
|
||
}
|
||
else {
|
||
break;
|
||
}
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
}
|
||
}
|
||
else if ( token->subtype & TT_IPADDRESS ) {
|
||
if ( c == ':' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
while( c >= '0' && c <= '9' ) {
|
||
token->AppendDirty( c );
|
||
c = *(++idLexer::script_p);
|
||
}
|
||
token->subtype |= TT_IPPORT;
|
||
}
|
||
}
|
||
token->data[token->len] = '\0';
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadPunctuation
|
||
================
|
||
*/
|
||
int idLexer::ReadPunctuation( idToken *token ) {
|
||
int l, n, i;
|
||
char *p;
|
||
const punctuation_t *punc;
|
||
|
||
#ifdef PUNCTABLE
|
||
for (n = idLexer::punctuationtable[(unsigned int)*(idLexer::script_p)]; n >= 0; n = idLexer::nextpunctuation[n])
|
||
{
|
||
punc = &(idLexer::punctuations[n]);
|
||
#else
|
||
int i;
|
||
|
||
for (i = 0; idLexer::punctuations[i].p; i++) {
|
||
punc = &idLexer::punctuations[i];
|
||
#endif
|
||
p = punc->p;
|
||
// check for this punctuation in the script
|
||
for ( l = 0; p[l] && idLexer::script_p[l]; l++ ) {
|
||
if ( idLexer::script_p[l] != p[l] ) {
|
||
break;
|
||
}
|
||
}
|
||
if ( !p[l] ) {
|
||
//
|
||
token->EnsureAlloced( l+1, false );
|
||
for ( i = 0; i <= l; i++ ) {
|
||
token->data[i] = p[i];
|
||
}
|
||
token->len = l;
|
||
//
|
||
idLexer::script_p += l;
|
||
token->type = TT_PUNCTUATION;
|
||
// sub type is the punctuation id
|
||
token->subtype = punc->n;
|
||
return 1;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadToken
|
||
================
|
||
*/
|
||
int idLexer::ReadToken( idToken *token ) {
|
||
int c;
|
||
|
||
if ( !loaded ) {
|
||
// RAVEN BEGIN
|
||
#ifndef _XENON
|
||
// nrausch: should not be an error on xenon since it prevents the precache stuff from working
|
||
idLib::common->Error( "idLexer::ReadToken: no file loaded" );
|
||
#else
|
||
idLib::common->Warning( "idLexer::ReadToken: no file loaded" );
|
||
#endif
|
||
// RAVEN END
|
||
return 0;
|
||
}
|
||
|
||
// if there is a token available (from unreadToken)
|
||
if ( tokenavailable ) {
|
||
tokenavailable = 0;
|
||
*token = idLexer::token;
|
||
return 1;
|
||
}
|
||
// save script pointer
|
||
lastScript_p = script_p;
|
||
// save line counter
|
||
lastline = line;
|
||
// clear the token stuff
|
||
token->data[0] = '\0';
|
||
token->len = 0;
|
||
// start of the white space
|
||
whiteSpaceStart_p = script_p;
|
||
token->whiteSpaceStart_p = script_p;
|
||
// read white space before token
|
||
if ( !ReadWhiteSpace() ) {
|
||
return 0;
|
||
}
|
||
// end of the white space
|
||
idLexer::whiteSpaceEnd_p = script_p;
|
||
token->whiteSpaceEnd_p = script_p;
|
||
// line the token is on
|
||
token->line = line;
|
||
// number of lines crossed before token
|
||
token->linesCrossed = line - lastline;
|
||
// clear token flags
|
||
token->flags = 0;
|
||
|
||
c = *idLexer::script_p;
|
||
|
||
// if we're keeping everything as whitespace deliminated strings
|
||
if ( idLexer::flags & LEXFL_ONLYSTRINGS ) {
|
||
// if there is a leading quote
|
||
if ( c == '\"' || c == '\'' ) {
|
||
if (!idLexer::ReadString( token, c )) {
|
||
return 0;
|
||
}
|
||
} else if ( !idLexer::ReadName( token ) ) {
|
||
return 0;
|
||
}
|
||
}
|
||
// if there is a number
|
||
else if ( (c >= '0' && c <= '9') ||
|
||
(c == '.' && (*(idLexer::script_p + 1) >= '0' && *(idLexer::script_p + 1) <= '9')) ) {
|
||
if ( !idLexer::ReadNumber( token ) ) {
|
||
return 0;
|
||
}
|
||
// if names are allowed to start with a number
|
||
if ( idLexer::flags & LEXFL_ALLOWNUMBERNAMES ) {
|
||
c = *idLexer::script_p;
|
||
if ( idStr::CharIsAlpha( c ) || idStr::CharIsNumeric( c ) || c == '_' ) {
|
||
if ( !idLexer::ReadName( token ) ) {
|
||
return 0;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// if there is a leading quote
|
||
else if ( c == '\"' || c == '\'' ) {
|
||
if (!idLexer::ReadString( token, c )) {
|
||
return 0;
|
||
}
|
||
}
|
||
// if there is a name
|
||
// RAVEN BEGIN
|
||
else if ( idStr::CharIsAlpha( c ) || idStr::CharIsNumeric( c ) || c == '_' ) {
|
||
// RAVEN END
|
||
if ( !idLexer::ReadName( token ) ) {
|
||
return 0;
|
||
}
|
||
}
|
||
// names may also start with a slash when pathnames are allowed
|
||
else if ( ( idLexer::flags & LEXFL_ALLOWPATHNAMES ) && ( (c == '/' || c == '\\') || c == '.' ) ) {
|
||
if ( !idLexer::ReadName( token ) ) {
|
||
return 0;
|
||
}
|
||
}
|
||
// check for punctuations
|
||
else if ( !idLexer::ReadPunctuation( token ) ) {
|
||
idLexer::Error( "unknown punctuation %c", c );
|
||
return 0;
|
||
}
|
||
// RAVEN BEGIN
|
||
// jsinger: added to write out the binary version of this token when binary writes have been turned on
|
||
WriteBinaryToken(token);
|
||
|
||
// RAVEN END
|
||
// succesfully read a token
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ExpectTokenString
|
||
================
|
||
*/
|
||
int idLexer::ExpectTokenString( const char *string ) {
|
||
idToken token;
|
||
|
||
if (!idLexer::ReadToken( &token )) {
|
||
idLexer::Error( "couldn't find expected '%s'", string );
|
||
return 0;
|
||
}
|
||
if ( token != string ) {
|
||
idLexer::Error( "expected '%s' but found '%s'", string, token.c_str() );
|
||
return 0;
|
||
}
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ExpectTokenType
|
||
================
|
||
*/
|
||
int idLexer::ExpectTokenType( int type, int subtype, idToken *token ) {
|
||
idStr str;
|
||
|
||
if ( !idLexer::ReadToken( token ) ) {
|
||
idLexer::Error( "couldn't read expected token" );
|
||
return 0;
|
||
}
|
||
|
||
if ( token->type != type ) {
|
||
switch( type ) {
|
||
case TT_STRING: str = "string"; break;
|
||
case TT_LITERAL: str = "literal"; break;
|
||
case TT_NUMBER: str = "number"; break;
|
||
case TT_NAME: str = "name"; break;
|
||
case TT_PUNCTUATION: str = "punctuation"; break;
|
||
default: str = "unknown type"; break;
|
||
}
|
||
idLexer::Error( "expected a %s but found '%s'", str.c_str(), token->c_str() );
|
||
return 0;
|
||
}
|
||
if ( token->type == TT_NUMBER ) {
|
||
if ( (token->subtype & subtype) != subtype ) {
|
||
str.Clear();
|
||
if ( subtype & TT_DECIMAL ) str = "decimal ";
|
||
if ( subtype & TT_HEX ) str = "hex ";
|
||
if ( subtype & TT_OCTAL ) str = "octal ";
|
||
if ( subtype & TT_BINARY ) str = "binary ";
|
||
if ( subtype & TT_UNSIGNED ) str += "unsigned ";
|
||
if ( subtype & TT_LONG ) str += "long ";
|
||
if ( subtype & TT_FLOAT ) str += "float ";
|
||
if ( subtype & TT_INTEGER ) str += "integer ";
|
||
str.StripTrailing( ' ' );
|
||
idLexer::Error( "expected %s but found '%s'", str.c_str(), token->c_str() );
|
||
return 0;
|
||
}
|
||
}
|
||
else if ( token->type == TT_PUNCTUATION ) {
|
||
if ( subtype < 0 ) {
|
||
idLexer::Error( "BUG: wrong punctuation subtype" );
|
||
return 0;
|
||
}
|
||
if ( token->subtype != subtype ) {
|
||
idLexer::Error( "expected '%s' but found '%s'", GetPunctuationFromId( subtype ), token->c_str() );
|
||
return 0;
|
||
}
|
||
}
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ExpectAnyToken
|
||
================
|
||
*/
|
||
int idLexer::ExpectAnyToken( idToken *token ) {
|
||
if (!idLexer::ReadToken( token )) {
|
||
idLexer::Error( "couldn't read expected token" );
|
||
return 0;
|
||
}
|
||
else {
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::CheckTokenString
|
||
================
|
||
*/
|
||
int idLexer::CheckTokenString( const char *string ) {
|
||
idToken tok;
|
||
|
||
if (!idLexer::ReadToken( &tok )) {
|
||
return 0;
|
||
}
|
||
|
||
// if the token is available
|
||
if ( tok == string ) {
|
||
return 1;
|
||
}
|
||
// token not available
|
||
UnreadToken( &tok );
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::CheckTokenType
|
||
================
|
||
*/
|
||
int idLexer::CheckTokenType( int type, int subtype, idToken *token ) {
|
||
idToken tok;
|
||
|
||
if (!idLexer::ReadToken( &tok )) {
|
||
return 0;
|
||
}
|
||
// if the type matches
|
||
if (tok.type == type && (tok.subtype & subtype) == subtype) {
|
||
*token = tok;
|
||
return 1;
|
||
}
|
||
// token is not available
|
||
idLexer::script_p = lastScript_p;
|
||
idLexer::line = lastline;
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::SkipUntilString
|
||
================
|
||
*/
|
||
int idLexer::SkipUntilString( const char *string ) {
|
||
idToken token;
|
||
|
||
while(idLexer::ReadToken( &token )) {
|
||
if ( token == string ) {
|
||
return 1;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::SkipRestOfLine
|
||
================
|
||
*/
|
||
int idLexer::SkipRestOfLine( void ) {
|
||
idToken token;
|
||
|
||
while(idLexer::ReadToken( &token )) {
|
||
if ( token.linesCrossed ) {
|
||
idLexer::script_p = lastScript_p;
|
||
idLexer::line = lastline;
|
||
return 1;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
=================
|
||
idLexer::SkipBracedSection
|
||
|
||
Skips until a matching close brace is found.
|
||
Internal brace depths are properly skipped.
|
||
=================
|
||
*/
|
||
int idLexer::SkipBracedSection( bool parseFirstBrace ) {
|
||
idToken token;
|
||
int depth;
|
||
|
||
depth = parseFirstBrace ? 0 : 1;
|
||
do {
|
||
if ( !ReadToken( &token ) ) {
|
||
return false;
|
||
}
|
||
if ( token.type == TT_PUNCTUATION ) {
|
||
if ( token == "{" ) {
|
||
depth++;
|
||
} else if ( token == "}" ) {
|
||
depth--;
|
||
}
|
||
}
|
||
} while( depth );
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::UnreadToken
|
||
================
|
||
*/
|
||
void idLexer::UnreadToken( const idToken *token ) {
|
||
if ( idLexer::tokenavailable ) {
|
||
idLib::common->FatalError( "idLexer::unreadToken, unread token twice\n" );
|
||
}
|
||
idLexer::token = *token;
|
||
idLexer::tokenavailable = 1;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadTokenOnLine
|
||
================
|
||
*/
|
||
int idLexer::ReadTokenOnLine( idToken *token ) {
|
||
idToken tok;
|
||
|
||
if (!idLexer::ReadToken( &tok )) {
|
||
idLexer::script_p = lastScript_p;
|
||
idLexer::line = lastline;
|
||
return false;
|
||
}
|
||
// if no lines were crossed before this token
|
||
if ( !tok.linesCrossed ) {
|
||
*token = tok;
|
||
return true;
|
||
}
|
||
// restore our position
|
||
idLexer::script_p = lastScript_p;
|
||
idLexer::line = lastline;
|
||
token->Clear();
|
||
return false;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ReadRestOfLine
|
||
================
|
||
*/
|
||
const char* idLexer::ReadRestOfLine(idStr& out) {
|
||
while(1) {
|
||
|
||
if(*idLexer::script_p == '\n') {
|
||
idLexer::line++;
|
||
break;
|
||
}
|
||
|
||
if(!*idLexer::script_p) {
|
||
break;
|
||
}
|
||
|
||
if(*idLexer::script_p <= ' ') {
|
||
out += " ";
|
||
} else {
|
||
out += *idLexer::script_p;
|
||
}
|
||
idLexer::script_p++;
|
||
|
||
}
|
||
|
||
out.Strip(' ');
|
||
return out.c_str();
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ParseInt
|
||
================
|
||
*/
|
||
int idLexer::ParseInt( void ) {
|
||
idToken token;
|
||
|
||
if ( !idLexer::ReadToken( &token ) ) {
|
||
idLexer::Error( "couldn't read expected integer" );
|
||
return 0;
|
||
}
|
||
if ( token.type == TT_PUNCTUATION && token == "-" ) {
|
||
idLexer::ExpectTokenType( TT_NUMBER, TT_INTEGER, &token );
|
||
return -((signed int) token.GetIntValue());
|
||
}
|
||
else if ( token.type != TT_NUMBER || token.subtype == TT_FLOAT ) {
|
||
idLexer::Error( "expected integer value, found '%s'", token.c_str() );
|
||
}
|
||
return token.GetIntValue();
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ParseBool
|
||
================
|
||
*/
|
||
bool idLexer::ParseBool( void ) {
|
||
idToken token;
|
||
|
||
if ( !idLexer::ExpectTokenType( TT_NUMBER, 0, &token ) ) {
|
||
idLexer::Error( "couldn't read expected boolean" );
|
||
return false;
|
||
}
|
||
return ( token.GetIntValue() != 0 );
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ParseFloat
|
||
================
|
||
*/
|
||
float idLexer::ParseFloat( bool *errorFlag ) {
|
||
idToken token;
|
||
|
||
if ( errorFlag ) {
|
||
*errorFlag = false;
|
||
}
|
||
|
||
if ( !idLexer::ReadToken( &token ) ) {
|
||
if ( errorFlag ) {
|
||
idLexer::Warning( "couldn't read expected floating point number" );
|
||
*errorFlag = true;
|
||
} else {
|
||
idLexer::Error( "couldn't read expected floating point number" );
|
||
}
|
||
return 0;
|
||
}
|
||
if ( token.type == TT_PUNCTUATION && token == "-" ) {
|
||
idLexer::ExpectTokenType( TT_NUMBER, 0, &token );
|
||
return -token.GetFloatValue();
|
||
}
|
||
else if ( token.type != TT_NUMBER ) {
|
||
if ( errorFlag ) {
|
||
idLexer::Warning( "expected float value, found '%s'", token.c_str() );
|
||
*errorFlag = true;
|
||
} else {
|
||
idLexer::Error( "expected float value, found '%s'", token.c_str() );
|
||
}
|
||
}
|
||
return token.GetFloatValue();
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::Parse1DMatrix
|
||
================
|
||
*/
|
||
int idLexer::Parse1DMatrix( int x, float *m ) {
|
||
int i;
|
||
|
||
if ( !idLexer::ExpectTokenString( "(" ) ) {
|
||
return false;
|
||
}
|
||
|
||
for ( i = 0; i < x; i++ ) {
|
||
m[i] = idLexer::ParseFloat();
|
||
}
|
||
|
||
if ( !idLexer::ExpectTokenString( ")" ) ) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
// RAVEN BEGIN
|
||
// rjohnson: added vertex color support to proc files. assume a default RGBA of 0x000000ff
|
||
/*
|
||
================
|
||
idLexer::Parse1DMatrixOpenEnded
|
||
================
|
||
*/
|
||
int idLexer::Parse1DMatrixOpenEnded( int MaxCount, float *m ) {
|
||
int i;
|
||
|
||
if ( !idLexer::ExpectTokenString( "(" ) ) {
|
||
return 0;
|
||
}
|
||
|
||
for ( i = 0; i < MaxCount; i++ ) {
|
||
idToken tok;
|
||
|
||
if (!idLexer::ReadToken( &tok )) {
|
||
return 0;
|
||
}
|
||
|
||
if ( tok == ")" ) {
|
||
return i;
|
||
}
|
||
|
||
idLexer::UnreadToken( &tok );
|
||
|
||
m[i] = idLexer::ParseFloat();
|
||
}
|
||
|
||
if ( !idLexer::ExpectTokenString( ")" ) ) {
|
||
return 0;
|
||
}
|
||
|
||
return i;
|
||
}
|
||
// RAVEN END
|
||
|
||
/*
|
||
================
|
||
idLexer::Parse2DMatrix
|
||
================
|
||
*/
|
||
int idLexer::Parse2DMatrix( int y, int x, float *m ) {
|
||
int i;
|
||
|
||
if ( !idLexer::ExpectTokenString( "(" ) ) {
|
||
return false;
|
||
}
|
||
|
||
for ( i = 0; i < y; i++ ) {
|
||
if ( !idLexer::Parse1DMatrix( x, m + i * x ) ) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if ( !idLexer::ExpectTokenString( ")" ) ) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::Parse3DMatrix
|
||
================
|
||
*/
|
||
int idLexer::Parse3DMatrix( int z, int y, int x, float *m ) {
|
||
int i;
|
||
|
||
if ( !idLexer::ExpectTokenString( "(" ) ) {
|
||
return false;
|
||
}
|
||
|
||
for ( i = 0 ; i < z; i++ ) {
|
||
if ( !idLexer::Parse2DMatrix( y, x, m + i * x*y ) ) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if ( !idLexer::ExpectTokenString( ")" ) ) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::ParseNumericStructArray
|
||
================
|
||
*/
|
||
void idLexer::ParseNumericStructArray( int numStructElements, int tokenSubTypeStructElements[], int arrayCount, byte *arrayStorage )
|
||
{
|
||
int arrayOffset, curElement;
|
||
|
||
for ( arrayOffset = 0; arrayOffset < arrayCount; arrayOffset++ )
|
||
{
|
||
for ( curElement = 0; curElement < numStructElements; curElement++ )
|
||
{
|
||
if ( tokenSubTypeStructElements[curElement] & TT_FLOAT )
|
||
{
|
||
*(float*)arrayStorage = idLexer::ParseFloat();
|
||
arrayStorage += sizeof(float);
|
||
}
|
||
else
|
||
{
|
||
*(int*)arrayStorage = idLexer::ParseInt();
|
||
arrayStorage += sizeof(int);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/*
|
||
=================
|
||
idParser::ParseBracedSection
|
||
|
||
The next token should be an open brace.
|
||
Parses until a matching close brace is found.
|
||
Maintains exact characters between braces.
|
||
|
||
FIXME: this should use ReadToken and replace the token white space with correct indents and newlines
|
||
=================
|
||
*/
|
||
const char *idLexer::ParseBracedSectionExact( idStr &out, int tabs ) {
|
||
int depth;
|
||
bool doTabs;
|
||
bool skipWhite;
|
||
|
||
out.Empty();
|
||
|
||
if ( !idLexer::ExpectTokenString( "{" ) ) {
|
||
return out.c_str( );
|
||
}
|
||
|
||
out = "{";
|
||
depth = 1;
|
||
skipWhite = false;
|
||
doTabs = tabs >= 0;
|
||
|
||
while( depth && *idLexer::script_p ) {
|
||
char c = *(idLexer::script_p++);
|
||
|
||
switch ( c ) {
|
||
case '\t':
|
||
case ' ': {
|
||
if ( skipWhite ) {
|
||
continue;
|
||
}
|
||
break;
|
||
}
|
||
case '\n': {
|
||
// RAVEN BEGIN
|
||
// jscott: now gives correct line number in error reports
|
||
line++;
|
||
// RAVEN END
|
||
if ( doTabs ) {
|
||
skipWhite = true;
|
||
out += c;
|
||
continue;
|
||
}
|
||
break;
|
||
}
|
||
case '{': {
|
||
depth++;
|
||
tabs++;
|
||
break;
|
||
}
|
||
case '}': {
|
||
depth--;
|
||
tabs--;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if ( skipWhite ) {
|
||
int i = tabs;
|
||
if ( c == '{' ) {
|
||
i--;
|
||
}
|
||
skipWhite = false;
|
||
for ( ; i > 0; i-- ) {
|
||
out += '\t';
|
||
}
|
||
}
|
||
out += c;
|
||
}
|
||
return out.c_str();
|
||
}
|
||
|
||
/*
|
||
=================
|
||
idLexer::ParseBracedSection
|
||
|
||
The next token should be an open brace.
|
||
Parses until a matching close brace is found.
|
||
Internal brace depths are properly skipped.
|
||
=================
|
||
*/
|
||
const char *idLexer::ParseBracedSection( idStr &out ) {
|
||
idToken token;
|
||
int i, depth;
|
||
|
||
out.Empty();
|
||
if ( !idLexer::ExpectTokenString( "{" ) ) {
|
||
return out.c_str();
|
||
}
|
||
out = "{";
|
||
depth = 1;
|
||
do {
|
||
if ( !idLexer::ReadToken( &token ) ) {
|
||
Error( "missing closing brace" );
|
||
return out.c_str();
|
||
}
|
||
|
||
// if the token is on a new line
|
||
for ( i = 0; i < token.linesCrossed; i++ ) {
|
||
out += "\r\n";
|
||
}
|
||
|
||
if ( token.type == TT_PUNCTUATION ) {
|
||
if ( token[0] == '{' ) {
|
||
depth++;
|
||
}
|
||
else if ( token[0] == '}' ) {
|
||
depth--;
|
||
}
|
||
}
|
||
|
||
if ( token.type == TT_STRING ) {
|
||
out += "\"" + token + "\"";
|
||
}
|
||
else {
|
||
out += token;
|
||
}
|
||
out += " ";
|
||
} while( depth );
|
||
|
||
return out.c_str();
|
||
}
|
||
|
||
/*
|
||
=================
|
||
idLexer::ParseRestOfLine
|
||
|
||
parse the rest of the line
|
||
=================
|
||
*/
|
||
const char *idLexer::ParseRestOfLine( idStr &out ) {
|
||
idToken token;
|
||
|
||
out.Empty();
|
||
while(idLexer::ReadToken( &token )) {
|
||
if ( token.linesCrossed ) {
|
||
idLexer::script_p = lastScript_p;
|
||
idLexer::line = lastline;
|
||
break;
|
||
}
|
||
if ( out.Length() ) {
|
||
out += " ";
|
||
}
|
||
out += token;
|
||
}
|
||
return out.c_str();
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::GetLastWhiteSpace
|
||
================
|
||
*/
|
||
int idLexer::GetLastWhiteSpace( idStr &whiteSpace ) const {
|
||
whiteSpace.Clear();
|
||
for ( const char *p = whiteSpaceStart_p; p < whiteSpaceEnd_p; p++ ) {
|
||
whiteSpace.Append( *p );
|
||
}
|
||
return whiteSpace.Length();
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::GetLastWhiteSpaceStart
|
||
================
|
||
*/
|
||
int idLexer::GetLastWhiteSpaceStart( void ) const {
|
||
return whiteSpaceStart_p - buffer;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::GetLastWhiteSpaceEnd
|
||
================
|
||
*/
|
||
int idLexer::GetLastWhiteSpaceEnd( void ) const {
|
||
return whiteSpaceEnd_p - buffer;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::Reset
|
||
================
|
||
*/
|
||
void idLexer::Reset( void ) {
|
||
// pointer in script buffer
|
||
idLexer::script_p = idLexer::buffer;
|
||
// pointer in script buffer before reading token
|
||
idLexer::lastScript_p = idLexer::buffer;
|
||
// begin of white space
|
||
idLexer::whiteSpaceStart_p = NULL;
|
||
// end of white space
|
||
idLexer::whiteSpaceEnd_p = NULL;
|
||
// set if there's a token available in idLexer::token
|
||
idLexer::tokenavailable = 0;
|
||
|
||
idLexer::line = 1;
|
||
idLexer::lastline = 1;
|
||
// clear the saved token
|
||
idLexer::token = "";
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::EndOfFile
|
||
================
|
||
*/
|
||
int idLexer::EndOfFile( void ) {
|
||
return idLexer::script_p >= idLexer::end_p;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::NumLinesCrossed
|
||
================
|
||
*/
|
||
int idLexer::NumLinesCrossed( void ) {
|
||
return idLexer::line - idLexer::lastline;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::LoadFile
|
||
================
|
||
*/
|
||
int idLexer::LoadFile( const char *filename, bool OSPath ) {
|
||
idFile *fp;
|
||
idStr pathname;
|
||
int length;
|
||
char *buf;
|
||
|
||
if ( idLexer::loaded ) {
|
||
idLib::common->Error("idLexer::LoadFile: another script already loaded");
|
||
return false;
|
||
}
|
||
|
||
if ( !OSPath && ( baseFolder[0] != '\0' ) ) {
|
||
pathname = va( "%s/%s", baseFolder, filename );
|
||
} else {
|
||
pathname = filename;
|
||
}
|
||
if ( OSPath ) {
|
||
fp = idLib::fileSystem->OpenExplicitFileRead( pathname );
|
||
} else {
|
||
fp = idLib::fileSystem->OpenFileRead( pathname );
|
||
}
|
||
if ( !fp ) {
|
||
return false;
|
||
}
|
||
length = fp->Length();
|
||
// RAVEN BEGIN
|
||
// amccarthy: Added memory allocation tag
|
||
buf = (char *) Mem_Alloc( length + 1, MA_LEXER );
|
||
if( !buf ) {
|
||
common->FatalError( "Memory system failure : out of memory" );
|
||
}
|
||
// RAVEN END
|
||
buf[length] = '\0';
|
||
fp->Read( buf, length );
|
||
idLexer::fileTime = fp->Timestamp();
|
||
idLexer::filename = fp->GetFullPath();
|
||
idLib::fileSystem->CloseFile( fp );
|
||
|
||
idLexer::buffer = buf;
|
||
idLexer::length = length;
|
||
// pointer in script buffer
|
||
idLexer::script_p = idLexer::buffer;
|
||
// pointer in script buffer before reading token
|
||
idLexer::lastScript_p = idLexer::buffer;
|
||
// pointer to end of script buffer
|
||
idLexer::end_p = &(idLexer::buffer[length]);
|
||
|
||
idLexer::tokenavailable = 0;
|
||
idLexer::line = 1;
|
||
idLexer::lastline = 1;
|
||
idLexer::allocated = true;
|
||
idLexer::loaded = true;
|
||
|
||
// RAVEN BEGIN
|
||
// jsinger: initialize compiled file
|
||
if(flags & LEXFL_WRITEBINARY)
|
||
{
|
||
pathname.Append(Lexer::sCompiledFileSuffix);
|
||
if ( OSPath ) {
|
||
mBinaryFile = idLib::fileSystem->OpenExplicitFileWrite( pathname );
|
||
} else {
|
||
mBinaryFile = idLib::fileSystem->OpenFileWrite( pathname );
|
||
}
|
||
}
|
||
// RAVEN END
|
||
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::LoadMemory
|
||
================
|
||
*/
|
||
int idLexer::LoadMemory( const char *ptr, int length, const char *name, int startLine ) {
|
||
if ( idLexer::loaded ) {
|
||
idLib::common->Error("idLexer::LoadMemory: another script already loaded");
|
||
return false;
|
||
}
|
||
idLexer::filename = name;
|
||
idLexer::buffer = ptr;
|
||
idLexer::fileTime = 0;
|
||
idLexer::length = length;
|
||
// pointer in script buffer
|
||
idLexer::script_p = idLexer::buffer;
|
||
// pointer in script buffer before reading token
|
||
idLexer::lastScript_p = idLexer::buffer;
|
||
// pointer to end of script buffer
|
||
idLexer::end_p = &(idLexer::buffer[length]);
|
||
|
||
idLexer::tokenavailable = 0;
|
||
idLexer::line = startLine;
|
||
idLexer::lastline = startLine;
|
||
idLexer::allocated = false;
|
||
idLexer::loaded = true;
|
||
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::FreeSource
|
||
================
|
||
*/
|
||
void idLexer::FreeSource( void ) {
|
||
#ifdef PUNCTABLE
|
||
if ( idLexer::punctuationtable && idLexer::punctuationtable != default_punctuationtable ) {
|
||
Mem_Free( (void *) idLexer::punctuationtable );
|
||
idLexer::punctuationtable = NULL;
|
||
}
|
||
if ( idLexer::nextpunctuation && idLexer::nextpunctuation != default_nextpunctuation ) {
|
||
Mem_Free( (void *) idLexer::nextpunctuation );
|
||
idLexer::nextpunctuation = NULL;
|
||
}
|
||
#endif //PUNCTABLE
|
||
if ( idLexer::allocated ) {
|
||
Mem_Free( (void *) idLexer::buffer );
|
||
idLexer::buffer = NULL;
|
||
idLexer::allocated = false;
|
||
}
|
||
idLexer::tokenavailable = 0;
|
||
idLexer::token = "";
|
||
idLexer::loaded = false;
|
||
// RAVEN BEGIN
|
||
// jsinger: close compile file if it exists
|
||
if(flags & LEXFL_WRITEBINARY)
|
||
{
|
||
if(mBinaryFile)
|
||
{
|
||
idLib::fileSystem->CloseFile(mBinaryFile);
|
||
mBinaryFile = NULL;
|
||
}
|
||
}
|
||
// RAVEN END
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::idLexer
|
||
================
|
||
*/
|
||
idLexer::idLexer( void ) {
|
||
idLexer::loaded = false;
|
||
idLexer::filename = "";
|
||
idLexer::flags = 0;
|
||
idLexer::SetPunctuations( NULL );
|
||
idLexer::allocated = false;
|
||
idLexer::fileTime = 0;
|
||
idLexer::length = 0;
|
||
idLexer::line = 0;
|
||
idLexer::lastline = 0;
|
||
idLexer::tokenavailable = 0;
|
||
idLexer::token = "";
|
||
idLexer::next = NULL;
|
||
idLexer::hadError = false;
|
||
// RAVEN BEGIN
|
||
// jsinger: initialize compiled file
|
||
idLexer::mBinaryFile = NULL;
|
||
// RAVEN END
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::idLexer
|
||
================
|
||
*/
|
||
idLexer::idLexer( int flags ) {
|
||
idLexer::loaded = false;
|
||
idLexer::filename = "";
|
||
idLexer::flags = flags;
|
||
idLexer::SetPunctuations( NULL );
|
||
idLexer::allocated = false;
|
||
idLexer::fileTime = 0;
|
||
idLexer::length = 0;
|
||
idLexer::line = 0;
|
||
idLexer::lastline = 0;
|
||
idLexer::tokenavailable = 0;
|
||
idLexer::token = "";
|
||
idLexer::next = NULL;
|
||
idLexer::hadError = false;
|
||
// RAVEN BEGIN
|
||
// jsinger: initialize compiled file
|
||
idLexer::mBinaryFile = NULL;
|
||
// RAVEN END
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::idLexer
|
||
================
|
||
*/
|
||
idLexer::idLexer( const char *filename, int flags, bool OSPath ) {
|
||
idLexer::loaded = false;
|
||
idLexer::flags = flags;
|
||
idLexer::SetPunctuations( NULL );
|
||
idLexer::allocated = false;
|
||
idLexer::token = "";
|
||
idLexer::next = NULL;
|
||
idLexer::hadError = false;
|
||
// RAVEN BEGIN
|
||
// jsinger: initialize compiled file
|
||
idLexer::mBinaryFile = NULL;
|
||
// RAVEN END
|
||
idLexer::LoadFile( filename, OSPath );
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::idLexer
|
||
================
|
||
*/
|
||
idLexer::idLexer( const char *ptr, int length, const char *name, int flags ) {
|
||
idLexer::loaded = false;
|
||
idLexer::flags = flags;
|
||
idLexer::SetPunctuations( NULL );
|
||
idLexer::allocated = false;
|
||
idLexer::token = "";
|
||
idLexer::next = NULL;
|
||
idLexer::hadError = false;
|
||
// RAVEN BEGIN
|
||
// jsinger: initialize compiled file
|
||
idLexer::mBinaryFile = NULL;
|
||
// RAVEN END
|
||
idLexer::LoadMemory( ptr, length, name );
|
||
}
|
||
|
||
/*
|
||
================
|
||
idLexer::~idLexer
|
||
================
|
||
*/
|
||
idLexer::~idLexer( void ) {
|
||
idLexer::FreeSource();
|
||
}
|
||
|
||
// RAVEN BEGIN
|
||
// jsinger: SetBaseFolder was moved to the Lexer base class to unify its functionality across all
|
||
// derived classes
|
||
/*
|
||
================
|
||
idLexer::SetBaseFolder
|
||
================
|
||
|
||
void idLexer::SetBaseFolder( const char *path ) {
|
||
idStr::Copynz( baseFolder, path, sizeof( baseFolder ) );
|
||
}
|
||
*/
|
||
// RAVEN END
|
||
/*
|
||
================
|
||
idLexer::HadError
|
||
================
|
||
*/
|
||
bool idLexer::HadError( void ) const {
|
||
return hadError;
|
||
}
|
||
|
||
|
||
// RAVEN BEGIN
|
||
// jsinger: This method can write out a binary representation of a token in a format
|
||
// suitable to be read by the Lexer
|
||
/*
|
||
================
|
||
idLexer::WriteBinaryToken
|
||
Writes a binary representation of the token value to the compiled file
|
||
when compiling is turned on
|
||
================
|
||
*/
|
||
void idLexer::WriteBinaryToken(idToken *tok)
|
||
{
|
||
bool swapBytes = (flags & LEXFL_BYTESWAP) == LEXFL_BYTESWAP;
|
||
|
||
if(flags & LEXFL_WRITEBINARY)
|
||
{
|
||
unsigned char prefix;
|
||
|
||
switch(tok->type)
|
||
{
|
||
case TT_NUMBER:
|
||
if(tok->subtype & TT_INTEGER)
|
||
{
|
||
if(tok->subtype & TT_UNSIGNED)
|
||
{
|
||
unsigned long val = tok->GetUnsignedLongValue();
|
||
unsigned char byteVal = (unsigned char)val;
|
||
unsigned short shortVal = (unsigned short)val;
|
||
|
||
if(byteVal == val)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_UNSIGNEDINT, BTT_STORED_1BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<unsigned char>(byteVal, mBinaryFile, swapBytes);
|
||
}
|
||
else if(shortVal == val)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_UNSIGNEDINT, BTT_STORED_2BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<unsigned short>(shortVal, mBinaryFile, swapBytes);
|
||
}
|
||
else
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_UNSIGNEDINT, BTT_STORED_4BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<unsigned int>(val, mBinaryFile, swapBytes);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
long val = tok->GetIntValue();
|
||
char byteVal = (char)val;
|
||
short shortVal = (short)val;
|
||
|
||
if(byteVal == val)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_INT, BTT_STORED_1BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<char>(byteVal, mBinaryFile, swapBytes);
|
||
}
|
||
else if(shortVal == val)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_INT, BTT_STORED_2BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<short>(shortVal, mBinaryFile, swapBytes);
|
||
}
|
||
else
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_INT, BTT_STORED_4BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<int>(val, mBinaryFile, swapBytes);
|
||
}
|
||
}
|
||
}
|
||
else if(tok->subtype & TT_FLOAT)
|
||
{
|
||
if(tok->subtype & TT_SINGLE_PRECISION)
|
||
{
|
||
float val = tok->GetFloatValue();
|
||
int intval = tok->GetIntValue();
|
||
if(((float)intval) == val) // integral check
|
||
{
|
||
char byteVal = (char)intval;
|
||
short shortVal = (short)intval;
|
||
|
||
if(byteVal == intval)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_1BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<char>(byteVal, mBinaryFile, swapBytes);
|
||
}
|
||
else if(shortVal == intval)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_2BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<short>(shortVal, mBinaryFile, swapBytes);
|
||
}
|
||
else
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_4BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<float>(val, mBinaryFile, swapBytes);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_4BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<float>(val, mBinaryFile, swapBytes);
|
||
}
|
||
}
|
||
else if(tok->subtype & TT_DOUBLE_PRECISION)
|
||
{
|
||
// we can write out doubles as floats because the text file doesn't have double precision anyway
|
||
float val = tok->GetDoubleValue();
|
||
int intval = tok->GetIntValue();
|
||
if(((float)intval) == val) // integral check
|
||
{
|
||
char byteVal = (char)intval;
|
||
short shortVal = (short)intval;
|
||
|
||
if(byteVal == intval)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_1BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<char>(byteVal, mBinaryFile, swapBytes);
|
||
}
|
||
else if(shortVal == intval)
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_FLOAT, BTT_STORED_2BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<short>(shortVal, mBinaryFile, swapBytes);
|
||
}
|
||
else
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_DOUBLE, BTT_STORED_4BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<float>(val, mBinaryFile, swapBytes);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
prefix = BTT_MAKENUMBER_PREFIX(BTT_NUMBER, BTT_SUBTYPE_DOUBLE, BTT_STORED_4BYTE);
|
||
TextCompiler::WriteValue<unsigned char>(prefix, mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<float>(val, mBinaryFile, swapBytes);
|
||
}
|
||
}
|
||
}
|
||
break;
|
||
case TT_STRING:
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKESTRING_PREFIX(BTT_STRING, tok->Length()), mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<idStr>(tok, mBinaryFile, swapBytes);
|
||
break;
|
||
case TT_LITERAL:
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKESTRING_PREFIX(BTT_LITERAL, tok->Length()), mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<idStr>(tok, mBinaryFile, swapBytes);
|
||
break;
|
||
case TT_NAME:
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKESTRING_PREFIX(BTT_NAME, tok->Length()), mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<idStr>(tok, mBinaryFile, swapBytes);
|
||
break;
|
||
case TT_PUNCTUATION:
|
||
if(*tok == "&&")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LOGICALAND), mBinaryFile, swapBytes);
|
||
}
|
||
else if (*tok == "&")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_AMPERSAND), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_EQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "==")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_EQUALEQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "!=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_NOTEQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "!")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_EXCLAMATION), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "<")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LESSTHAN), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "<=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LESSOREQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "<<")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_SHIFTLEFT), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ">")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_GREATERTHAN), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ">=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_GREATEROREQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ">>")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_SHIFTRIGHT), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "%")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PERCENT), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "[")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LEFTBRACKET), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "]")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_RIGHTBRACKET), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "-")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_MINUS), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "--")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_MINUSMINUS), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "-=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_MINUSEQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "+")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PLUS), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "+=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PLUSEQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "++")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PLUSPLUS), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "(")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LEFTPAREN), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ")")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_RIGHTPAREN), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "{")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_LEFTBRACE), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "}")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_RIGHTBRACE), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ",")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_COMMA), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "::")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOUBLECOLON), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "#")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_HASH), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "##")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOUBLEHASH), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "/")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_FORWARDSLASH), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "\\")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_BACKSLASH), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ";")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_SEMICOLON), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == ".")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PERIOD), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "$")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOLLARSIGN), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "~")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_TILDE), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "|")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_PIPE), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "||")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_DOUBLEPIPE), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "*")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_ASTERISK), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "*=")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_TIMESEQUAL), mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "<EFBFBD>")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_PUNC_INVERTEDPLING, mBinaryFile, swapBytes);
|
||
}
|
||
else if(*tok == "<EFBFBD>")
|
||
{
|
||
TextCompiler::WriteValue<unsigned char>(BTT_PUNC_INVERTEDQUERY, mBinaryFile, swapBytes);
|
||
}
|
||
else
|
||
{
|
||
common->Warning("Unsupported punctuation: '%s'", tok->c_str());
|
||
|
||
TextCompiler::WriteValue<unsigned char>(BTT_MAKEPUNCTUATION_PREFIX(BTT_PUNC_ASNULLTERMINATED), mBinaryFile, swapBytes);
|
||
TextCompiler::WriteValue<idStr>(tok, mBinaryFile, swapBytes);
|
||
}
|
||
break;
|
||
default:
|
||
// wtf?
|
||
assert(false);
|
||
}
|
||
}
|
||
}
|
||
|
||
// jsinger: this method takes an existing file and causes a binary tokenized
|
||
// version of the file to be generated
|
||
void idLexer::WriteBinaryFile(char const * const filename)
|
||
{
|
||
if(!cvarSystem->GetCVarBool("com_BinaryRead"))
|
||
{
|
||
unsigned int swap=0;
|
||
|
||
switch(cvarSystem->GetCVarInteger("com_BinaryWrite"))
|
||
{
|
||
case 1:
|
||
swap = 0;
|
||
break;
|
||
case 2:
|
||
swap = LEXFL_BYTESWAP;
|
||
break;
|
||
}
|
||
|
||
idLexer src(filename, LEXFL_WRITEBINARY | swap);
|
||
idToken token;
|
||
if(src.IsLoaded())
|
||
{
|
||
while(src.ReadToken(&token))
|
||
{
|
||
// we don't need to do anything, because just reading the file
|
||
// causes it to write a tokenized version
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// jsinger: implementation for a class that can load binary tokenized files
|
||
using namespace TextCompiler;
|
||
Lexer::Lexer(int flags)
|
||
{
|
||
#ifndef LEXER_READ_AHEAD
|
||
mFile = NULL;
|
||
#endif
|
||
offset = 0;
|
||
tokenAvailable = false;
|
||
mFlags = flags;
|
||
if(flags & LEXFL_READBINARY)
|
||
{
|
||
mDelegate = NULL;
|
||
}
|
||
else
|
||
{
|
||
mDelegate = new idLexer(flags);
|
||
}
|
||
}
|
||
|
||
Lexer::Lexer(char const * const ptr, int length, char const * const name, int flags)
|
||
{
|
||
if(flags & LEXFL_READBINARY)
|
||
{
|
||
#ifdef LEXER_READ_AHEAD
|
||
mLexerIOWrapper.InitFromMemory(ptr, length);
|
||
#else
|
||
assert(false);
|
||
#endif
|
||
mDelegate = NULL;
|
||
}
|
||
else
|
||
{
|
||
mDelegate = new idLexer(ptr, length, name, flags);
|
||
}
|
||
}
|
||
|
||
Lexer::Lexer(char const * const filename, int flags, bool OSPath)
|
||
{
|
||
#ifndef LEXER_READ_AHEAD
|
||
mFile = NULL;
|
||
#endif
|
||
offset = 0;
|
||
tokenAvailable = false;
|
||
mFlags = flags;
|
||
if(flags & LEXFL_READBINARY)
|
||
{
|
||
mDelegate = NULL;
|
||
LoadFile(filename, OSPath);
|
||
}
|
||
else
|
||
{
|
||
mDelegate = new idLexer(filename, flags, OSPath);
|
||
}
|
||
}
|
||
|
||
Lexer::~Lexer()
|
||
{
|
||
FreeSource();
|
||
}
|
||
|
||
char const * Lexer::GetFileName()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetFileName();
|
||
}
|
||
else
|
||
{
|
||
assert(false);
|
||
return "";
|
||
}
|
||
}
|
||
|
||
bool Lexer::HadError() const
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->HadError();
|
||
}
|
||
else
|
||
{
|
||
assert(false);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
void Lexer::Warning(char const *str, ...)
|
||
{
|
||
char text[MAX_STRING_CHARS];
|
||
va_list ap;
|
||
|
||
va_start( ap, str );
|
||
vsprintf( text, str, ap );
|
||
va_end( ap );
|
||
idLib::common->Warning( "Lexer: '%s'", text );
|
||
}
|
||
|
||
void Lexer::Error(char const *str, ...)
|
||
{
|
||
char text[MAX_STRING_CHARS];
|
||
va_list ap;
|
||
|
||
va_start( ap, str );
|
||
vsprintf( text, str, ap );
|
||
va_end( ap );
|
||
|
||
assert(false);
|
||
|
||
idLib::common->Error( "Lexer: offset: %d '%s'", offset, text );
|
||
}
|
||
|
||
int const Lexer::GetLineNum()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetLineNum();
|
||
}
|
||
else
|
||
{
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
unsigned int const Lexer::GetFileTime()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetFileTime();
|
||
}
|
||
else
|
||
{
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int const Lexer::GetFileOffset()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetFileOffset();
|
||
}
|
||
{
|
||
assert(false);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int Lexer::EndOfFile()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->EndOfFile();
|
||
}
|
||
else
|
||
{
|
||
#ifdef LEXER_READ_AHEAD
|
||
return mLexerIOWrapper.Tell()>= mLexerIOWrapper.Length();
|
||
#else
|
||
return mFile->Tell()>= mFile->Length();
|
||
#endif
|
||
}
|
||
}
|
||
|
||
void Lexer::Reset()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
mDelegate->Reset();
|
||
}
|
||
else
|
||
{
|
||
#ifdef LEXER_READ_AHEAD
|
||
mLexerIOWrapper.Seek(0, FS_SEEK_SET);
|
||
#else
|
||
mFile->Seek(0, FS_SEEK_SET);
|
||
#endif
|
||
offset = 0;
|
||
tokenAvailable = false;
|
||
}
|
||
}
|
||
|
||
int Lexer::GetFlags()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetFlags();
|
||
}
|
||
else
|
||
{
|
||
return mFlags;
|
||
}
|
||
}
|
||
|
||
void Lexer::SetFlags(int flags)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
mDelegate->SetFlags(flags);
|
||
}
|
||
else
|
||
{
|
||
mFlags = flags;
|
||
}
|
||
}
|
||
|
||
int Lexer::GetPunctuationId(char const *str)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetPunctuationId(str);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
void Lexer::SetPunctuations(struct punctuation_s const *punc)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
mDelegate->SetPunctuations(punc);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
}
|
||
}
|
||
|
||
int Lexer::GetLastWhiteSpaceEnd() const
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetLastWhiteSpaceEnd();
|
||
}
|
||
else
|
||
{
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
char const *Lexer::GetPunctuationFromId(int val)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetPunctuationFromId(val);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return "";
|
||
}
|
||
}
|
||
|
||
int Lexer::GetLastWhiteSpaceStart() const
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetLastWhiteSpaceStart();
|
||
}
|
||
else
|
||
{
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int Lexer::GetLastWhiteSpace(idStr &str) const
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->GetLastWhiteSpace(str);
|
||
}
|
||
else
|
||
{
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
char const *Lexer::ParseRestOfLine(idStr &str)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ParseRestOfLine(str);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return "";
|
||
}
|
||
}
|
||
|
||
char const *Lexer::ParseBracedSectionExact(idStr &str, int val)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ParseBracedSectionExact(str, val);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return "";
|
||
}
|
||
}
|
||
|
||
char const *Lexer::ParseBracedSection(idStr &str)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ParseBracedSection(str);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return "";
|
||
}
|
||
}
|
||
|
||
int Lexer::Parse3DMatrix(int z, int y, int x, float *m)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->Parse3DMatrix(z, y, x, m);
|
||
}
|
||
else
|
||
{
|
||
int i;
|
||
|
||
if ( !ExpectTokenString( "(" ) ) {
|
||
return false;
|
||
}
|
||
|
||
for ( i = 0 ; i < z; i++ ) {
|
||
if ( !Parse2DMatrix( y, x, m + i * x*y ) ) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if ( !ExpectTokenString( ")" ) ) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
}
|
||
|
||
int Lexer::Parse2DMatrix(int y, int x, float *m)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->Parse2DMatrix(y, x, m);
|
||
}
|
||
else
|
||
{
|
||
int i;
|
||
|
||
if ( !ExpectTokenString( "(" ) ) {
|
||
return false;
|
||
}
|
||
|
||
for ( i = 0; i < y; i++ ) {
|
||
if ( !Parse1DMatrix( x, m + i * x ) ) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if ( !ExpectTokenString( ")" ) ) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
}
|
||
|
||
int Lexer::Parse1DMatrixOpenEnded(int MaxCount, float *m)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->Parse1DMatrixOpenEnded(MaxCount, m);
|
||
}
|
||
else
|
||
{
|
||
int i;
|
||
|
||
if ( !ExpectTokenString( "(" ) ) {
|
||
return 0;
|
||
}
|
||
|
||
for ( i = 0; i < MaxCount; i++ ) {
|
||
idToken tok;
|
||
|
||
if (!ReadToken( &tok )) {
|
||
return 0;
|
||
}
|
||
|
||
if ( tok == ")" ) {
|
||
return i;
|
||
}
|
||
|
||
UnreadToken( &tok );
|
||
|
||
m[i] = ParseFloat();
|
||
}
|
||
|
||
if ( !ExpectTokenString( ")" ) ) {
|
||
return 0;
|
||
}
|
||
|
||
return i;
|
||
}
|
||
}
|
||
|
||
int Lexer::Parse1DMatrix(int x, float *m)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->Parse1DMatrix(x, m);
|
||
}
|
||
else
|
||
{
|
||
int i;
|
||
|
||
if ( !ExpectTokenString( "(" ) ) {
|
||
return false;
|
||
}
|
||
|
||
for ( i = 0; i < x; i++ ) {
|
||
m[i] = ParseFloat();
|
||
}
|
||
|
||
if ( !ExpectTokenString( ")" ) ) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
}
|
||
|
||
float Lexer::ParseFloat(bool *errorFlag)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ParseFloat(errorFlag);
|
||
}
|
||
else
|
||
{
|
||
idToken token;
|
||
|
||
if(EndOfFile())
|
||
{
|
||
return 0;
|
||
}
|
||
|
||
if ( !ReadToken( &token ) ) {
|
||
Warning( "couldn't read expected floating point number" );
|
||
return 0;
|
||
}
|
||
if ( token.type == TT_PUNCTUATION && token == "-" ) {
|
||
ExpectTokenType( TT_NUMBER, 0, &token );
|
||
return -token.GetFloatValue();
|
||
}
|
||
else if ( token.type != TT_NUMBER ) {
|
||
Warning( "expected float value, found '%s'", token.c_str() );
|
||
}
|
||
return token.GetFloatValue();
|
||
}
|
||
}
|
||
|
||
bool Lexer::ParseBool()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ParseBool();
|
||
}
|
||
else
|
||
{
|
||
idToken token;
|
||
|
||
if ( !ExpectTokenType( TT_NUMBER, 0, &token ) ) {
|
||
Error( "couldn't read expected boolean" );
|
||
return false;
|
||
}
|
||
return ( token.GetIntValue() != 0 );
|
||
}
|
||
}
|
||
|
||
int Lexer::ParseInt()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ParseInt();
|
||
}
|
||
else
|
||
{
|
||
idToken token;
|
||
|
||
if ( !ReadToken( &token ) ) {
|
||
Error( "couldn't read expected integer" );
|
||
return 0;
|
||
}
|
||
if ( token.type == TT_PUNCTUATION && token == "-" ) {
|
||
ExpectTokenType( TT_NUMBER, TT_INTEGER, &token );
|
||
return -((signed int) token.GetIntValue());
|
||
}
|
||
else if ( token.type != TT_NUMBER || token.subtype == TT_FLOAT ) {
|
||
Error( "expected integer value, found '%s'", token.c_str() );
|
||
}
|
||
return token.GetIntValue();
|
||
}
|
||
}
|
||
|
||
int Lexer::ReadTokenOnLine(idToken *token)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ReadTokenOnLine(token);
|
||
}
|
||
else
|
||
{
|
||
// lines are meaningless in the binary format
|
||
return ReadToken(token);
|
||
}
|
||
}
|
||
|
||
void Lexer::UnreadToken(idToken const *token)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
mDelegate->UnreadToken(token);
|
||
}
|
||
else
|
||
{
|
||
if ( tokenAvailable )
|
||
{
|
||
idLib::common->FatalError( "Lexer::unreadToken, unread token twice\n" );
|
||
}
|
||
unreadToken = *token;
|
||
tokenAvailable = true;
|
||
offset -= unreadSize+1; // the +1 is for the prefix
|
||
}
|
||
}
|
||
|
||
int Lexer::SkipBracedSection(bool parseFirstBrace)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->SkipBracedSection(parseFirstBrace);
|
||
}
|
||
else
|
||
{
|
||
idToken token;
|
||
int depth;
|
||
|
||
depth = parseFirstBrace ? 0 : 1;
|
||
do {
|
||
if ( !ReadToken( &token ) ) {
|
||
return false;
|
||
}
|
||
if ( token.type == TT_PUNCTUATION ) {
|
||
if ( token == "{" ) {
|
||
depth++;
|
||
} else if ( token == "}" ) {
|
||
depth--;
|
||
}
|
||
}
|
||
} while( depth );
|
||
return true;
|
||
}
|
||
}
|
||
|
||
int Lexer::SkipRestOfLine()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->SkipRestOfLine();
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int Lexer::SkipUntilString(char const *str)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->SkipUntilString(str);
|
||
}
|
||
else
|
||
{
|
||
// this method is not supported
|
||
assert(false);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int Lexer::CheckTokenType(int type, int subtype, idToken *token)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->CheckTokenType(type, subtype, token);
|
||
}
|
||
else
|
||
{
|
||
idToken tok;
|
||
|
||
if (!ReadToken( &tok )) {
|
||
return 0;
|
||
}
|
||
// if the type matches
|
||
if (tok.type == type && (tok.subtype & subtype) == subtype) {
|
||
*token = tok;
|
||
return 1;
|
||
}
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int Lexer::CheckTokenString(char const *string)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->CheckTokenString(string);
|
||
}
|
||
else
|
||
{
|
||
idToken tok;
|
||
|
||
if (!ReadToken( &tok )) {
|
||
return 0;
|
||
}
|
||
|
||
// if the token is available
|
||
if ( tok == string ) {
|
||
return 1;
|
||
}
|
||
|
||
UnreadToken( &tok );
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
int Lexer::ExpectAnyToken(idToken *token)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ExpectAnyToken(token);
|
||
}
|
||
else
|
||
{
|
||
if (!ReadToken( token )) {
|
||
Error( "couldn't read expected token" );
|
||
return 0;
|
||
}
|
||
else {
|
||
return 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
int Lexer::ExpectTokenType(int type, int subtype, idToken *token)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ExpectTokenType(type, subtype, token);
|
||
}
|
||
else
|
||
{
|
||
idStr str;
|
||
|
||
if ( !ReadToken( token ) ) {
|
||
Error( "couldn't read expected token" );
|
||
return 0;
|
||
}
|
||
|
||
if ( token->type != type ) {
|
||
switch( type ) {
|
||
case TT_STRING: str = "string"; break;
|
||
case TT_LITERAL: str = "literal"; break;
|
||
case TT_NUMBER: str = "number"; break;
|
||
case TT_NAME: str = "name"; break;
|
||
case TT_PUNCTUATION: str = "punctuation"; break;
|
||
default: str = "unknown type"; break;
|
||
}
|
||
Error( "expected a %s but found '%s'", str.c_str(), token->c_str() );
|
||
return 0;
|
||
}
|
||
if ( token->type == TT_NUMBER ) {
|
||
if ( (token->subtype & subtype) != subtype ) {
|
||
str.Clear();
|
||
if ( subtype & TT_DECIMAL ) str = "decimal ";
|
||
if ( subtype & TT_HEX ) str = "hex ";
|
||
if ( subtype & TT_OCTAL ) str = "octal ";
|
||
if ( subtype & TT_BINARY ) str = "binary ";
|
||
if ( subtype & TT_UNSIGNED ) str += "unsigned ";
|
||
if ( subtype & TT_LONG ) str += "long ";
|
||
if ( subtype & TT_FLOAT ) str += "float ";
|
||
if ( subtype & TT_INTEGER ) str += "integer ";
|
||
str.StripTrailing( ' ' );
|
||
Error( "expected %s but found '%s'", str.c_str(), token->c_str() );
|
||
return 0;
|
||
}
|
||
}
|
||
else if ( token->type == TT_PUNCTUATION ) {
|
||
if ( subtype < 0 ) {
|
||
Error( "BUG: wrong punctuation subtype" );
|
||
return 0;
|
||
}
|
||
if ( token->subtype != subtype ) {
|
||
Error( "expected '%s' but found '%s'", GetPunctuationFromId( subtype ), token->c_str() );
|
||
return 0;
|
||
}
|
||
}
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
int Lexer::ExpectTokenString(char const *string)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ExpectTokenString(string);
|
||
}
|
||
else
|
||
{
|
||
idToken token;
|
||
|
||
if(!ReadToken( &token ))
|
||
{
|
||
Error( "couldn't find expected '%s'", string );
|
||
return 0;
|
||
}
|
||
if( token != string )
|
||
{
|
||
Error( "expected '%s' but found '%s'", string, token.c_str() );
|
||
return 0;
|
||
}
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
int Lexer::ReadToken(idToken *token)
|
||
{
|
||
|
||
#ifdef LEXER_READ_AHEAD
|
||
#define OBJ &mLexerIOWrapper
|
||
#else
|
||
#define OBJ mFile
|
||
#endif
|
||
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->ReadToken(token);
|
||
}
|
||
else
|
||
{
|
||
// if there are any unread tokens, use them first
|
||
if(tokenAvailable)
|
||
{
|
||
*token = unreadToken;
|
||
tokenAvailable = false;
|
||
}
|
||
else
|
||
{
|
||
#ifndef LEXER_READ_AHEAD
|
||
assert(mFile);
|
||
#endif
|
||
if(EndOfFile())
|
||
return 0;
|
||
unsigned char prefix = ReadValue<unsigned char>(OBJ);
|
||
token->Clear();
|
||
|
||
// BTT types are equivalent to the TT types except they are one less, so we add one to get the token type
|
||
if(BTT_GET_TYPE(prefix) == BTT_PUNCTUATION2)
|
||
token->type = TT_PUNCTUATION;
|
||
else
|
||
token->type = BTT_GET_TYPE(prefix)+1;
|
||
|
||
// this is used only to determine if the punctuation was encoded in the prefix or came afterwards as a null terminated string
|
||
bool encoded = true;
|
||
|
||
switch(token->type)
|
||
{
|
||
case TT_STRING:
|
||
case TT_NAME:
|
||
case TT_LITERAL:
|
||
if(BTT_GET_STRING_LENGTH(prefix) != 0)
|
||
{
|
||
// short string
|
||
int length = BTT_GET_STRING_LENGTH(prefix);
|
||
if ( length == 0 ) {
|
||
assert( false );
|
||
}
|
||
int i;
|
||
for( i = 0; i < length; i++ )
|
||
{
|
||
char c = ReadValue<char>(OBJ);
|
||
token->Append(c);
|
||
}
|
||
// short strings have no null, so we must add one
|
||
//token->Append('\0');
|
||
|
||
token->subtype = token->Length();
|
||
unreadSize = token->Length();
|
||
}
|
||
else
|
||
{
|
||
// null terminated string
|
||
*token = ReadValue<idStr>(OBJ);
|
||
token->subtype = token->Length();
|
||
unreadSize = token->Length()+1;
|
||
}
|
||
break;
|
||
case TT_PUNCTUATION:
|
||
switch(BTT_GET_PUNCTUATION(prefix))
|
||
{
|
||
case BTT_PUNC_ASNULLTERMINATED:
|
||
// null terminated string
|
||
*token = ReadValue<idStr>(OBJ);
|
||
token->subtype = token->Length();
|
||
unreadSize = token->Length()+1;
|
||
encoded = false;
|
||
break;
|
||
case BTT_PUNC_RIGHTPAREN:
|
||
*token = ")";
|
||
break;
|
||
case BTT_PUNC_LEFTBRACE:
|
||
*token = "{";
|
||
break;
|
||
case BTT_PUNC_RIGHTBRACE:
|
||
*token = "}";
|
||
break;
|
||
case BTT_PUNC_MINUS:
|
||
*token = "-";
|
||
break;
|
||
case BTT_PUNC_PLUS:
|
||
*token = "+";
|
||
break;
|
||
case BTT_PUNC_COMMA:
|
||
*token = ",";
|
||
break;
|
||
case BTT_PUNC_PLUSPLUS:
|
||
*token = "++";
|
||
break;
|
||
case BTT_PUNC_LEFTBRACKET:
|
||
*token = "[";
|
||
break;
|
||
case BTT_PUNC_RIGHTBRACKET:
|
||
*token = "]";
|
||
break;
|
||
case BTT_PUNC_EQUAL:
|
||
*token = "=";
|
||
break;
|
||
case BTT_PUNC_EQUALEQUAL:
|
||
*token = "==";
|
||
break;
|
||
case BTT_PUNC_NOTEQUAL:
|
||
*token = "!=";
|
||
break;
|
||
case BTT_PUNC_PERCENT:
|
||
*token = "%";
|
||
break;
|
||
case BTT_PUNC_LESSTHAN:
|
||
*token = "<";
|
||
break;
|
||
case BTT_PUNC_GREATERTHAN:
|
||
*token = ">";
|
||
break;
|
||
case BTT_PUNC_LOGICALAND:
|
||
*token = "&&";
|
||
break;
|
||
case BTT_PUNC_AMPERSAND:
|
||
*token = "&";
|
||
break;
|
||
case BTT_PUNC_MINUSMINUS:
|
||
*token = "--";
|
||
break;
|
||
case BTT_PUNC_HASH:
|
||
*token = "#";
|
||
break;
|
||
case BTT_PUNC_LESSOREQUAL:
|
||
*token = "<=";
|
||
break;
|
||
case BTT_PUNC_GREATEROREQUAL:
|
||
*token = ">=";
|
||
break;
|
||
case BTT_PUNC_FORWARDSLASH:
|
||
*token = "/";
|
||
break;
|
||
case BTT_PUNC_SHIFTLEFT:
|
||
*token = "<<";
|
||
break;
|
||
case BTT_PUNC_SHIFTRIGHT:
|
||
*token = ">>";
|
||
break;
|
||
case BTT_PUNC_LEFTPAREN:
|
||
*token = "(";
|
||
break;
|
||
case BTT_PUNC_SEMICOLON:
|
||
*token = ";";
|
||
break;
|
||
case BTT_PUNC_ASTERISK:
|
||
*token = "*";
|
||
break;
|
||
case BTT_PUNC_PERIOD:
|
||
*token = ".";
|
||
break;
|
||
case BTT_PUNC_DOLLARSIGN:
|
||
*token = "$";
|
||
break;
|
||
case BTT_PUNC_PLUSEQUAL:
|
||
*token = "+=";
|
||
break;
|
||
case BTT_PUNC_MINUSEQUAL:
|
||
*token = "-=";
|
||
break;
|
||
case BTT_PUNC_TILDE:
|
||
*token = "~";
|
||
break;
|
||
case BTT_PUNC_EXCLAMATION:
|
||
*token = "!";
|
||
break;
|
||
case BTT_PUNC_PIPE:
|
||
*token = "|";
|
||
break;
|
||
case BTT_PUNC_BACKSLASH:
|
||
*token = "\\";
|
||
break;
|
||
case BTT_PUNC_DOUBLEHASH:
|
||
*token = "##";
|
||
break;
|
||
case BTT_PUNC_TIMESEQUAL:
|
||
*token = "*=";
|
||
break;
|
||
case BTT_PUNC_DOUBLEPIPE:
|
||
*token = "||";
|
||
break;
|
||
case BTT_PUNC_INVERTEDPLING:
|
||
*token = "<EFBFBD>";
|
||
break;
|
||
case BTT_PUNC_INVERTEDQUERY:
|
||
*token = "<EFBFBD>";
|
||
break;
|
||
default:
|
||
assert(false); // unrecognized punctuation
|
||
}
|
||
if(encoded)
|
||
unreadSize = 0; // punctuation encoded in prefix
|
||
break;
|
||
case TT_NUMBER:
|
||
{
|
||
// number of bytes actually in the stream used to represent this value
|
||
unsigned int size = BTT_GET_STORED_SIZE(prefix);
|
||
const int buffersize = 100;
|
||
char buffer[buffersize]; // big enough buffer for the int to string conversion routines
|
||
|
||
switch(BTT_GET_SUBTYPE(prefix))
|
||
{
|
||
case BTT_SUBTYPE_INT:
|
||
switch(size)
|
||
{
|
||
case BTT_STORED_1BYTE:
|
||
token->intvalue = ReadValue<char>(OBJ);
|
||
unreadSize = sizeof(char);
|
||
break;
|
||
case BTT_STORED_2BYTE:
|
||
token->intvalue = ReadValue<short>(OBJ);
|
||
unreadSize = sizeof(short);
|
||
break;
|
||
case BTT_STORED_4BYTE:
|
||
token->intvalue = ReadValue<int>(OBJ);
|
||
unreadSize = sizeof(int);
|
||
break;
|
||
default:
|
||
// invalid stored size for an integer
|
||
assert(false);
|
||
}
|
||
|
||
token->floatvalue = token->intvalue;
|
||
// I hate the fact that I have to copy into the string, but there's no way
|
||
// of easily knowing whether it is used later or not
|
||
|
||
// This conversion to a string assumes that long and int are the same
|
||
assert(sizeof(long) == sizeof(int));
|
||
|
||
//ltoa(token->intvalue, buffer, 10);
|
||
idStr::snPrintf( buffer, buffersize, "%ld", token->intvalue );
|
||
assert(token->intvalue == atol(buffer));
|
||
*token = buffer;
|
||
token->subtype = TT_INTEGER | TT_DECIMAL | TT_VALUESVALID;
|
||
break;
|
||
|
||
case BTT_SUBTYPE_UNSIGNEDINT:
|
||
switch(size)
|
||
{
|
||
case BTT_STORED_1BYTE:
|
||
token->intvalue = ReadValue<unsigned char>(OBJ);
|
||
unreadSize = sizeof(unsigned char);
|
||
break;
|
||
case BTT_STORED_2BYTE:
|
||
token->intvalue = ReadValue<unsigned short>(OBJ);
|
||
unreadSize = sizeof(unsigned short);
|
||
break;
|
||
case BTT_STORED_4BYTE:
|
||
token->intvalue = ReadValue<unsigned int>(OBJ);
|
||
unreadSize = sizeof(unsigned int);
|
||
break;
|
||
default:
|
||
// invalid stored size for an unsigned integer
|
||
assert(false);
|
||
}
|
||
token->floatvalue = token->intvalue;
|
||
|
||
// I hate the fact that I have to copy into the string, but there's no way
|
||
// of easily knowing whether it is used later or not
|
||
|
||
// This conversion to a string assumes that long and int are the same
|
||
assert(sizeof(long) == sizeof(int));
|
||
|
||
//ultoa(token->intvalue, buffer, 10);
|
||
idStr::snPrintf( buffer, buffersize, "%lu", token->intvalue );
|
||
assert(token->intvalue == ((unsigned long)atol(buffer)));
|
||
*token = buffer;
|
||
token->subtype = TT_INTEGER | TT_UNSIGNED | TT_DECIMAL | TT_VALUESVALID;
|
||
break;
|
||
|
||
case BTT_SUBTYPE_FLOAT:
|
||
// invalid stored size for a float
|
||
assert(sizeof(float) == 4);
|
||
|
||
switch(size)
|
||
{
|
||
case BTT_STORED_4BYTE:
|
||
token->floatvalue = ReadValue<float>(OBJ);
|
||
token->intvalue = token->floatvalue;
|
||
unreadSize = sizeof(float);
|
||
break;
|
||
case BTT_STORED_2BYTE: // requested a float, but it was integral, so it was saved that way
|
||
token->floatvalue = ReadValue<short>(OBJ);
|
||
token->intvalue = token->floatvalue;
|
||
unreadSize = sizeof(short);
|
||
break;
|
||
case BTT_STORED_1BYTE: // requested a float, but it was integral, so it was saved that way
|
||
token->floatvalue = ReadValue<char>(OBJ);
|
||
token->intvalue = token->floatvalue;
|
||
unreadSize = sizeof(char);
|
||
break;
|
||
default:
|
||
assert(false);
|
||
}
|
||
|
||
// I hate the fact that I have to copy into the string, but there's no way
|
||
// of easily knowing whether it is used later or not
|
||
idStr::snPrintf( buffer, buffersize, "%f", token->floatvalue );
|
||
*token = buffer;
|
||
token->subtype = TT_FLOAT | TT_DECIMAL | TT_SINGLE_PRECISION | TT_VALUESVALID;
|
||
|
||
unreadSize = sizeof(float);
|
||
break;
|
||
|
||
case BTT_SUBTYPE_DOUBLE:
|
||
// invalid stored size for a double
|
||
assert(sizeof(double) == 8);
|
||
|
||
// doubles are stored as floats since the original text file never uses full double precision anyway
|
||
//assert(size == BTT_STORED_4BYTE);
|
||
|
||
switch(size)
|
||
{
|
||
case BTT_STORED_4BYTE:
|
||
token->floatvalue = ReadValue<float>(OBJ);
|
||
token->intvalue = token->floatvalue;
|
||
unreadSize = sizeof(float);
|
||
break;
|
||
case BTT_STORED_2BYTE: // requested a float, but it was integral, so it was saved that way
|
||
token->floatvalue = ReadValue<short>(OBJ);
|
||
token->intvalue = token->floatvalue;
|
||
unreadSize = sizeof(short);
|
||
break;
|
||
case BTT_STORED_1BYTE: // requested a float, but it was integral, so it was saved that way
|
||
token->floatvalue = ReadValue<char>(OBJ);
|
||
token->intvalue = token->floatvalue;
|
||
unreadSize = sizeof(char);
|
||
break;
|
||
default:
|
||
assert(false);
|
||
}
|
||
|
||
// I hate the fact that I have to copy into the string, but there's no way
|
||
// of easily knowing whether it is used later or not
|
||
idStr::snPrintf( buffer, buffersize, "%f", token->floatvalue );
|
||
*token = buffer;
|
||
token->subtype = TT_FLOAT | TT_DECIMAL | TT_DOUBLE_PRECISION | TT_VALUESVALID;
|
||
break;
|
||
}
|
||
}
|
||
break;
|
||
|
||
default:
|
||
// unsupported binary type
|
||
assert(false);
|
||
}
|
||
}
|
||
|
||
//common->Warning("Read Token: '%s (int: %d)(float: %f)'\n", token->c_str(), token->GetIntValue(), token->GetFloatValue());
|
||
//common->Printf("Read Token: %s\n", token->c_str());
|
||
offset += unreadSize+1; // the +1 is for the prefix
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
int Lexer::IsLoaded()
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->IsLoaded();
|
||
}
|
||
else
|
||
{
|
||
#ifdef LEXER_READ_AHEAD
|
||
return mLexerIOWrapper.IsLoaded();
|
||
#else
|
||
return mFile != NULL;
|
||
#endif
|
||
}
|
||
}
|
||
|
||
void Lexer::FreeSource()
|
||
{
|
||
#ifdef LEXER_READ_AHEAD
|
||
mLexerIOWrapper.Close();
|
||
#else
|
||
if(mFile)
|
||
{
|
||
idLib::fileSystem->CloseFile(mFile);
|
||
mFile = NULL;
|
||
}
|
||
#endif
|
||
|
||
if(mDelegate)
|
||
{
|
||
delete mDelegate;
|
||
mDelegate = NULL;
|
||
}
|
||
}
|
||
|
||
int Lexer::LoadMemory(char const *ptr, int length, char const *name, int startLine)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->LoadMemory(ptr, length, name, startLine);
|
||
}
|
||
else
|
||
{
|
||
// this is essentially a no op
|
||
return true;
|
||
}
|
||
}
|
||
|
||
int Lexer::LoadFile(char const *filename, bool OSPath)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
return mDelegate->LoadFile(filename, OSPath);
|
||
}
|
||
else
|
||
{
|
||
idStr fullName;
|
||
FreeSource();
|
||
|
||
fullName = filename;
|
||
fullName.Append(sCompiledFileSuffix);
|
||
bool binaryFound = OpenFile(fullName, OSPath);
|
||
if(binaryFound)
|
||
{
|
||
return binaryFound;
|
||
}
|
||
else
|
||
{
|
||
mDelegate = new idLexer(filename, mFlags, OSPath);
|
||
int isLoaded = mDelegate->IsLoaded();
|
||
if(isLoaded)
|
||
{
|
||
// don't do this right now until I clean up the Lexer class
|
||
if(mFlags & LEXFL_READBINARY)
|
||
{
|
||
Warning("%s not found, loading ascii version", fullName.c_str());
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// didn't find the ascii version either, make sure and clean up in case this lexer is reused
|
||
delete mDelegate;
|
||
mDelegate = NULL;
|
||
}
|
||
|
||
return isLoaded;
|
||
}
|
||
}
|
||
}
|
||
|
||
void Lexer::WriteBinaryToken(idToken *tok)
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
mDelegate->WriteBinaryToken(tok);
|
||
}
|
||
else
|
||
{
|
||
// can't write out an already binary file
|
||
assert(false);
|
||
}
|
||
}
|
||
|
||
bool Lexer::OpenFile(char const *filename, bool OSPath)
|
||
{
|
||
idStr pathname;
|
||
|
||
if ( !OSPath && ( baseFolder[0] != '\0' ) ) {
|
||
pathname = va( "%s/%s", baseFolder, filename );
|
||
} else {
|
||
pathname = filename;
|
||
}
|
||
|
||
#ifdef LEXER_READ_AHEAD
|
||
if ( !mLexerIOWrapper.OpenFile( pathname, OSPath ) ) {
|
||
return false;
|
||
}
|
||
#else
|
||
if ( OSPath ) {
|
||
mFile = idLib::fileSystem->OpenExplicitFileRead( pathname );
|
||
} else {
|
||
mFile = idLib::fileSystem->OpenFileRead( pathname );
|
||
}
|
||
if ( !mFile ) {
|
||
return false;
|
||
}
|
||
#endif
|
||
|
||
return true;
|
||
}
|
||
|
||
void Lexer::SetBaseFolder( const char *path ) {
|
||
idStr::Copynz( baseFolder, path, sizeof( baseFolder ) );
|
||
}
|
||
|
||
|
||
// RAVEN BEGIN
|
||
// dluetscher: added method to parse a structure array that is made up of numerics (floats, ints), and stores them in the given storage
|
||
void Lexer::ParseNumericStructArray( int numStructElements, int tokenSubTypeStructElements[], int arrayCount, byte *arrayStorage )
|
||
{
|
||
if(mDelegate)
|
||
{
|
||
mDelegate->ParseNumericStructArray( numStructElements, tokenSubTypeStructElements, arrayCount, arrayStorage );
|
||
}
|
||
else
|
||
{
|
||
int arrayOffset, curElement;
|
||
|
||
for ( arrayOffset = 0; arrayOffset < arrayCount; arrayOffset++ )
|
||
{
|
||
for ( curElement = 0; curElement < numStructElements; curElement++ )
|
||
{
|
||
if ( tokenSubTypeStructElements[curElement] & TT_FLOAT )
|
||
{
|
||
*(float*)arrayStorage = Lexer::ParseFloat();
|
||
arrayStorage += sizeof(float);
|
||
}
|
||
else
|
||
{
|
||
*(int*)arrayStorage = Lexer::ParseInt();
|
||
arrayStorage += sizeof(int);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// RAVEN END
|
||
|
||
|
||
char idLexer::baseFolder[256];
|
||
// RAVEN END
|