// Copyright (C) 2007 Id Software, Inc.
//

#include "../precompiled.h"
#pragma hdrstop

#define PUNCTABLE

// Auto-close files on destruction
// Prefer this to using idFile* in most cases, since this is exception-safe
struct sdCleanupPolicy_CloseIdLibMemoryFile {
	static void Free( idFile_Memory* fp ) {
		idLib::fileSystem->CloseFile( fp );
	}
};

//longer punctuations first
static punctuation_t default_punctuations[] = {
	// binary operators
	{">>=",P_RSHIFT_ASSIGN},
	{"<<=",P_LSHIFT_ASSIGN},
	//
	{"...",P_PARMS},
	{"->*",P_POINTER_TO_MEMBER_POINTER},
	// define merge operator
	{"##",P_PRECOMPMERGE},				// pre-compiler
	// logic operators
	{"&&",P_LOGIC_AND},					// pre-compiler
	{"||",P_LOGIC_OR},					// pre-compiler
	{">=",P_LOGIC_GEQ},					// pre-compiler
	{"<=",P_LOGIC_LEQ},					// pre-compiler
	{"==",P_LOGIC_EQ},					// pre-compiler
	{"!=",P_LOGIC_UNEQ},				// pre-compiler
	// arithmetic operators
	{"*=",P_MUL_ASSIGN},
	{"/=",P_DIV_ASSIGN},
	{"%=",P_MOD_ASSIGN},
	{"+=",P_ADD_ASSIGN},
	{"-=",P_SUB_ASSIGN},
	{"++",P_INC},
	{"--",P_DEC},
	// binary operators
	{"&=",P_BIN_AND_ASSIGN},
	{"|=",P_BIN_OR_ASSIGN},
	{"^=",P_BIN_XOR_ASSIGN},
	{">>",P_RSHIFT},					// pre-compiler
	{"<<",P_LSHIFT},					// pre-compiler
	// member selection
	{"->",P_MEMBER_SELECTION_POINTER},
	{"::",P_SCOPE_RESOLUTION},
	{".*",P_POINTER_TO_MEMBER_OBJECT},
	// arithmetic operators
	{"*",P_MUL},						// pre-compiler
	{"/",P_DIV},						// pre-compiler
	{"%",P_MOD},						// pre-compiler
	{"+",P_ADD},						// pre-compiler
	{"-",P_SUB},						// pre-compiler
	{"=",P_ASSIGN},
	// binary operators
	{"&",P_BIN_AND},					// pre-compiler
	{"|",P_BIN_OR},						// pre-compiler
	{"^",P_BIN_XOR},					// pre-compiler
	{"~",P_BIN_NOT},					// pre-compiler
	// logic operators
	{"!",P_LOGIC_NOT},					// pre-compiler
	{">",P_LOGIC_GREATER},				// pre-compiler
	{"<",P_LOGIC_LESS},					// pre-compiler
	// member selection
	{".",P_MEMBER_SELECTION_OBJECT},
	// separators
	{",",P_COMMA},						// pre-compiler
	{";",P_SEMICOLON},
	// label indication
	{":",P_COLON},						// pre-compiler
	// if statement
	{"?",P_QUESTIONMARK},				// pre-compiler
	// embracements
	{"(",P_PARENTHESESOPEN},			// pre-compiler
	{")",P_PARENTHESESCLOSE},			// pre-compiler
	{"{",P_BRACEOPEN},					// pre-compiler
	{"}",P_BRACECLOSE},					// pre-compiler
	{"[",P_SQBRACKETOPEN},
	{"]",P_SQBRACKETCLOSE},
	//
	{"\\",P_BACKSLASH},
	// precompiler operator
	{"#",P_PRECOMP},					// pre-compiler
	{"$",P_DOLLAR},
	{NULL, 0}
};

int default_punctuationtable[256];
int default_nextpunctuation[sizeof(default_punctuations) / sizeof(punctuation_t)];
int default_setup;

char idLexer::baseFolder[ 256 ];

/*
================
idLexer::CreatePunctuationTable
================
*/
void idLexer::CreatePunctuationTable( const punctuation_t *punctuations ) {
	int i, n, lastp;
	const punctuation_t *p, *newp;

	//get memory for the table
	if ( punctuations == default_punctuations ) {
		idLexer::punctuationtable = default_punctuationtable;
		idLexer::nextpunctuation = default_nextpunctuation;
		if ( default_setup ) {
			return;
		}
		default_setup = true;
		i = sizeof(default_punctuations) / sizeof(punctuation_t);
	}
	else {
		if ( !idLexer::punctuationtable || idLexer::punctuationtable == default_punctuationtable ) {
			idLexer::punctuationtable = (int *) Mem_Alloc(256 * sizeof(int));
		}
		if ( idLexer::nextpunctuation && idLexer::nextpunctuation != default_nextpunctuation ) {
			Mem_Free( idLexer::nextpunctuation );
		}
		for (i = 0; punctuations[i].p; i++) {
		}
		idLexer::nextpunctuation = (int *) Mem_Alloc(i * sizeof(int));
	}
	memset(idLexer::punctuationtable, 0xFF, 256 * sizeof(int));
	memset(idLexer::nextpunctuation, 0xFF, i * sizeof(int));
	//add the punctuations in the list to the punctuation table
	for (i = 0; punctuations[i].p; i++) {
		newp = &punctuations[i];
		lastp = -1;
		//sort the punctuations in this table entry on length (longer punctuations first)
		for (n = idLexer::punctuationtable[(unsigned int) newp->p[0]]; n >= 0; n = idLexer::nextpunctuation[n] ) {
			p = &punctuations[n];
			if (idStr::Length(p->p) < idStr::Length(newp->p)) {
				idLexer::nextpunctuation[i] = n;
				if (lastp >= 0) {
					idLexer::nextpunctuation[lastp] = i;
				}
				else {
					idLexer::punctuationtable[(unsigned int) newp->p[0]] = i;
				}
				break;
			}
			lastp = n;
		}
		if (n < 0) {
			idLexer::nextpunctuation[i] = -1;
			if (lastp >= 0) {
				idLexer::nextpunctuation[lastp] = i;
			}
			else {
				idLexer::punctuationtable[(unsigned int) newp->p[0]] = i;
			}
		}
	}
}

/*
================
idLexer::GetPunctuationFromId
================
*/
const char *idLexer::GetPunctuationFromId( int id ) const {
	int i;

	for (i = 0; idLexer::punctuations[i].p; i++) {
		if ( idLexer::punctuations[i].n == id ) {
			return idLexer::punctuations[i].p;
		}
	}
	return "unkown punctuation";
}

/*
================
idLexer::GetPunctuationId
================
*/
int idLexer::GetPunctuationId( const char *p ) const {
	int i;

	for (i = 0; idLexer::punctuations[i].p; i++) {
		if ( !idStr::Cmp(idLexer::punctuations[i].p, p) ) {
			return idLexer::punctuations[i].n;
		}
	}
	return 0;
}

/*
================
idLexer::Error
================
*/
void idLexer::Error( const char *str, ... ) {
	char text[MAX_STRING_CHARS];
	va_list ap;

	hadError = true;

	if ( idLexer::flags & LEXFL_NOERRORS ) {
		return;
	}

	va_start(ap, str);
	vsprintf(text, str, ap);
	va_end(ap);

	if ( idLexer::flags & LEXFL_VCSTYLEREPORTS ) {
		idStr path = idLexer::filename;
		path.SlashesToBackSlashes();
		idLib::common->Printf( "%s(%d) : error : %s\n", path.c_str(), idLexer::line, text );
	} else if ( idLexer::flags & LEXFL_NOFATALERRORS ) {
		idLib::common->Warning( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text );
	} else {
		idLib::common->Error( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text );
	}
}

/*
================
idLexer::Warning
================
*/
void idLexer::Warning( const char *str, ... ) {
	char text[4096];
	va_list ap;

	hadWarning = true;

	if ( idLexer::flags & LEXFL_NOWARNINGS ) {
		return;
	}

	va_start( ap, str );
	vsprintf( text, str, ap );
	va_end( ap );

	if ( idLexer::flags & LEXFL_VCSTYLEREPORTS ) {
		idStr path = idLexer::filename;
		path.SlashesToBackSlashes();
		idLib::common->Printf( "%s(%d) : warning : %s\n", path.c_str(), idLexer::line, text );
	} else {
		idLib::common->Warning( "file %s, line %d: %s", idLexer::filename.c_str(), idLexer::line, text );
	}
}

/*
================
idLexer::SetPunctuations
================
*/
void idLexer::SetPunctuations( const punctuation_t *p ) {
	assert( !loaded );		// punctuations must be set before loading the file
#ifdef PUNCTABLE
	if ( p != NULL ) {
		CreatePunctuationTable( p );
	} else {
		CreatePunctuationTable( default_punctuations );
	}
#endif //PUNCTABLE
	if ( p != NULL ) {
		punctuations = p;
	} else {
		punctuations = default_punctuations;
	}
}

/*
================
idLexer::SkipWhiteSpace

Reads spaces, tabs, C-like comments etc.
When a newline character is found the scripts line counter is increased.
Returns 0 if there is no token left to be read.
================
*/
int idLexer::SkipWhiteSpace( bool currentLine ) {
	while( 1 ) {
		// skip white space
		while( *idLexer::script_p <= ' ' ) {
			if ( !*idLexer::script_p ) {
				return 0;
			}
			if ( *idLexer::script_p == '\n' ) {
				idLexer::line++;
				if ( currentLine ) {
					idLexer::script_p++;
					return 1;
				}
			}
			idLexer::script_p++;
		}
		// skip comments
		if ( *idLexer::script_p == '/' ) {
			// comments //
			if ( *(idLexer::script_p+1) == '/' ) {
				idLexer::script_p++;
				do {
					idLexer::script_p++;
					if ( !*idLexer::script_p ) {
						return 0;
					}
				}
				while( *idLexer::script_p != '\n' );
				idLexer::line++;
				idLexer::script_p++;
				if ( currentLine ) {
					return 1;
				}
				if ( !*idLexer::script_p ) {
					return 0;
				}
				continue;
			}
			// comments /* */
			else if ( *(idLexer::script_p+1) == '*' ) {
				idLexer::script_p++;
				while( 1 ) {
					idLexer::script_p++;
					if ( !*idLexer::script_p ) {
						return 0;
					}
					if ( *idLexer::script_p == '\n' ) {
						idLexer::line++;
					}
					else if ( *idLexer::script_p == '/' ) {
						if ( *(idLexer::script_p-1) == '*' ) {
							break;
						}
						if ( *(idLexer::script_p+1) == '*' ) {
							idLexer::Warning( "nested comment" );
						}
					}
				}
				idLexer::script_p++;
				if ( !*idLexer::script_p ) {
					return 0;
				}
				continue;
			}
		}
		break;
	}
	return 1;
}

/*
================
idLexer::ReadEscapeCharacter
================
*/
int idLexer::ReadEscapeCharacter( char *ch ) {
	int c, val, i;

	// step over the leading '\\'
	idLexer::script_p++;
	// determine the escape character
	if( !( flags & LEXFL_NOEMITSTRINGESCAPECHARS )) {
		switch(*idLexer::script_p) {
		case '\\': c = '\\'; break;
		case 'n': c = '\n'; break;
		case 'r': c = '\r'; break;
		case 't': c = '\t'; break;
		case 'v': c = '\v'; break;
		case 'b': c = '\b'; break;
		case 'f': c = '\f'; break;
		case 'a': c = '\a'; break;
		case '\'': c = '\''; break;
		case '\"': c = '\"'; break;
		case '\?': c = '\?'; break;
		case 'x':
			{
				idLexer::script_p++;
				for (i = 0, val = 0; ; i++, idLexer::script_p++) {
					c = *idLexer::script_p;
					if (c >= '0' && c <= '9')
						c = c - '0';
					else if (c >= 'A' && c <= 'Z')
						c = c - 'A' + 10;
					else if (c >= 'a' && c <= 'z')
						c = c - 'a' + 10;
					else
						break;
					val = (val << 4) + c;
				}
				idLexer::script_p--;
				if (val > 0xFF) {
					idLexer::Warning( "too large value in escape character" );
					val = 0xFF;
				}
				c = val;
				break;
			}
		default: //NOTE: decimal ASCII code, NOT octal
			{
				if (*idLexer::script_p < '0' || *idLexer::script_p > '9') {
					idLexer::Error("unknown escape char");
				}
				for (i = 0, val = 0; ; i++, idLexer::script_p++) {
					c = *idLexer::script_p;
					if (c >= '0' && c <= '9')
						c = c - '0';
					else
						break;
					val = val * 10 + c;
				}
				idLexer::script_p--;
				if (val > 0xFF) {
					idLexer::Warning( "too large value in escape character" );
					val = 0xFF;
				}
				c = val;
				break;
			}
		}
	} else {
		c = *script_p;
	}
	
	// step over the escape character or the last digit of the number
	idLexer::script_p++;
	// store the escape character
	*ch = c;
	// succesfully read escape character
	return 1;
}

/*
================
idLexer::ReadString

Escape characters are interpretted.
Reads two strings with only a white space between them as one string.
================
*/
int idLexer::ReadString( idToken *token, int quote ) {
	int tmpline;
	const char *tmpscript_p;
	char ch;

	if ( quote == '\"' ) {
		token->type = TT_STRING;
	} else {
		token->type = TT_LITERAL;
	}

	// leading quote
	idLexer::script_p++;

	while(1) {
		// if there is an escape character and escape characters are allowed
		if (*idLexer::script_p == '\\' && !(idLexer::flags & LEXFL_NOSTRINGESCAPECHARS)) {
			if(( flags & LEXFL_NOEMITSTRINGESCAPECHARS )) {
				token->AppendDirty( '\\' );
			}

			if ( !idLexer::ReadEscapeCharacter( &ch ) ) {
				return 0;
			}
			token->AppendDirty( ch );
		}
		// if a trailing quote
		else if (*idLexer::script_p == quote) {
			// step over the quote
			idLexer::script_p++;
			// if consecutive strings should not be concatenated
			if ( (idLexer::flags & LEXFL_NOSTRINGCONCAT) &&
					(!(idLexer::flags & LEXFL_ALLOWBACKSLASHSTRINGCONCAT) || (quote != '\"')) ) {
				break;
			}

			tmpscript_p = idLexer::script_p;
			tmpline = idLexer::line;
			// read white space between possible two consecutive strings
			if ( !idLexer::SkipWhiteSpace( false ) ) {
				idLexer::script_p = tmpscript_p;
				idLexer::line = tmpline;
				break;
			}

			if ( idLexer::flags & LEXFL_NOSTRINGCONCAT ) {
				if ( *idLexer::script_p != '\\' ) {
					idLexer::script_p = tmpscript_p;
					idLexer::line = tmpline;
					break;
				}
				// step over the '\\'
				idLexer::script_p++;
				if ( !idLexer::SkipWhiteSpace( false ) || ( *idLexer::script_p != quote ) ) {
					idLexer::Error( "expecting string after '\\' terminated line" );
					return 0;
				}
			}

			// if there's no leading qoute
			if ( *idLexer::script_p  != quote ) {
				idLexer::script_p = tmpscript_p;
				idLexer::line = tmpline;
				break;
			}
			// step over the new leading quote
			idLexer::script_p++;
		}
		else {
			if (*idLexer::script_p == '\0') {
				idLexer::Error( "missing trailing quote" );
				return 0;
			}
			if (*idLexer::script_p == '\n') {
				idLexer::Error( "newline inside string" );
				return 0;
			}
			token->AppendDirty( *idLexer::script_p++ );
		}
	}
	token->data[token->len] = '\0';

	if ( token->type == TT_LITERAL ) {
		if ( !(idLexer::flags & LEXFL_ALLOWMULTICHARLITERALS) ) {
			if ( token->Length() != 1 ) {
				idLexer::Warning( "literal is not one character long" );
			}
		}
		token->subtype = (*token)[0];
	}
	else {
		// the sub type is the length of the string
		token->subtype = token->Length();
	}
	return 1;
}

/*
================
idLexer::ReadName
================
*/
int idLexer::ReadName( idToken *token ) {
	char c;

	token->type = TT_NAME;
	do {
		token->AppendDirty( *idLexer::script_p++ );
		c = *idLexer::script_p;
	} while ((c >= 'a' && c <= 'z') ||
				(c >= 'A' && c <= 'Z') ||
				(c >= '0' && c <= '9') ||
				c == '_' ||
				// if treating all tokens as strings, don't parse '-' as a separate token
				((idLexer::flags & LEXFL_ONLYSTRINGS) && (c == '-')) ||
				// if special path name characters are allowed
				((idLexer::flags & LEXFL_ALLOWPATHNAMES) && (c == '/' || c == '\\' || c == ':' || c == '.')) );
	token->data[token->len] = '\0';
	//the sub type is the length of the name
	token->subtype = token->Length();
	return 1;
}

/*
================
idLexer::CheckString
================
*/
ID_INLINE int idLexer::CheckString( const char *str ) const {
	int i;

	for ( i = 0; str[i]; i++ ) {
		if ( idLexer::script_p[i] != str[i] ) {
			return false;
		}
	}
	return true;
}

/*
================
idLexer::ReadNumber
================
*/
int idLexer::ReadNumber( idToken *token ) {
	int i;
	int dot;
	char c, c2;

	token->type = TT_NUMBER;
	token->subtype = 0;
	token->intvalue = 0;
	token->floatvalue = 0;

	c = *idLexer::script_p;
	c2 = *(idLexer::script_p + 1);

	if ( c == '0' && c2 != '.' ) {
		// check for a hexadecimal number
		if ( c2 == 'x' || c2 == 'X' ) {
			token->AppendDirty( *idLexer::script_p++ );
			token->AppendDirty( *idLexer::script_p++ );
			c = *idLexer::script_p;
			while((c >= '0' && c <= '9') ||
						(c >= 'a' && c <= 'f') ||
						(c >= 'A' && c <= 'F')) {
				token->AppendDirty( c );
				c = *(++idLexer::script_p);
			}
			token->subtype = TT_HEX | TT_INTEGER;
		}
		// check for a binary number
		else if ( c2 == 'b' || c2 == 'B' ) {
			token->AppendDirty( *idLexer::script_p++ );
			token->AppendDirty( *idLexer::script_p++ );
			c = *idLexer::script_p;
			while( c == '0' || c == '1' ) {
				token->AppendDirty( c );
				c = *(++idLexer::script_p);
			}
			token->subtype = TT_BINARY | TT_INTEGER;
		}
		// its an octal number
		else {
			token->AppendDirty( *idLexer::script_p++ );
			c = *idLexer::script_p;
			while( c >= '0' && c <= '7' ) {
				token->AppendDirty( c );
				c = *(++idLexer::script_p);
			}
			token->subtype = TT_OCTAL | TT_INTEGER;
		}
	}
	else {
		// decimal integer or floating point number or ip address
		dot = 0;
		while( 1 ) {
			if ( c >= '0' && c <= '9' ) {
			} else if ( c == '.' ) {
				dot++;
			} else {
				break;
			}
			token->AppendDirty( c );
			c = *(++idLexer::script_p);
		}
		// if a floating point number
		if ( dot == 1 || c == 'e' ) {
			token->subtype = TT_DECIMAL | TT_FLOAT;
			// check for floating point exponent
			if ( c == 'e' ) {
				c = *(++idLexer::script_p);
				if ( c == '-' ) {
					token->AppendDirty( c );
					c = *(++idLexer::script_p);
				} else if ( c == '+' ) {
					token->AppendDirty( c );
					c = *(++idLexer::script_p);
				}
				while( c >= '0' && c <= '9' ) {
					token->AppendDirty( c );
					c = *(++idLexer::script_p);
				}
			}
			// check for floating point exception infinite 1.#INF or indefinite 1.#IND or NaN
			else if ( c == '#' ) {
				token->AppendDirty( c );
				c = *(++idLexer::script_p);
				if ( CheckString( "INF" ) ) {
					token->subtype |= TT_INFINITE;
					c2 = 3;
				} else if ( CheckString( "IND" ) ) {
					token->subtype |= TT_INDEFINITE;
					c2 = 3;
				} else if ( CheckString( "NAN" ) ) {
					token->subtype |= TT_NAN;
					c2 = 3;
				} else if ( CheckString( "QNAN" ) ) {
					token->subtype |= TT_NAN;
					c2 = 4;
				} else if ( CheckString( "SNAN" ) ) {
					token->subtype |= TT_NAN;
					c2 = 4;
				}
				for ( i = 0; i < c2; i++ ) {
					token->AppendDirty( c );
					c = *(++idLexer::script_p);
				}
				while( c >= '0' && c <= '9' ) {
					token->AppendDirty( c );
					c = *(++idLexer::script_p);
				}
				if ( !(idLexer::flags & LEXFL_ALLOWFLOATEXCEPTIONS) ) {
					token->AppendDirty( 0 );	// zero terminate for c_str
					idLexer::Error( "parsed %s", token->c_str() );
				}
			}
		}
		else if ( dot > 1 ) {
			if ( !( idLexer::flags & LEXFL_ALLOWIPADDRESSES ) ) {
				idLexer::Error( "more than one dot in number" );
				return 0;
			}
			if ( dot != 3 ) {
				idLexer::Error( "ip address should have three dots" );
				return 0;
			}
			token->subtype = TT_IPADDRESS;
		}
		else {
			token->subtype = TT_DECIMAL | TT_INTEGER;
		}
	}

	if ( token->subtype & TT_FLOAT ) {
		if ( c > ' ' ) {
			// single-precision: float
			if ( c == 'f' || c == 'F' ) {
				token->subtype |= TT_SINGLE_PRECISION;
				idLexer::script_p++;
			}
			// extended-precision: long double
			else if ( c == 'l' || c == 'L' ) {
				token->subtype |= TT_EXTENDED_PRECISION;
				idLexer::script_p++;
			}
			// default is double-precision: double
			else {
				token->subtype |= TT_DOUBLE_PRECISION;
			}
		}
		else {
			token->subtype |= TT_DOUBLE_PRECISION;
		}
	}
	else if ( token->subtype & TT_INTEGER ) {
		if ( c > ' ' ) {
			// default: signed long
			for ( i = 0; i < 2; i++ ) {
				// long integer
				if ( c == 'l' || c == 'L' ) {
					token->subtype |= TT_LONG;
				}
				// unsigned integer
				else if ( c == 'u' || c == 'U' ) {
					token->subtype |= TT_UNSIGNED;
				}
				else {
					break;
				}
				c = *(++idLexer::script_p);
			}
		}
	}
	else if ( token->subtype & TT_IPADDRESS ) {
		if ( c == ':' ) {
			token->AppendDirty( c );
			c = *(++idLexer::script_p);
			while( c >= '0' && c <= '9' ) {
				token->AppendDirty( c );
				c = *(++idLexer::script_p);
			}
			token->subtype |= TT_IPPORT;
		}
	}
	token->data[token->len] = '\0';
	return 1;
}

/*
================
idLexer::ReadRawStringBlock
================
*/
int idLexer::ReadRawStringBlock( idToken *token ) {
	token->type = TT_STRING;

	// leading identifier
	idLexer::script_p += 2;

	while ( true ) {
		// if a trailing identifier
		if ( idLexer::script_p[0] == '%' && idLexer::script_p[1] == '>' ) {
			// step over the identifier
			idLexer::script_p += 2;

			break;
		} else {
			if ( *idLexer::script_p == '\0' ) {
				idLexer::Error( "missing trailing identifier" );
				return 0;
			}
			if ( *idLexer::script_p == '\n' ) {
				line++;
			}
			token->AppendDirty( *idLexer::script_p++ );
		}
	}
	token->data[token->len] = '\0';

	// the sub type is the length of the string
	token->subtype = token->Length();

	return 1;
}

/*
================
idLexer::ReadPunctuation
================
*/
int idLexer::ReadPunctuation( idToken *token ) {
	int l, n, i;
	char *p;
	const punctuation_t *punc;

#ifdef PUNCTABLE
	for (n = idLexer::punctuationtable[(unsigned int)*(idLexer::script_p)]; n >= 0; n = idLexer::nextpunctuation[n])
	{
		punc = &(idLexer::punctuations[n]);
#else
	for (n = 0; idLexer::punctuations[n].p; n++) {
		punc = &idLexer::punctuations[n];
#endif
		p = punc->p;
		// check for this punctuation in the script
		for ( l = 0; p[l] && idLexer::script_p[l]; l++ ) {
			if ( idLexer::script_p[l] != p[l] ) {
				break;
			}
		}
		if ( !p[l] ) {
			//
			token->EnsureAlloced( l+1, false );
			for ( i = 0; i <= l; i++ ) {
				token->data[i] = p[i];
			}
			token->len = l;
			//
			idLexer::script_p += l;
			token->type = TT_PUNCTUATION;
			// sub type is the punctuation id
			token->subtype = punc->n;
			return 1;
		}
	}
	return 0;
}

/*
================
idLexer::ReadToken
================
*/
int idLexer::ReadToken( idToken *token ) {

	if ( !loaded ) {
		idLib::common->Error( "idLexer::ReadToken: no file loaded" );
		return 0;
	}

	// if there is a token available (from unreadToken)
	if ( tokens.Num() ) {
		*token = tokens[ tokens.Num() - 1 ];
		tokens.RemoveIndex( tokens.Num() - 1 );
		return 1;
	}

	// read from binary file if loaded
	if ( binary.IsLoaded() ) {
		return binary.ReadToken( token );
	}

	// save script pointer
	lastScript_p = script_p;
	// save line counter
	lastline = line;
	// clear the token stuff
	token->data[0] = '\0';
	token->len = 0;
	// start of the white space
	whiteSpaceStart_p = script_p;
	token->whiteSpaceStart_p = script_p;
	// read white space before token
	if ( !SkipWhiteSpace( false ) ) {
		return 0;
	}
	// end of the white space
	idLexer::whiteSpaceEnd_p = script_p;
	token->whiteSpaceEnd_p = script_p;
	// line the token is on
	token->line = line;
	// number of lines crossed before token
	token->linesCrossed = line - lastline;
	// clear token flags
	token->flags = 0;

	int c = *idLexer::script_p;

	// if we're keeping everything as whitespace deliminated strings
	if ( idLexer::flags & LEXFL_ONLYSTRINGS ) {
		// if there is a leading quote
		if ( c == '\"' || c == '\'' ) {
			if (!idLexer::ReadString( token, c )) {
				return 0;
			}
		} else if ( !idLexer::ReadName( token ) ) {
			return 0;
		}
	}
	// if there is a number
	else if ( (c >= '0' && c <= '9') ||
			(c == '.' && (*(idLexer::script_p + 1) >= '0' && *(idLexer::script_p + 1) <= '9')) ) {
		if ( !idLexer::ReadNumber( token ) ) {
			return 0;
		}
		// if names are allowed to start with a number
		if ( idLexer::flags & LEXFL_ALLOWNUMBERNAMES ) {
			c = *idLexer::script_p;
			if ( (c >= 'a' && c <= 'z') ||	(c >= 'A' && c <= 'Z') || c == '_' ) {
				if ( !idLexer::ReadName( token ) ) {
					return 0;
				}
			}
		}
	}
	// if there is a leading quote
	else if ( c == '\"' || c == '\'' ) {
		if (!idLexer::ReadString( token, c )) {
			return 0;
		}
	}
	// if there is a name
	else if ( (c >= 'a' && c <= 'z') ||	(c >= 'A' && c <= 'Z') || c == '_' ) {
		if ( !idLexer::ReadName( token ) ) {
			return 0;
		}
	}
	// names may also start with a slash when pathnames are allowed
	else if ( ( idLexer::flags & LEXFL_ALLOWPATHNAMES ) && ( (c == '/' || c == '\\') || c == '.' ) ) {
		if ( !idLexer::ReadName( token ) ) {
			return 0;
		}
	}
	// if there is a raw string block
	else if ( ( idLexer::flags & LEXFL_ALLOWRAWSTRINGBLOCKS ) && ( idLexer::script_p[0] == '<' && idLexer::script_p[1] == '%' ) ) {
		if ( !idLexer::ReadRawStringBlock( token ) ) {
			return 0;
		}
	}
	// check for punctuations
	else if ( !idLexer::ReadPunctuation( token ) ) {
		idLexer::Error( "unknown punctuation %c", c );
		return 0;
	}
	// succesfully read a token
	return 1;
}

/*
================
idLexer::ExpectTokenString
================
*/
bool idLexer::ExpectTokenString( const char *string ) {
	idToken token;

	if (!idLexer::ReadToken( &token )) {
		idLexer::Error( "couldn't find expected '%s'", string );
		return false;
	}
	if ( token != string ) {
		idLexer::Error( "expected '%s' but found '%s'", string, token.c_str() );
		return false;
	}
	return true;
}

/*
================
idLexer::ExpectTokenType
================
*/
bool idLexer::ExpectTokenType( int type, int subtype, idToken *token ) {
	idStr str;

	if ( !idLexer::ReadToken( token ) ) {
		idLexer::Error( "couldn't read expected token" );
		return false;
	}

	if ( token->type != type ) {
		switch( type ) {
			case TT_STRING: str = "string"; break;
			case TT_LITERAL: str = "literal"; break;
			case TT_NUMBER: str = "number"; break;
			case TT_NAME: str = "name"; break;
			case TT_PUNCTUATION: str = "punctuation"; break;
			default: str = "unknown type"; break;
		}
		idLexer::Error( "expected a %s but found '%s'", str.c_str(), token->c_str() );
		return false;
	}
	if ( token->type == TT_NUMBER ) {
		if ( (token->subtype & subtype) != subtype ) {
			str.Clear();
			if ( subtype & TT_DECIMAL ) str = "decimal ";
			if ( subtype & TT_HEX ) str = "hex ";
			if ( subtype & TT_OCTAL ) str = "octal ";
			if ( subtype & TT_BINARY ) str = "binary ";
			if ( subtype & TT_UNSIGNED ) str += "unsigned ";
			if ( subtype & TT_LONG ) str += "long ";
			if ( subtype & TT_FLOAT ) str += "float ";
			if ( subtype & TT_INTEGER ) str += "integer ";
			str.StripTrailing( ' ' );
			idLexer::Error( "expected %s but found '%s'", str.c_str(), token->c_str() );
			return false;
		}
	}
	else if ( token->type == TT_PUNCTUATION ) {
		if ( subtype < 0 ) {
			idLexer::Error( "BUG: wrong punctuation subtype" );
			return false;
		}
		if ( token->subtype != subtype ) {
			idLexer::Error( "expected '%s' but found '%s'", GetPunctuationFromId( subtype ), token->c_str() );
			return false;
		}
	}
	return true;
}

/*
================
idLexer::ExpectAnyToken
================
*/
bool idLexer::ExpectAnyToken( idToken *token ) {
	if (!idLexer::ReadToken( token )) {
		idLexer::Error( "couldn't read expected token" );
		return false;
	}
	else {
		return true;
	}
}

/*
================
idLexer::CheckTokenString
================
*/
bool idLexer::CheckTokenString( const char *string ) {
	idToken tok;

	if ( !ReadToken( &tok ) ) {
		return false;
	}
	// if the given string is available
	if ( tok == string ) {
		return true;
	}
	// unread token
	UnreadToken( &tok );
	return false;
}

/*
================
idLexer::CheckTokenType
================
*/
bool idLexer::CheckTokenType( int type, int subtype, idToken *token ) {
	idToken tok;

	if ( !ReadToken( &tok ) ) {
		return false;
	}
	// if the type matches
	if ( tok.type == type && ( tok.subtype & subtype ) == subtype ) {
		*token = tok;
		return true;
	}
	// unread token
	UnreadToken( &tok );
	return false;
}

/*
================
idLexer::PeekTokenString
================
*/
int idLexer::PeekTokenString( const char *string ) {
	idToken tok;

	if ( !ReadToken( &tok ) ) {
		return 0;
	}

	// unread token
	UnreadToken( &tok );

	// if the given string is available
	if ( tok == string ) {
		return 1;
	}
	return 0;
}

/*
================
idLexer::PeekTokenType
================
*/
int idLexer::PeekTokenType( int type, int subtype, idToken *token ) {
	idToken tok;

	if ( !ReadToken( &tok ) ) {
		return 0;
	}

	// unread token
	UnreadToken( &tok );

	// if the type matches
	if ( tok.type == type && ( tok.subtype & subtype ) == subtype ) {
		*token = tok;
		return 1;
	}
	return 0;
}

/*
================
idLexer::SkipUntilString
================
*/
bool idLexer::SkipUntilString( const char *string, idToken* token ) {
	idToken _token;
	if ( !token ) {
		token = &_token;
	}

	while( idLexer::ReadToken( token ) ) {
		if ( *token == string ) {
			return true;
		}
	}
	return false;
}

/*
================
idLexer::SkipRestOfLine
================
*/
int idLexer::SkipRestOfLine( void ) {
	idToken token;

	while( ReadToken( &token ) ) {
		if ( token.linesCrossed ) {
			UnreadToken( &token );
			return 1;
		}
	}
	return 0;
}

/*
=================
idLexer::SkipBracedSection

Skips until a matching close brace is found.
Internal brace depths are properly skipped.
=================
*/
int idLexer::SkipBracedSection( bool parseFirstBrace ) {
	idToken token;
	int depth;

	depth = parseFirstBrace ? 0 : 1;
	do {
		if ( !ReadToken( &token ) ) {
			return false;
		}
		if ( token.type == TT_PUNCTUATION ) {
			if ( token == "{" ) {
				depth++;
			} else if ( token == "}" ) {
				depth--;
			}
		}
	} while( depth );
	return true;
}

/*
=================
idLexer::SkipBracedSectionExact

Skips until a matching close brace is found.
Internal brace depths are properly skipped.
Reads exact characters between braces.
=================
*/
bool idLexer::SkipBracedSectionExact( int tabs, bool parseFirstBrace ) {
	int		depth;
	bool	doTabs;
	bool	skipWhite;

	if ( parseFirstBrace ) {
		if ( !idLexer::ExpectTokenString( "{" ) ) {
			return false;
		}
	}

	depth = 1;	
	skipWhite = false;
	doTabs = ( tabs >= 0 );

	while( depth ) {
		if ( !*idLexer::script_p ) {
			return false;
		}

		char c = *(idLexer::script_p++);

		switch ( c ) {
			case '\t':
			case ' ': {
				if ( skipWhite ) {
					continue;
				}
				break;
			}
			case '\n': {
				if ( doTabs ) {
					skipWhite = true;
					continue;
				}
				line++;
				break;
			}
			case '{': {
				depth++;
				tabs++;
				break;
			}
			case '}': {
				depth--;
				tabs--;
				break;				
			}
		}

		if ( skipWhite ) {
			skipWhite = false;
		}
	}
	return true;
}

/*
================
idLexer::UnreadToken
================
*/
void idLexer::UnreadToken( const idToken *token ) {
	idToken& tokenLocal = tokens.Alloc();
	tokenLocal = *token;
}

/*
================
idLexer::ReadTokenOnLine
================
*/
int idLexer::ReadTokenOnLine( idToken *token ) {
	idToken tok;

	if ( !ReadToken( &tok ) ) {
		return false;
	}

	// if no lines were crossed before this token
	if ( !tok.linesCrossed ) {
		*token = tok;
		return true;
	}

	// restore our position
	UnreadToken( &tok );

	token->Clear();
	return false;
}

/*
================
idLexer::ParseInt
================
*/
int idLexer::ParseInt( void ) {
	idToken token;

	if ( !idLexer::ReadToken( &token ) ) {
		idLexer::Error( "couldn't read expected integer" );
		return 0;
	}
	if ( token.type == TT_PUNCTUATION && token == "-" ) {
		ExpectTokenType( TT_NUMBER, TT_INTEGER, &token );
		return -((signed int) token.GetIntValue());
	}
	else if ( token.type != TT_NUMBER || token.subtype == TT_FLOAT ) {
		idLexer::Error( "expected integer value, found '%s'", token.c_str() );
	}
	return token.GetIntValue();
}

/*
================
idLexer::ParseBool
================
*/
bool idLexer::ParseBool( void ) {
	idToken token;

	if ( !idLexer::ExpectTokenType( TT_NUMBER, 0, &token ) ) {
		idLexer::Error( "couldn't read expected boolean" );
		return false;
	}
	return ( token.GetIntValue() != 0 );
}

/*
================
idLexer::ParseFloat
================
*/
float idLexer::ParseFloat( bool *errorFlag ) {
	idToken token;

	if ( errorFlag ) {
		*errorFlag = false;
	}

	if ( !idLexer::ReadToken( &token ) ) {
		if ( errorFlag ) {
			idLexer::Warning( "couldn't read expected floating point number" );
			*errorFlag = true;
		} else {
			idLexer::Error( "couldn't read expected floating point number" );
		}
		return 0;
	}
	if ( token.type == TT_PUNCTUATION && token == "-" ) {
		idLexer::ExpectTokenType( TT_NUMBER, 0, &token );
		return -token.GetFloatValue();
	}
	else if ( token.type != TT_NUMBER ) {
		if ( errorFlag ) {
			idLexer::Warning( "expected float value, found '%s'", token.c_str() );
			*errorFlag = true;
		} else {
			idLexer::Error( "expected float value, found '%s'", token.c_str() );
		}
	}
	return token.GetFloatValue();
}

/*
================
idLexer::Parse1DMatrix
================
*/
int idLexer::Parse1DMatrix( int x, float *m, bool expectCommas ) {
	int i;

	if ( !idLexer::ExpectTokenString( "(" ) ) {
		return false;
	}

	for ( i = 0; i < x; i++ ) {
		m[i] = idLexer::ParseFloat();
		if ( expectCommas && i != x - 1 ) {
			if ( !idLexer::ExpectTokenString( "," ) ) {
				return false;
			}
		}
	}

	if ( !idLexer::ExpectTokenString( ")" ) ) {
		return false;
	}
	return true;
}

/*
================
idLexer::Parse2DMatrix
================
*/
int idLexer::Parse2DMatrix( int y, int x, float *m ) {
	int i;

	if ( !idLexer::ExpectTokenString( "(" ) ) {
		return false;
	}

	for ( i = 0; i < y; i++ ) {
		if ( !idLexer::Parse1DMatrix( x, m + i * x ) ) {
			return false;
		}
	}

	if ( !idLexer::ExpectTokenString( ")" ) ) {
		return false;
	}
	return true;
}

/*
================
idLexer::Parse3DMatrix
================
*/
int idLexer::Parse3DMatrix( int z, int y, int x, float *m ) {
	int i;

	if ( !idLexer::ExpectTokenString( "(" ) ) {
		return false;
	}

	for ( i = 0 ; i < z; i++ ) {
		if ( !idLexer::Parse2DMatrix( y, x, m + i * x*y ) ) {
			return false;
		}
	}

	if ( !idLexer::ExpectTokenString( ")" ) ) {
		return false;
	}
	return true;
}

/*
=================
idLexer::ParseBracedSectionExact

The next token should be an open brace.
Parses until a matching close brace is found.
Maintains exact characters between braces.

  FIXME: this should use ReadToken and replace the token white space with correct indents and newlines
=================
*/
bool idLexer::ParseBracedSectionExact( idStr &out, int tabs, bool parseFirstBrace ) {
	int		depth;
	bool	doTabs;
	bool	skipWhite;

	sdStringBuilder_Heap builder;

	out.Empty();

	if ( parseFirstBrace ) {
		if ( !idLexer::ExpectTokenString( "{" ) ) {
			return false;
		}
		builder = "{";
	}

	depth = 1;	
	skipWhite = false;
	doTabs = ( tabs >= 0 );

	while( depth ) {
		if ( !*idLexer::script_p ) {
			return false;
		}

		char c = *(idLexer::script_p++);

		switch ( c ) {
			case '\t':
			case ' ': {
				if ( skipWhite ) {
					continue;
				}
				break;
			}
			case '\n': {
				if ( doTabs ) {
					skipWhite = true;
					builder += c;
					continue;
				}
				line++;
				break;
			}
			case '{': {
				depth++;
				tabs++;
				break;
			}
			case '}': {
				depth--;
				tabs--;
				break;				
			}
		}

		if ( skipWhite ) {
			int i = tabs;
			if ( c == '{' ) {
				i--;
			}
			skipWhite = false;
			for ( ; i > 0; i-- ) {
				builder += '\t';
			}
		}
		builder += c;
	}
	builder.ToString( out );
	return true;
}

/*
=================
idLexer::ParseBracedSection

The next token should be an open brace.
Parses until a matching close brace is found.
Internal brace depths are properly skipped.
=================
*/
const char *idLexer::ParseBracedSection( idStr &out, int tabs, bool parseFirstBrace, char intro, char outro ) {
	idToken token;
	int i, depth;
	bool doTabs;

	sdStringBuilder_Heap builder;

	char temp[ 2 ] = { '\0', '\0' };
	temp[ 0 ] = intro;

	out.Empty();
	if ( parseFirstBrace ) {
		if ( !idLexer::ExpectTokenString( temp ) ) {
			return out.c_str();
		}
		builder = temp;
	}
	depth = 1;
	doTabs = ( tabs >= 0 );
	do {
		if ( !idLexer::ReadToken( &token ) ) {
			Error( "missing closing brace" );
			builder.ToString( out );
			return out.c_str();
		}

		// if the token is on a new line
		for ( i = 0; i < token.linesCrossed; i++ ) {
			builder += "\r\n";
		}

		if ( doTabs && token.linesCrossed ) {
			i = tabs;
			if ( token[0] == '}' && i > 0 ) {
				i--;
			}
			while( i-- > 0 ) {
				builder += "\t";
			}
		}
		if ( token.type != TT_STRING ) {
			if ( token[ 0 ] == intro ) {
				depth++;
				if ( doTabs ) {
					tabs++;
				}
			} else if ( token[ 0 ] == outro ) {
				depth--;
				if ( doTabs ) {
					tabs--;
				}
			}
			if ( builder.Length() ) {
				builder += " ";
			}
			builder += token;
		} else {
			if ( builder.Length() ) {
				builder += " ";
			}
			builder += "\"" + token + "\"";
		}
	} while( depth );

	builder.ToString( out );
	return out.c_str();
}

/*
=================
idLexer::ParseRestOfLine

Like ParseCompleteLine, but doesn't advance to next line and doesn't return \n
It does, however, trim leading and trailing white space
=================
*/
const char *idLexer::ParseRestOfLine( idStr &out ) {
	idToken token;
	const char	*start;

	start = idLexer::script_p;

	while ( 1 ) {
		// end of buffer
		if ( *idLexer::script_p == 0 ) {
			break;
		}
		if ( *idLexer::script_p == '\n' ) {
			break;
		}
		idLexer::script_p++;
	}

	// Trim leading white space
	while( *start <= ' ' && start < idLexer::script_p ) {
		start++;
	}
	// Trim trailing white space
	while( *(idLexer::script_p-1) <= ' ' && idLexer::script_p > start ) {
		idLexer::script_p--;
	}

	out.Empty();
	out.Append( start, idLexer::script_p - start );

	return out.c_str();
}

/*
=================
idLexer::ParseRestOfLine

Returns a string up to the \n, but doesn't eat any whitespace at the beginning of the next line
=================
*/
const char *idLexer::ParseCompleteLine( idStr &out ) {
	idToken token;
	const char	*start;

	start = idLexer::script_p;

	while ( 1 ) {
		// end of buffer
		if ( *idLexer::script_p == 0 ) {
			break;
		}
		if ( *idLexer::script_p == '\n' ) {
			idLexer::line++;
			idLexer::script_p++;
			break;
		}
		idLexer::script_p++;
	}

	out.Empty();
	out.Append( start, idLexer::script_p - start );

	return out.c_str();
}

/*
================
idLexer::GetNextWhiteSpace
================
*/
int idLexer::GetNextWhiteSpace( idStr &whiteSpace, bool currentLine ) {
	whiteSpaceStart_p = script_p;
	SkipWhiteSpace( currentLine );
	whiteSpaceEnd_p = script_p;
	return GetLastWhiteSpace( whiteSpace );
}

/*
================
idLexer::GetLastWhiteSpace
================
*/
int idLexer::GetLastWhiteSpace( idStr &whiteSpace ) const {
	whiteSpace.Clear();
	for ( const char *p = whiteSpaceStart_p; p < whiteSpaceEnd_p; p++ ) {
		whiteSpace.Append( *p );
	}
	return whiteSpace.Length();
}

/*
================
idLexer::GetLastWhiteSpaceStart
================
*/
int idLexer::GetLastWhiteSpaceStart( void ) const {
	return whiteSpaceStart_p - buffer;
}

/*
================
idLexer::GetLastWhiteSpaceEnd
================
*/
int idLexer::GetLastWhiteSpaceEnd( void ) const {
	return whiteSpaceEnd_p - buffer;
}

/*
================
idLexer::Reset
================
*/
void idLexer::Reset( void ) {
	// pointer in script buffer
	idLexer::script_p = idLexer::buffer;
	// pointer in script buffer before reading token
	idLexer::lastScript_p = idLexer::buffer;
	// begin of white space
	idLexer::whiteSpaceStart_p = NULL;
	// end of white space
	idLexer::whiteSpaceEnd_p = NULL;

	tokens.Clear();

	idLexer::line = 1;
	idLexer::lastline = 1;
}

/*
================
idLexer::EndOfFile
================
*/
int idLexer::EndOfFile( void ) const {
	if( binary.IsLoaded() ) {
		return binary.EndOfFile();
	}
	return idLexer::script_p >= idLexer::end_p;
}

/*
================
idLexer::NumLinesCrossed
================
*/
int idLexer::NumLinesCrossed( void ) const {
	return idLexer::line - idLexer::lastline;
}

/*
================
idLexer::LoadFile
================
*/
bool idLexer::LoadFile( const char *filename, bool OSPath, int startLine ) {
	idFile *fp;
	idStr pathname;
	int length;
	char *buf;

	if ( idLexer::loaded ) {
		idLib::common->Error("idLexer::LoadFile: another script already loaded");
		return false;
	}
/*
	if ( !OSPath ) {
		if ( binary.ReadFile( filename ) ) {
			this->loaded = true;
			this->filename = fileSystem->RelativePathToOSPath( filename, "fs_basepath" );
			this->filename.CollapsePath();
			return true;
		}
	}
*/	
	if ( !OSPath && ( baseFolder[0] != '\0' ) ) {
		pathname = va( "%s/%s", baseFolder, filename );
	} else {
		pathname = filename;
	}
	if ( OSPath ) {
		fp = idLib::fileSystem->OpenExplicitFileRead( pathname );
	} else {
		fp = idLib::fileSystem->OpenFileRead( pathname );
	}
	if ( fp == NULL ) {
		return false;
	}
	length = fp->Length();
	buf = (char *) Mem_Alloc( length + 1 );
	buf[length] = '\0';
	fp->Read( buf, length );
	this->fileTime = fp->Timestamp();
	if ( OSPath ) {
		this->filename = fp->GetFullPath();
	} else {
		this->filename = pathname;
	}
	this->filename.CollapsePath();
	idLib::fileSystem->CloseFile( fp );

	this->buffer = buf;
	this->length = length;
	// pointer in script buffer
	this->script_p = this->buffer;
	// pointer in script buffer before reading token
	this->lastScript_p = this->buffer;
	// pointer to end of script buffer
	this->end_p = &(this->buffer[length]);

	tokens.Clear();

	this->line = startLine;
	this->lastline = startLine;
	this->allocated = true;
	this->loaded = true;
/*
	if ( !OSPath ) {
		binary.WriteFile( *this, pathname );
	}
*/
	return true;
}

/*
================
idLexer::LoadMemory
================
*/
bool idLexer::LoadMemory( const char *ptr, int length, const char *name, int startLine ) {
	if ( idLexer::loaded ) {
		idLib::common->Error("idLexer::LoadMemory: another script already loaded");
		return false;
	}
	idLexer::filename = name;
	idLexer::filename.CollapsePath();
	idLexer::buffer = ptr;
	idLexer::fileTime = 0;
	idLexer::length = length;
	// pointer in script buffer
	idLexer::script_p = idLexer::buffer;
	// pointer in script buffer before reading token
	idLexer::lastScript_p = idLexer::buffer;
	// pointer to end of script buffer
	idLexer::end_p = &(idLexer::buffer[length]);

	tokens.Clear();
	idLexer::line = startLine;
	idLexer::lastline = startLine;
	idLexer::allocated = false;
	idLexer::loaded = true;

	return true;
}


/*
================
idLexer::LoadMemoryBinary
================
*/
bool idLexer::LoadMemoryBinary( const byte* ptr, int length, const char *name, idTokenCache* globals ) {
	if( ptr == NULL || length == 0 )  {
		return false;
	}
	idLexer::filename = name;
	idLexer::filename.CollapsePath();
	idLexer::buffer = NULL;
	idLexer::fileTime = 0;
	idLexer::length = 0;
	// pointer in script buffer
	idLexer::script_p = NULL;
	
	idLexer::lastScript_p = NULL;
	// pointer to end of script buffer
	idLexer::end_p = NULL;

	tokens.Clear();
	idLexer::line = 0;
	idLexer::lastline = 0;
	idLexer::allocated = false;
	idLexer::loaded = true;

	sdLibFileMemoryPtr memFile( idLib::fileSystem->OpenMemoryFile( name ) );
	
	memFile->SetData( reinterpret_cast< const char* >( ptr ), length );
	
	bool retVal = binary.Read( memFile.Get() );
	binary.SetData( NULL, globals );
	return retVal;
}

/*
============
idLexer::LoadTokenStream
============
*/
bool idLexer::LoadTokenStream( const idList<unsigned short>& indices, const idTokenCache& tokens, const char* name ) {
	idLexer::filename = name;
	idLexer::filename.CollapsePath();
	idLexer::buffer = NULL;
	idLexer::fileTime = 0;
	idLexer::length = 0;
	// pointer in script buffer
	idLexer::script_p = NULL;

	idLexer::lastScript_p = NULL;
	// pointer to end of script buffer
	idLexer::end_p = NULL;

	idLexer::tokens.Clear();
	idLexer::line = 0;
	idLexer::lastline = 0;
	idLexer::allocated = false;
	idLexer::loaded = true;
	
	binary.Clear();
	binary.SetData( &indices, &tokens );

	return true;
}

/*
================
idLexer::FreeSource
================
*/
void idLexer::FreeSource( void ) {
#ifdef PUNCTABLE
	if ( idLexer::punctuationtable && idLexer::punctuationtable != default_punctuationtable ) {
		Mem_Free( (void *) idLexer::punctuationtable );
		idLexer::punctuationtable = NULL;
	}
	if ( idLexer::nextpunctuation && idLexer::nextpunctuation != default_nextpunctuation ) {
		Mem_Free( (void *) idLexer::nextpunctuation );
		idLexer::nextpunctuation = NULL;
	}
#endif //PUNCTABLE
	if ( idLexer::allocated ) {
		Mem_Free( (void *) idLexer::buffer );
		idLexer::buffer = NULL;
		idLexer::allocated = false;
	}
	idLexer::filename.Clear();
	idLexer::tokens.Clear();
	idLexer::loaded = false;
	binary.Clear();
}

/*
================
idLexer::idLexer
================
*/
idLexer::idLexer( void ) {
	idLexer::loaded = false;
	idLexer::filename = "";
	idLexer::flags = 0;
	idLexer::SetPunctuations( NULL );
	idLexer::allocated = false;
	idLexer::fileTime = 0;
	idLexer::length = 0;
	idLexer::line = 0;
	idLexer::lastline = 0;
	tokens.Clear();
	idLexer::next = NULL;
	idLexer::hadError = false;
	idLexer::hadWarning = false;
}

/*
================
idLexer::idLexer
================
*/
idLexer::idLexer( int flags ) {
	idLexer::loaded = false;
	idLexer::filename = "";
	this->flags = flags;
	idLexer::SetPunctuations( NULL );
	idLexer::allocated = false;
	idLexer::fileTime = 0;
	idLexer::length = 0;
	idLexer::line = 0;
	idLexer::lastline = 0;
	tokens.Clear();
	idLexer::next = NULL;
	idLexer::hadError = false;
	idLexer::hadWarning = false;
}

/*
================
idLexer::idLexer
================
*/
idLexer::idLexer( const char *filename, int flags, bool OSPath, int startLine ) {
	idLexer::loaded = false;
	this->flags = flags;
	idLexer::SetPunctuations( NULL );
	idLexer::allocated = false;
	tokens.Clear();
	idLexer::next = NULL;
	idLexer::hadError = false;
	idLexer::hadWarning = false;
	idLexer::LoadFile( filename, OSPath, startLine );
}

/*
================
idLexer::idLexer
================
*/
idLexer::idLexer( const char *ptr, int length, const char *name, int flags, int startLine ) {
	idLexer::loaded = false;
	this->flags = flags;
	idLexer::SetPunctuations( NULL );
	idLexer::allocated = false;
	tokens.Clear();
	idLexer::next = NULL;
	idLexer::hadError = false;
	idLexer::hadWarning = false;
	idLexer::LoadMemory( ptr, length, name, startLine );
}

/*
================
idLexer::~idLexer
================
*/
idLexer::~idLexer( void ) {
	idLexer::FreeSource();
}

/*
================
idLexer::SetBaseFolder
================
*/
void idLexer::SetBaseFolder( const char *path ) {
	idStr::Copynz( baseFolder, path, sizeof( baseFolder ) );
}

/*
================
idLexer::HadError
================
*/
bool idLexer::HadError( void ) const {
	return hadError;
}

/*
================
idLexer::HadWarning
================
*/
bool idLexer::HadWarning( void ) const {
	return hadWarning;
}