worldspawn/libs/script/scripttokeniser.h

/*
   Copyright (C) 2001-2006, William Joseph.
   All Rights Reserved.

   This file is part of GtkRadiant.

   GtkRadiant is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   GtkRadiant is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with GtkRadiant; if not, write to the Free Software
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#if !defined( INCLUDED_SCRIPT_SCRIPTTOKENISER_H )
#define INCLUDED_SCRIPT_SCRIPTTOKENISER_H

#include "iscriplib.h"

class ScriptTokeniser : public Tokeniser
{
enum CharType
{
	eWhitespace,
	eCharToken,
	eNewline,
	eCharQuote,
	eCharSolidus,
	eCharStar,
	eCharSpecial,
};

typedef bool ( ScriptTokeniser::*Tokenise )( char c );

Tokenise m_stack[3];
Tokenise* m_state;
SingleCharacterInputStream<TextInputStream> m_istream;
std::size_t m_scriptline;
std::size_t m_scriptcolumn;

char m_token[MAXTOKEN];
char* m_write;

char m_current;
bool m_eof;
bool m_crossline;
bool m_unget;
bool m_emit;

bool m_special;

CharType charType( const char c ){
	switch ( c )
	{
	case '\n': return eNewline;
	case '"': return eCharQuote;
	case '/': return eCharSolidus;
	case '*': return eCharStar;
	case '{': case '(': case '}': case ')': case '[': case ']': case ',': case ':': return ( m_special ) ? eCharSpecial : eCharToken;
	}

	if ( c > 32 ) {
		return eCharToken;
	}
	return eWhitespace;
}

Tokenise state(){
	return *m_state;
}
void push( Tokenise state ){
	ASSERT_MESSAGE( m_state != m_stack + 2, "token parser: illegal stack push" );
	*( ++m_state ) = state;
}
void pop(){
	ASSERT_MESSAGE( m_state != m_stack, "token parser: illegal stack pop" );
	--m_state;
}
void add( const char c ){
	if ( m_write < m_token + MAXTOKEN - 1 ) {
		*m_write++ = c;
	}
}
void remove(){
	ASSERT_MESSAGE( m_write > m_token, "no char to remove" );
	--m_write;
}

bool tokeniseDefault( char c ){
	switch ( charType( c ) )
	{
	case eNewline:
		if ( !m_crossline ) {
			globalErrorStream() << Unsigned( getLine() ) << ":" << Unsigned( getColumn() ) << ": unexpected end-of-line before token\n";
			return false;
		}
		break;
	case eCharToken:
	case eCharStar:
		push( Tokenise( &ScriptTokeniser::tokeniseToken ) );
		add( c );
		break;
	case eCharSpecial:
		push( Tokenise( &ScriptTokeniser::tokeniseSpecial ) );
		add( c );
		break;
	case eCharQuote:
		push( Tokenise( &ScriptTokeniser::tokeniseQuotedToken ) );
		break;
	case eCharSolidus:
		push( Tokenise( &ScriptTokeniser::tokeniseSolidus ) );
		break;
	default:
		break;
	}
	return true;
}
bool tokeniseToken( char c ){
	switch ( charType( c ) )
	{
	case eNewline:
	case eWhitespace:
	case eCharQuote:
	case eCharSpecial:
		pop();
		m_emit = true; // emit token
		break;
	case eCharSolidus:
#if 0 //SPoG: ignore comments in the middle of tokens.
		push( Tokenise( &ScriptTokeniser::tokeniseSolidus ) );
		break;
#endif
	case eCharToken:
	case eCharStar:
		add( c );
		break;
	default:
		break;
	}
	return true;
}
bool tokeniseQuotedToken( char c ){
	switch ( charType( c ) )
	{
	case eNewline:
		if ( m_crossline ) {
			globalErrorStream() << Unsigned( getLine() ) << ":" << Unsigned( getColumn() ) << ": unexpected end-of-line in quoted token\n";
			return false;
		}
		break;
	case eWhitespace:
	case eCharToken:
	case eCharSolidus:
	case eCharStar:
	case eCharSpecial:
		add( c );
		break;
	case eCharQuote:
		pop();
		push( Tokenise( &ScriptTokeniser::tokeniseEndQuote ) );
		break;
	default:
		break;
	}
	return true;
}
bool tokeniseSolidus( char c ){
	switch ( charType( c ) )
	{
	case eNewline:
	case eWhitespace:
	case eCharQuote:
	case eCharSpecial:
		pop();
		add( '/' );
		m_emit = true; // emit single slash
		break;
	case eCharToken:
		pop();
		add( '/' );
		add( c );
		break;
	case eCharSolidus:
		pop();
		push( Tokenise( &ScriptTokeniser::tokeniseComment ) );
		break; // dont emit single slash
	case eCharStar:
		pop();
		push( Tokenise( &ScriptTokeniser::tokeniseBlockComment ) );
		break; // dont emit single slash
	default:
		break;
	}
	return true;
}
bool tokeniseComment( char c ){
	if ( c == '\n' ) {
		pop();
		if ( state() == Tokenise( &ScriptTokeniser::tokeniseToken ) ) {
			pop();
			m_emit = true; // emit token immediatly preceding comment
		}
	}
	return true;
}
bool tokeniseBlockComment( char c ){
	if ( c == '*' ) {
		pop();
		push( Tokenise( &ScriptTokeniser::tokeniseEndBlockComment ) );
	}
	return true;
}
bool tokeniseEndBlockComment( char c ){
	switch ( c )
	{
	case '/':
		pop();
		if ( state() == Tokenise( &ScriptTokeniser::tokeniseToken ) ) {
			pop();
			m_emit = true; // emit token immediatly preceding comment
		}
		break; // dont emit comment
	case '*':
		break; // no state change
	default:
		pop();
		push( Tokenise( &ScriptTokeniser::tokeniseBlockComment ) );
		break;
	}
	return true;
}
bool tokeniseEndQuote( char c ){
	pop();
	m_emit = true; // emit quoted token
	return true;
}
bool tokeniseSpecial( char c ){
	pop();
	m_emit = true; // emit single-character token
	return true;
}

/// Returns true if a token was successfully parsed.
bool tokenise(){
	m_write = m_token;
	while ( !eof() )
	{
		char c = m_current;

		if ( !( ( *this ).*state() )( c ) ) {
			// parse error
			m_eof = true;
			return false;
		}
		if ( m_emit ) {
			m_emit = false;
			return true;
		}

		if ( c == '\n' ) {
			++m_scriptline;
			m_scriptcolumn = 1;
		}
		else
		{
			++m_scriptcolumn;
		}

		m_eof = !m_istream.readChar( m_current );
	}
	return m_write != m_token;
}

const char* fillToken(){
	if ( !tokenise() ) {
		return 0;
	}

	add( '\0' );
	return m_token;
}

bool eof(){
	return m_eof;
}

public:
ScriptTokeniser( TextInputStream& istream, bool special )
	: m_state( m_stack ),
	m_istream( istream ),
	m_scriptline( 1 ),
	m_scriptcolumn( 1 ),
	m_crossline( false ),
	m_unget( false ),
	m_emit( false ),
	m_special( special ){
	m_stack[0] = Tokenise( &ScriptTokeniser::tokeniseDefault );
	m_eof = !m_istream.readChar( m_current );
	m_token[MAXTOKEN - 1] = '\0';
}
void release(){
	delete this;
}
void nextLine(){
	m_crossline = true;
}
const char* getToken(){
	if ( m_unget ) {
		m_unget = false;
		return m_token;
	}

	return fillToken();
}
void ungetToken(){
	ASSERT_MESSAGE( !m_unget, "can't unget more than one token" );
	m_unget = true;
}
std::size_t getLine() const {
	return m_scriptline;
}
std::size_t getColumn() const {
	return m_scriptcolumn;
}
};


inline Tokeniser& NewScriptTokeniser( TextInputStream& istream ){
	return *( new ScriptTokeniser( istream, true ) );
}

inline Tokeniser& NewSimpleTokeniser( TextInputStream& istream ){
	return *( new ScriptTokeniser( istream, false ) );
}

#endif