gtkradiant/tools/urt/libs/script/scripttokeniser.h

#if !defined(INCLUDED_SCRIPT_SCRIPTTOKENISER_H)
#define INCLUDED_SCRIPT_SCRIPTTOKENISER_H

#include "iscriplib.h"

class ScriptTokeniser : public Tokeniser
{
  enum CharType
  {
    eWhitespace,
    eCharToken,
    eNewline,
    eCharQuote,
    eCharSolidus,
    eCharStar,
    eCharSpecial,
  };

  typedef bool (ScriptTokeniser::*Tokenise)(char c);

  Tokenise m_stack[3];
  Tokenise* m_state;
  SingleCharacterInputStream<TextInputStream> m_istream;
  std::size_t m_scriptline;

  char m_token[MAXTOKEN];
  char* m_write;

  char m_current;
  bool m_eof;
  bool m_crossline;
  bool m_unget;

  bool m_special;

  CharType charType(const char c)
  {
    switch(c)
    {
    case '\n': return eNewline;
    case '"': return eCharQuote;
    case '/': return eCharSolidus;
    case '*': return eCharStar;
    case '{': case '(': case '}': case ')': case '[': case ']': case ',': case ':': return (m_special) ? eCharSpecial : eCharToken;
    }
    
    if(c > 32)
    {
      return eCharToken;
    }
    return eWhitespace;
  }

  Tokenise state()
  {
    return *m_state;
  }
  void push(Tokenise state)
  {
    ASSERT_MESSAGE(m_state != m_stack + 2, "token parser: illegal stack push");
    *(++m_state) = state;
  }
  void pop()
  {
    ASSERT_MESSAGE(m_state != m_stack, "token parser: illegal stack pop");
    --m_state;
  }
  void add(const char c)
  {
    if(m_write < m_token + MAXTOKEN - 1)
    {
      *m_write++ = c;
    }
  }
  void remove()
  {
    ASSERT_MESSAGE(m_write > m_token, "no char to remove");
    --m_write;
  }

  bool tokeniseDefault(char c)
  {
    switch(charType(c))
    {
    case eNewline:
      ASSERT_MESSAGE(m_crossline, "unexpected end-of-line before token");
      break;
    case eCharToken:
    case eCharStar:
      push(Tokenise(&ScriptTokeniser::tokeniseToken));
      add(c);
      break;
    case eCharSpecial:
      push(Tokenise(&ScriptTokeniser::tokeniseSpecial));
      add(c);
      break;
    case eCharQuote:
      push(Tokenise(&ScriptTokeniser::tokeniseQuotedToken));
      break;
    case eCharSolidus:
      push(Tokenise(&ScriptTokeniser::tokeniseSolidus));
      break;
    default:
      break;
    }
    return false;
  }
  bool tokeniseToken(char c)
  {
    switch(charType(c))
    {
    case eNewline:
    case eWhitespace:
    case eCharQuote:
    case eCharSpecial:
      pop();
      return true; // emit token
    case eCharSolidus:
      push(Tokenise(&ScriptTokeniser::tokeniseSolidus));
      break;
    case eCharToken:
    case eCharStar:
      add(c);
      break;
    default:
      break;
    }
    return false;
  }
  bool tokeniseQuotedToken(char c)
  {
    switch(charType(c))
    {
    case eNewline:
      ASSERT_MESSAGE(!m_crossline, "unexpected end-of-line in quoted token");
      break;
    case eWhitespace:
    case eCharToken:
    case eCharSolidus:
    case eCharStar:
    case eCharSpecial:
      add(c);
      break;
    case eCharQuote:
      pop();
      push(Tokenise(&ScriptTokeniser::tokeniseEndQuote));
      break;
    default:
      break;
    }
    return false;
  }
  bool tokeniseSolidus(char c)
  {
    switch(charType(c))
    {
    case eNewline:
    case eWhitespace:
    case eCharQuote:
    case eCharSpecial:
      pop();
      add('/');
      return true; // emit single slash
    case eCharToken:
      pop();
      add('/');
      add(c);
      break;
    case eCharSolidus:
      pop();
      push(Tokenise(&ScriptTokeniser::tokeniseComment));
      break; // dont emit single slash
    case eCharStar:
      pop();
      push(Tokenise(&ScriptTokeniser::tokeniseBlockComment));
      break; // dont emit single slash
    default:
      break;
    }
    return false;
  }
  bool tokeniseComment(char c)
  {
    if(c == '\n')
    {
      pop();
      if(state() == Tokenise(&ScriptTokeniser::tokeniseToken))
      {
        pop();
        return true; // emit token immediatly preceding comment
      }
    }
    return false;
  }
  bool tokeniseBlockComment(char c)
  {
    if(c == '*')
    {
      pop();
      push(Tokenise(&ScriptTokeniser::tokeniseEndBlockComment));
    }
    return false;
  }
  bool tokeniseEndBlockComment(char c)
  {
    switch(c)
    {
    case '/':
      pop();
      if(state() == Tokenise(&ScriptTokeniser::tokeniseToken))
      {
        pop();
        return true; // emit token immediatly preceding comment
      }
      break; // dont emit comment
    case '*':
      break; // no state change
    default:
      pop();
      push(Tokenise(&ScriptTokeniser::tokeniseBlockComment));
      break;
    }
    return false;
  }
  bool tokeniseEndQuote(char c)
  {
    pop();
    return true; // emit quoted token
  }
  bool tokeniseSpecial(char c)
  {
    pop();
    return true; // emit single-character token
  }

  void tokenise()
  {
    m_write = m_token;
    while(!eof())
    {
      char c = m_current;

      if(c == '\n')
      {
        ++m_scriptline;
      }

      if(((*this).*state())(c))
      {
        return;
      }

      m_eof = !m_istream.readChar(m_current);
    }
  }

  const char* fillToken()
  {
    if(eof())
    {
      return 0;
    }

    tokenise();

    if(eof() && m_write == m_token)
    {
      return 0;
    }

    add('\0');
    return m_token;
  }

  bool eof()
  {
    return m_eof;
  }

public:
  ScriptTokeniser(TextInputStream& istream, bool special)
    : m_state(m_stack),
    m_istream(istream),
    m_scriptline(0),
    m_crossline(false),
    m_unget(false),
    m_special(special)
  {
    m_stack[0] = Tokenise(&ScriptTokeniser::tokeniseDefault);
    m_eof = !m_istream.readChar(m_current);
    m_token[MAXTOKEN - 1] = '\0';
  }
  void release()
  {
    delete this;
  }
  void nextLine()
  {
    m_crossline = true;
  }
  const char* getToken()
  {
    if(m_unget)
    {
      m_unget = false;
      return m_token;
    }
    
    return fillToken();
  }
  void ungetToken()
  {
    ASSERT_MESSAGE(!m_unget, "can't unget more than one token");
    m_unget = true;
  }
  std::size_t getLine() const
  {
    return m_scriptline;
  }
};


inline Tokeniser& NewScriptTokeniser(TextInputStream& istream)
{
  return *(new ScriptTokeniser(istream, true));
}

inline Tokeniser& NewSimpleTokeniser(TextInputStream& istream)
{
  return *(new ScriptTokeniser(istream, false));
}

#endif