From b3a179b6918303870aece3e86da73e7895ba3883 Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Sun, 27 Sep 2020 11:36:13 +0200 Subject: [PATCH] - backported symbol table support for FScanner from Raze. --- src/common/engine/sc_man.cpp | 220 ++++++++++++++++++++++++++++----- src/common/engine/sc_man.h | 61 +++++++-- src/common/engine/serializer.h | 6 + 3 files changed, 246 insertions(+), 41 deletions(-) diff --git a/src/common/engine/sc_man.cpp b/src/common/engine/sc_man.cpp index 96c79538c..6cf0044c3 100644 --- a/src/common/engine/sc_man.cpp +++ b/src/common/engine/sc_man.cpp @@ -47,6 +47,7 @@ #include "templates.h" #include "zstring.h" #include "name.h" +#include #include "filesystem.h" // MACROS ------------------------------------------------------------------ @@ -300,6 +301,12 @@ void FScanner :: OpenLumpNum (int lump) void FScanner::PrepareScript () { + // If the file got a UTF-8 byte order mark, remove that. + if (ScriptBuffer.Len() > 3 && ScriptBuffer[0] == (char)0xEF && ScriptBuffer[1] == (char)0xBB && ScriptBuffer[2] == (char)0xBF) + { + ScriptBuffer = ScriptBuffer.Mid(3); + } + // The scanner requires the file to end with a '\n', so add one if // it doesn't already. if (ScriptBuffer.Len() == 0 || ScriptBuffer.Back() != '\n') @@ -400,6 +407,13 @@ void FScanner::RestorePos (const FScanner::SavedPos &pos) Crossed = false; } +long long FScanner::mystrtoll(const char* p, char** endp, int base) +{ + // Do not treat a leading 0 as an octal identifier if so desired. + if (NoOctals && *p == '0' && p[1] != 'x' && p[1] != 'X' && base == 0) base = 10; + return strtoll(p, endp, base); +} + //========================================================================== // // FScanner :: isText @@ -491,6 +505,7 @@ bool FScanner::ScanString (bool tokens) const char *marker, *tok; bool return_val; + ParseError = false; CheckOpen(); if (AlreadyGot) { @@ -590,7 +605,7 @@ bool FScanner::CheckString (const char *name) // //========================================================================== -bool FScanner::GetToken () +bool FScanner::GetToken (bool evaluate) { if (ScanString (true)) { @@ -608,7 +623,7 @@ bool FScanner::GetToken () } else { - BigNumber = strtoll(String, &stopper, 0); + BigNumber = mystrtoll(String, &stopper, 0); Number = (int)BigNumber;// clamp(BigNumber, 0, UINT_MAX); Float = Number; } @@ -620,7 +635,19 @@ bool FScanner::GetToken () } else if (TokenType == TK_StringConst) { - StringLen = strbin(String); + StringLen = strbin(const_cast(String)); + } + else if (TokenType == TK_Identifier && evaluate && symbols.CountUsed() > 0) + { + auto sym = symbols.CheckKey(String); + if (sym) + { + TokenType = sym->tokenType; + BigNumber = sym->Number; + Number = (int)sym->Number; + Float = sym->Float; + // String will retain the actual symbol name. + } } return true; } @@ -633,9 +660,9 @@ bool FScanner::GetToken () // //========================================================================== -void FScanner::MustGetAnyToken (void) +void FScanner::MustGetAnyToken (bool evaluate) { - if (GetToken () == false) + if (GetToken (evaluate) == false) { ScriptError ("Missing token (unexpected end of file)."); } @@ -663,9 +690,9 @@ void FScanner::TokenMustBe (int token) // //========================================================================== -void FScanner::MustGetToken (int token) +void FScanner::MustGetToken (int token, bool evaluate) { - MustGetAnyToken (); + MustGetAnyToken (evaluate); TokenMustBe(token); } @@ -678,9 +705,9 @@ void FScanner::MustGetToken (int token) // //========================================================================== -bool FScanner::CheckToken (int token) +bool FScanner::CheckToken (int token, bool evaluate) { - if (GetToken ()) + if (GetToken (evaluate)) { if (TokenType == token) { @@ -697,7 +724,7 @@ bool FScanner::CheckToken (int token) // //========================================================================== -bool FScanner::GetNumber () +bool FScanner::GetNumber (bool evaluate) { char *stopper; @@ -710,11 +737,25 @@ bool FScanner::GetNumber () } else { - BigNumber = strtoll(String, &stopper, 0); + BigNumber = mystrtoll(String, &stopper, 0); Number = (int)BigNumber;// clamp(BigNumber, 0, UINT_MAX); if (*stopper != 0) { + if (evaluate && symbols.CountUsed()) + { + auto sym = symbols.CheckKey(String); + if (sym && sym->tokenType == TK_IntConst) + { + BigNumber = sym->Number; + Number = (int)sym->Number; + Float = sym->Float; + // String will retain the actual symbol name. + return true; + } + + } ScriptError ("SC_GetNumber: Bad numeric constant \"%s\".", String); + return false; } } Float = Number; @@ -732,9 +773,9 @@ bool FScanner::GetNumber () // //========================================================================== -void FScanner::MustGetNumber () +void FScanner::MustGetNumber (bool evaluate) { - if (GetNumber() == false) + if (GetNumber(evaluate) == false) { ScriptError ("Missing integer (unexpected end of file)."); } @@ -749,7 +790,7 @@ void FScanner::MustGetNumber () // //========================================================================== -bool FScanner::CheckNumber () +bool FScanner::CheckNumber (bool evaluate) { char *stopper; @@ -767,10 +808,23 @@ bool FScanner::CheckNumber () } else { - BigNumber = strtoll (String, &stopper, 0); + BigNumber = mystrtoll (String, &stopper, 0); Number = (int)BigNumber;// clamp(BigNumber, 0, UINT_MAX); if (*stopper != 0) { + if (evaluate && symbols.CountUsed()) + { + auto sym = symbols.CheckKey(String); + if (sym && sym->tokenType == TK_IntConst) + { + BigNumber = sym->Number; + Number = (int)sym->Number; + Float = sym->Float; + // String will retain the actual symbol name. + return true; + } + + } UnGet(); return false; } @@ -792,7 +846,7 @@ bool FScanner::CheckNumber () // //========================================================================== -bool FScanner::CheckFloat () +bool FScanner::CheckFloat (bool evaluate) { char *stopper; @@ -807,6 +861,20 @@ bool FScanner::CheckFloat () Float = strtod (String, &stopper); if (*stopper != 0) { + if (evaluate && symbols.CountUsed()) + { + auto sym = symbols.CheckKey(String); + if (sym && sym->tokenType == TK_IntConst && sym->tokenType != TK_FloatConst) + { + BigNumber = sym->Number; + Number = (int)sym->Number; + Float = sym->Float; + // String will retain the actual symbol name. + return true; + } + + } + UnGet(); return false; } @@ -825,7 +893,7 @@ bool FScanner::CheckFloat () // //========================================================================== -bool FScanner::GetFloat () +bool FScanner::GetFloat (bool evaluate) { char *stopper; @@ -835,7 +903,21 @@ bool FScanner::GetFloat () Float = strtod (String, &stopper); if (*stopper != 0) { + if (evaluate && symbols.CountUsed()) + { + auto sym = symbols.CheckKey(String); + if (sym && sym->tokenType == TK_IntConst && sym->tokenType != TK_FloatConst) + { + BigNumber = sym->Number; + Number = (int)sym->Number; + Float = sym->Float; + // String will retain the actual symbol name. + return true; + } + } + ScriptError ("SC_GetFloat: Bad numeric constant \"%s\".", String); + return false; } Number = (int)Float; return true; @@ -852,9 +934,9 @@ bool FScanner::GetFloat () // //========================================================================== -void FScanner::MustGetFloat () +void FScanner::MustGetFloat (bool evaluate) { - if (GetFloat() == false) + if (GetFloat(evaluate) == false) { ScriptError ("Missing floating-point number (unexpected end of file)."); } @@ -938,44 +1020,54 @@ bool FScanner::Compare (const char *text) // //========================================================================== -bool FScanner::ScanValue(bool allowfloat) +bool FScanner::ScanValue(bool allowfloat, bool evaluate) { bool neg = false; - if (!GetToken()) + if (!GetToken(evaluate)) { return false; } if (TokenType == '-' || TokenType == '+') { neg = TokenType == '-'; - if (!GetToken()) + if (!GetToken(evaluate)) { return false; } } - if (TokenType != TK_IntConst && (TokenType != TK_FloatConst || !allowfloat)) - { + + if (TokenType == TK_FloatConst && !allowfloat) return false; + + if (TokenType != TK_IntConst && TokenType != TK_FloatConst) + { + auto d = constants.CheckKey(String); + if (!d) return false; + if (!allowfloat && int64_t(*d) != *d) return false; + BigNumber = *d; + Number = *d; + Float = *d; } if (neg) { + BigNumber = -BigNumber; Number = -Number; Float = -Float; } return true; } -bool FScanner::CheckValue(bool allowfloat) +bool FScanner::CheckValue(bool allowfloat, bool evaluate) { auto savedstate = SavePos(); - bool res = ScanValue(allowfloat); + bool res = ScanValue(allowfloat, evaluate); if (!res) RestorePos(savedstate); return res; } -void FScanner::MustGetValue(bool allowfloat) +void FScanner::MustGetValue(bool allowfloat, bool evaluate) { - if (!ScanValue(allowfloat)) ScriptError(allowfloat ? "Numeric constant expected" : "Integer constant expected"); + if (!ScanValue(allowfloat, evaluate)) ScriptError(allowfloat ? "Numeric constant expected" : "Integer constant expected"); } bool FScanner::CheckBoolToken() @@ -1079,6 +1171,12 @@ void FScanner::ScriptError (const char *message, ...) va_end (arglist); } + if (NoFatalErrors) + { + Printf(TEXTCOLOR_RED "Script error, \"%s\"" TEXTCOLOR_RED " line %d:\n" TEXTCOLOR_RED "%s\n", ScriptName.GetChars(), + AlreadyGot ? AlreadyGotLine : Line, composed.GetChars()); + return; + } I_Error ("Script error, \"%s\" line %d:\n%s\n", ScriptName.GetChars(), AlreadyGot? AlreadyGotLine : Line, composed.GetChars()); } @@ -1123,6 +1221,72 @@ void FScanner::CheckOpen() } } +//========================================================================== +// +// +// +//========================================================================== + +void FScanner::AddSymbol(const char *name, int64_t value) +{ + Symbol sym; + sym.tokenType = TK_IntConst; + sym.Number = value; + sym.Float = value; + symbols.Insert(name, sym); +} + +//========================================================================== +// +// +// +//========================================================================== + +void FScanner::AddSymbol(const char* name, uint64_t value) +{ + Symbol sym; + sym.tokenType = TK_UIntConst; + sym.Number = value; + sym.Float = value; + symbols.Insert(name, sym); +} + +//========================================================================== +// +// +// +//========================================================================== + +void FScanner::SkipToEndOfBlock() +{ + int depth = 0; + while (1) + { + MustGetString(); // this will abort if it reaches the end of the file + if (Compare("{")) depth++; + else if (Compare("}")) + { + depth--; + if (depth < 0) return; + } + } +} + +//========================================================================== +// +// +// +//========================================================================== + +void FScanner::AddSymbol(const char* name, double value) +{ + Symbol sym; + sym.tokenType = TK_FloatConst; + sym.Number = (int64_t)value; + sym.Float = value; + symbols.Insert(name, sym); +} + //========================================================================== // // a class that remembers a parser position diff --git a/src/common/engine/sc_man.h b/src/common/engine/sc_man.h index 684482e6c..6c37806a8 100644 --- a/src/common/engine/sc_man.h +++ b/src/common/engine/sc_man.h @@ -2,6 +2,7 @@ #define __SC_MAN_H__ #include "zstring.h" +#include "tarray.h" #include "name.h" #include "basics.h" @@ -46,6 +47,16 @@ public: int SavedScriptLine; }; + struct Symbol + { + int tokenType; + int64_t Number; + double Float; + }; + + + TMap symbols; + // Methods ------------------------------------------------------ FScanner(); FScanner(const FScanner &other); @@ -70,11 +81,19 @@ public: } void SetCMode(bool cmode); + void SetNoOctals(bool cmode) { NoOctals = cmode; } + void SetNoFatalErrors(bool cmode) { NoFatalErrors = cmode; } void SetEscape(bool esc); void SetStateMode(bool stately); void DisableStateOptions(); const SavedPos SavePos(); void RestorePos(const SavedPos &pos); + void AddSymbol(const char* name, int64_t value); + void AddSymbol(const char* name, uint64_t value); + inline void AddSymbol(const char* name, int32_t value) { return AddSymbol(name, int64_t(value)); } + inline void AddSymbol(const char* name, uint32_t value) { return AddSymbol(name, uint64_t(value)); } + void AddSymbol(const char* name, double value); + void SkipToEndOfBlock(); static FString TokenName(int token, const char *string=NULL); @@ -83,30 +102,40 @@ public: void MustGetStringName(const char *name); bool CheckString(const char *name); - bool GetToken(); - void MustGetAnyToken(); + bool GetToken(bool evaluate = false); + void MustGetAnyToken(bool evaluate = false); void TokenMustBe(int token); - void MustGetToken(int token); - bool CheckToken(int token); + void MustGetToken(int token, bool evaluate = false); + bool CheckToken(int token, bool evaluate = false); bool CheckTokenId(ENamedName id); - bool GetNumber(); - void MustGetNumber(); - bool CheckNumber(); + bool GetNumber(bool evaluate = false); + void MustGetNumber(bool evaluate = false); + bool CheckNumber(bool evaluate = false); - bool GetFloat(); - void MustGetFloat(); - bool CheckFloat(); + bool GetFloat(bool evaluate = false); + void MustGetFloat(bool evaluate = false); + bool CheckFloat(bool evaluate = false); + + double *LookupConstant(FName name) + { + return constants.CheckKey(name); + } // Token based variant - bool CheckValue(bool allowfloat); - void MustGetValue(bool allowfloat); + bool CheckValue(bool allowfloat, bool evaluate = true); + void MustGetValue(bool allowfloat, bool evaluate = true); bool CheckBoolToken(); void MustGetBoolToken(); void UnGet(); bool Compare(const char *text); + inline bool Compare(const std::initializer_list& list) + { + for (auto c : list) if (Compare(c)) return true; + return false; + } int MatchString(const char * const *strings, size_t stride = sizeof(char*)); int MustMatchString(const char * const *strings, size_t stride = sizeof(char*)); int GetMessageLine(); @@ -125,11 +154,13 @@ public: double Float; int Line; bool End; + bool ParseError = false; bool Crossed; int LumpNum; FString ScriptName; protected: + long long mystrtoll(const char* p, char** endp, int base); void PrepareScript(); void CheckOpen(); bool ScanString(bool tokens); @@ -137,6 +168,8 @@ protected: // Strings longer than this minus one will be dynamically allocated. static const int MAX_STRING_SIZE = 128; + TMap constants; + bool ScriptOpen; FString ScriptBuffer; const char *ScriptPtr; @@ -149,13 +182,15 @@ protected: const char *LastGotPtr; int LastGotLine; bool CMode; + bool NoOctals = false; + bool NoFatalErrors = false; uint8_t StateMode; bool StateOptions; bool Escape; VersionInfo ParseVersion = { 0, 0, 0 }; // no ZScript extensions by default - bool ScanValue(bool allowfloat); + bool ScanValue(bool allowfloat, bool evaluate); }; enum diff --git a/src/common/engine/serializer.h b/src/common/engine/serializer.h index 95d184aec..b25aeeb0f 100644 --- a/src/common/engine/serializer.h +++ b/src/common/engine/serializer.h @@ -240,6 +240,12 @@ FSerializer &Serialize(FSerializer &arc, const char *key, TArray &value, return arc; } +template +FSerializer& Serialize(FSerializer& arc, const char* key, FixedBitArray& value, FixedBitArray* def) +{ + return arc.SerializeMemory(key, value.Storage(), value.StorageSize()); +} + template<> FSerializer& Serialize(FSerializer& arc, const char* key, PClass*& clst, PClass** def); template<> FSerializer& Serialize(FSerializer& arc, const char* key, FFont*& font, FFont** def); template<> FSerializer &Serialize(FSerializer &arc, const char *key, Dictionary *&dict, Dictionary **def);