- started to make ZScript parser work on real-life data. It will now read a lump named ZSCRIPT which is a list of filenames to be compiled.

- place generated symbols into GlobalSymbols instead of a scratch table that will be discarded right away.
- allow the state object to change source file scanners (I hope this works, but the old implementation was unable to do more than one with with a parse state so I had to change it.)
- It can now parse constants.txt and insert everything in it into the global symbol table and make subsequent DECORATE compile properly.

src/thingdef/thingdef.cpp

@@ -445,6 +445,7 @@ static void FinishThingdef()
 // Called from FActor::StaticInit()
 //
 //==========================================================================
+void ParseScripts();
 
 void LoadActors ()
 {
@@ -456,19 +457,10 @@ void LoadActors ()
 	FScriptPosition::ResetErrorCounter();
 	InitThingdef();
 	lastlump = 0;
+	ParseScripts();
 	while ((lump = Wads.FindLump ("DECORATE", &lastlump)) != -1)
 	{
 		FScanner sc(lump);
-
-		if (Wads.GetLumpFile(lump) == 0)
-		{
-			// define namespace 'zdoom'
-		}
-		else
-		{
-			// use namespace 'zdoom'
-		}
-
 		ParseDecorate (sc);
 	}
 	FinishThingdef();

src/zscript/zcc-parse.lemon

@@ -96,9 +96,9 @@ static void SetNodeLine(ZCC_TreeNode *name, int line)
 			}
 		}
 	}
-	stat->sc.ScriptMessage("%s\n%s\n", unexpected.GetChars(), expecting.GetChars());
+	stat->sc->ScriptMessage("%s\n%s\n", unexpected.GetChars(), expecting.GetChars());
 }
-%parse_accept { stat->sc.ScriptMessage("input accepted\n"); }
+%parse_accept { DPrintf(DMSG_SPAMMY, "Input accepted\n"); }
 %parse_failure { /**failed = true;*/ }
 
 %nonassoc EQ MULEQ DIVEQ MODEQ ADDEQ SUBEQ LSHEQ RSHEQ ANDEQ OREQ XOREQ.
@@ -124,7 +124,7 @@ static void SetNodeLine(ZCC_TreeNode *name, int line)
 %type opt_func_body {ZCC_CompoundStmt *}
 %type function_body {ZCC_CompoundStmt *}
 
-main ::= translation_unit(A). { stat->TopNode = A; stat->sc.ScriptMessage("Parse complete\n"); }
+main ::= translation_unit(A). { stat->TopNode = A; DPrintf(DMSG_SPAMMY, "Parse complete\n"); }
 
 %type translation_unit {ZCC_TreeNode *}
 translation_unit(X) ::= .												{ X = NULL; }
@@ -339,7 +339,7 @@ enum_def(X) ::= ENUM(T) IDENTIFIER(A) enum_type(B) LBRACE opt_enum_list(C) RBRAC
 	X = def;
 }
 
-enum_type(X) ::= .										{ X.Int = ZCC_IntAuto; X.SourceLoc = stat->sc.GetMessageLine(); }
+enum_type(X) ::= .										{ X.Int = ZCC_IntAuto; X.SourceLoc = stat->sc->GetMessageLine(); }
 enum_type(X) ::= COLON int_type(A).						{ X = A; /*X-overwrites-A*/ }
 
 enum_list(X) ::= error.									{ X = NULL; }
@@ -398,7 +398,7 @@ states_def(X) ::= STATES(T) scanner_mode LBRACE states_body(A) RBRACE.
  * set immediately after LBRACE is consumed, rather than immediately after
  * STATES is consumed.
  */
-scanner_mode ::= . { stat->sc.SetStateMode(true); }
+scanner_mode ::= . { stat->sc->SetStateMode(true); }
 
 states_body(X) ::= .								{ X = NULL; }
 states_body(X) ::= error.							{ X = NULL; }
@@ -538,7 +538,7 @@ type_name(X) ::= DOT dottable_id(A).
  * straight away.)
  */
 %token_class intconst INTCONST|UINTCONST.
-vector_size(X) ::= .						{ X.Int = ZCC_Vector3; X.SourceLoc = stat->sc.GetMessageLine(); }
+vector_size(X) ::= .						{ X.Int = ZCC_Vector3; X.SourceLoc = stat->sc->GetMessageLine(); }
 vector_size(X) ::= LT intconst(A) GT.
 {
 	if (A.Int >= 2 && A.Int <= 4)
@@ -548,7 +548,7 @@ vector_size(X) ::= LT intconst(A) GT.
 	else
 	{
 		X.Int = ZCC_Vector3;
-		stat->sc.ScriptMessage("Invalid vector size %d\n", A.Int);
+		stat->sc->ScriptMessage("Invalid vector size %d\n", A.Int);
 	}
 	X.SourceLoc = A.SourceLoc;
 }
@@ -656,11 +656,11 @@ declarator(X) ::= decl_flags(A) type_list_or_void(B) variables_or_function(C).
 	{ // An invalid
 		if (B == NULL)
 		{
-			stat->sc.ScriptMessage("Variables may not be of type void.\n");
+			stat->sc->ScriptMessage("Variables may not be of type void.\n");
 		}
 		else
 		{
-			stat->sc.ScriptMessage("Variables may be of only one type.\n");
+			stat->sc->ScriptMessage("Variables may be of only one type.\n");
 		}
 		X = NULL;
 	}
@@ -750,7 +750,7 @@ decl_flags(X) ::= decl_flags(A) ACTION(T).		{ X.Int = A.Int | ZCC_Action; X.Sour
 decl_flags(X) ::= decl_flags(A) READONLY(T).	{ X.Int = A.Int | ZCC_ReadOnly; X.SourceLoc = A.SourceLoc ? A.SourceLoc : T.SourceLoc; }
 decl_flags(X) ::= decl_flags(A) DEPRECATED(T).	{ X.Int = A.Int | ZCC_Deprecated; X.SourceLoc = A.SourceLoc ? A.SourceLoc : T.SourceLoc; }
 
-func_const(X) ::= .							{ X.Int = 0; X.SourceLoc = stat->sc.GetMessageLine(); }
+func_const(X) ::= .							{ X.Int = 0; X.SourceLoc = stat->sc->GetMessageLine(); }
 func_const(X) ::= CONST(T).					{ X.Int = ZCC_FuncConst; X.SourceLoc = T.SourceLoc; }
 
 opt_func_body(X) ::= SEMICOLON.				{ X = NULL; }

src/zscript/zcc_parser.cpp

@@ -5,6 +5,7 @@
 #include "w_wad.h"
 #include "cmdlib.h"
 #include "m_alloc.h"
+#include "i_system.h"
 #include "zcc_parser.h"
 #include "zcc_compile.h"
 
@@ -161,44 +162,26 @@ static void InitTokenMap()
 #undef TOKENDEF
 #undef TOKENDEF2
 
-static void DoParse(const char *filename)
+static void ParseSingleFile(const char *filename, void *parser, ZCCParseState &state)
 {
-	if (TokenMap.CountUsed() == 0)
-	{
-		InitTokenMap();
-	}
-
-	FScanner sc;
-	void *parser;
 	int tokentype;
 	int lump;
-	bool failed;
+	//bool failed;
 	ZCCToken value;
+	FScanner sc;
 
 	lump = Wads.CheckNumForFullName(filename, true);
 	if (lump >= 0)
 	{
 		sc.OpenLumpNum(lump);
 	}
-	else if (FileExists(filename))
-	{
-		sc.OpenFile(filename);
-	}
 	else
 	{
 		Printf("Could not find script lump '%s'\n", filename);
 		return;
 	}
 
-	parser = ZCCParseAlloc(malloc);
-	failed = false;
-#ifdef _DEBUG
-	FILE *f = fopen("trace.txt", "w");
-	char prompt = '\0';
-	ZCCParseTrace(f, &prompt);
-#endif
-	ZCCParseState state(sc);
-
+	state.sc = ≻
 	while (sc.GetToken())
 	{
 		value.SourceLoc = sc.GetMessageLine();
@@ -241,7 +224,7 @@ static void DoParse(const char *filename)
 
 		default:
 			TokenMapEntry *zcctoken = TokenMap.CheckKey(sc.TokenType);
-			if (zcctoken != NULL)
+			if (zcctoken != nullptr)
 			{
 				tokentype = zcctoken->TokenType;
 				value.Int = zcctoken->TokenName;
@@ -254,20 +237,55 @@ static void DoParse(const char *filename)
 			break;
 		}
 		ZCCParse(parser, tokentype, value, &state);
-		if (failed)
-		{
-			sc.ScriptMessage("Parse failed\n");
-			goto parse_end;
-		}
 	}
 parse_end:
 	value.Int = -1;
 	ZCCParse(parser, ZCC_EOF, value, &state);
+	state.sc = nullptr;
+}
+
+static void DoParse(int lumpnum)
+{
+	if (TokenMap.CountUsed() == 0)
+	{
+		InitTokenMap();
+	}
+
+	FScanner sc;
+	void *parser;
+	ZCCToken value;
+
+	parser = ZCCParseAlloc(malloc);
+	ZCCParseState state;
+#ifdef _DEBUG
+	FILE *f = fopen("trace.txt", "w");
+	char prompt = '\0';
+	ZCCParseTrace(f, &prompt);
+#endif
+
+	sc.OpenLumpNum(lumpnum);
+	// parse all files from this list in one go.
+	while (sc.GetString())
+	{
+		if (Wads.GetLumpFile(sc.LumpNum) == 0)
+		{
+			int includefile = Wads.GetLumpFile(Wads.CheckNumForFullName(sc.String, true));
+			if (includefile != 0)
+			{
+				I_FatalError("File %s is overriding core lump %s.",
+					Wads.GetWadFullName(includefile), sc.String);
+			}
+		}
+
+		ParseSingleFile(sc.String, parser, state);
+	}
+
+	value.Int = -1;
+	value.SourceLoc = sc.GetMessageLine();
 	ZCCParse(parser, 0, value, &state);
 	ZCCParseFree(parser, free);
 
-	PSymbolTable symbols(&GlobalSymbols);
-	ZCCCompiler cc(state, NULL, symbols);
+	ZCCCompiler cc(state, NULL, GlobalSymbols);
 	cc.Compile();
 #ifdef _DEBUG
 	if (f != NULL)
@@ -275,6 +293,7 @@ parse_end:
 		fclose(f);
 	}
 	FString ast = ZCC_PrintAST(state.TopNode);
+	FString filename = Wads.GetLumpFullName(lumpnum);
 	FString astfile = ExtractFileBase(filename, false);
 	astfile << ".ast";
 	f = fopen(astfile, "w");
@@ -286,6 +305,16 @@ parse_end:
 #endif
 }
 
+void ParseScripts()
+{
+	int lump, lastlump = 0;
+	while ((lump = Wads.FindLump("ZSCRIPT", &lastlump)) != -1)
+	{
+		DoParse(lump);
+	}
+}
+
+/*
 CCMD(parse)
 {
 	if (argv.argc() == 2)
@@ -293,6 +322,7 @@ CCMD(parse)
 		DoParse(argv[1]);
 	}
 }
+*/
 
 static FString ZCCTokenName(int terminal)
 {
@@ -333,6 +363,6 @@ ZCC_TreeNode *ZCC_AST::InitNode(size_t size, EZCCTreeNodeType type, ZCC_TreeNode
 ZCC_TreeNode *ZCCParseState::InitNode(size_t size, EZCCTreeNodeType type)
 {
 	ZCC_TreeNode *node = ZCC_AST::InitNode(size, type, NULL);
-	node->SourceName = Strings.Alloc(sc.ScriptName);
+	node->SourceName = Strings.Alloc(sc->ScriptName);
 	return node;
 }

src/zscript/zcc_parser.h

@@ -514,10 +514,10 @@ struct ZCC_AST
 
 struct ZCCParseState : public ZCC_AST
 {
-	ZCCParseState(FScanner &scanner) : sc(scanner) {}
+	ZCCParseState(FScanner *scanner = nullptr) : sc(scanner) {}
 	ZCC_TreeNode *InitNode(size_t size, EZCCTreeNodeType type);
 
-	FScanner ≻
+	FScanner *sc;
 };
 
 #endif