From f1d2c7755a233d76a8a2f8ebd5cdbaa9ffbebc96 Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Sun, 20 Mar 2016 02:08:05 +0100 Subject: [PATCH 01/47] - fixed z-offset handling of A_SkelMissile. I can't believe I missed this for more than 10 years, considering that A_CustomMissile explicitly implements this case: It makes a crucial difference whether P_SpawnMissileZ is used or the actual z-position is temporarily changed. Reverted this function to the position changing method of the original. --- src/g_doom/a_revenant.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/g_doom/a_revenant.cpp b/src/g_doom/a_revenant.cpp index 252a1c1cd..1b36d6cb2 100644 --- a/src/g_doom/a_revenant.cpp +++ b/src/g_doom/a_revenant.cpp @@ -28,8 +28,9 @@ DEFINE_ACTION_FUNCTION(AActor, A_SkelMissile) return 0; A_FaceTarget (self); - missile = P_SpawnMissileZ (self, self->Z() + 48*FRACUNIT, - self->target, PClass::FindActor("RevenantTracer")); + self->AddZ(16*FRACUNIT); + missile = P_SpawnMissile (self, self->target, PClass::FindActor("RevenantTracer")); + self->AddZ(-16*FRACUNIT); if (missile != NULL) { From 288f01a0c2f43e03a7e9fed5588dadac1a97ea57 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 11:34:41 -0500 Subject: [PATCH 02/47] GCC fixes for zcc_compile.(cpp|h) --- src/zscript/zcc_compile.cpp | 1 + src/zscript/zcc_compile.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zscript/zcc_compile.cpp b/src/zscript/zcc_compile.cpp index 037e4fa42..b7bfe81ba 100644 --- a/src/zscript/zcc_compile.cpp +++ b/src/zscript/zcc_compile.cpp @@ -39,6 +39,7 @@ ZCCCompiler::ZCCCompiler(ZCC_AST &ast, DObject *_outer, PSymbolTable &_symbols) case AST_Class: Classes.Push(static_cast(node)); break; case AST_Struct: Structs.Push(static_cast(node)); break; case AST_ConstantDef: Constants.Push(static_cast(node)); break; + default: assert(0 && "Default case is just here to make GCC happy. It should never be reached"); } } break; diff --git a/src/zscript/zcc_compile.h b/src/zscript/zcc_compile.h index 7f1f71c93..d034464b7 100644 --- a/src/zscript/zcc_compile.h +++ b/src/zscript/zcc_compile.h @@ -37,7 +37,7 @@ private: ZCC_Expression *NodeFromSymbol(PSymbol *sym, ZCC_Expression *source, PSymbolTable *table); ZCC_ExprConstant *NodeFromSymbolConst(PSymbolConst *sym, ZCC_Expression *idnode); ZCC_ExprTypeRef *NodeFromSymbolType(PSymbolType *sym, ZCC_Expression *idnode); - PSymbol *ZCCCompiler::CompileNode(ZCC_NamedNode *node); + PSymbol *CompileNode(ZCC_NamedNode *node); void Warn(ZCC_TreeNode *node, const char *msg, ...); From cafbc8b50aab3353cc19632bed7bca5872d9a8ed Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:30:44 -0500 Subject: [PATCH 03/47] Lemon update 2009-11-03 13:02:26 on branch trunk - Adjust the lemon implementation so that it always computes the same PDA regardless of qsort() implementation on the host platform. In other words, make all sorts in lemon stable. (user: drh) --- tools/lemon/lemon.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 026e3f1d9..b22db6977 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -374,6 +374,9 @@ struct action *ap2; if( rc==0 && ap1->type==REDUCE ){ rc = ap1->x.rp->index - ap2->x.rp->index; } + if( rc==0 ){ + rc = ap2 - ap1; + } return rc; } @@ -1591,7 +1594,7 @@ static void *merge(void *a,void *b,int (*cmp)(),size_t offset) }else if( b==0 ){ head = a; }else{ - if( (*cmp)(a,b)<0 ){ + if( (*cmp)(a,b)<=0 ){ ptr = a; a = NEXT(a); }else{ @@ -1600,7 +1603,7 @@ static void *merge(void *a,void *b,int (*cmp)(),size_t offset) } head = ptr; while( a && b ){ - if( (*cmp)(a,b)<0 ){ + if( (*cmp)(a,b)<=0 ){ NEXT(ptr) = a; ptr = a; a = NEXT(a); @@ -1649,7 +1652,7 @@ static void *msort(void *list,void *next,int (*cmp)()) set[i] = ep; } ep = 0; - for(i=0; inAction - p1->nAction; + int c; + c = p2->nAction - p1->nAction; + if( c==0 ){ + c = p2->iOrder - p1->iOrder; + } + assert( c!=0 || p1==p2 ); + return c; } /* @@ -3728,6 +3738,7 @@ int mhflag; /* Output in makeheaders format if true */ ** action table to a minimum, the heuristic of placing the largest action ** sets first is used. */ + for(i=0; instate*2; i++) ax[i].iOrder = i; qsort(ax, lemp->nstate*2, sizeof(ax[0]), axset_compare); pActtab = acttab_alloc(); for(i=0; instate*2 && ax[i].nAction>0; i++){ @@ -4145,7 +4156,11 @@ static int stateResortCompare(const void *a, const void *b){ n = pB->nNtAct - pA->nNtAct; if( n==0 ){ n = pB->nTknAct - pA->nTknAct; + if( n==0 ){ + n = pB->statenum - pA->statenum; + } } + assert( n!=0 ); return n; } @@ -4449,6 +4464,7 @@ char *x; int Symbolcmpp(struct symbol **a, struct symbol **b){ int i1 = (**a).index + 10000000*((**a).name[0]>'Z'); int i2 = (**b).index + 10000000*((**b).name[0]>'Z'); + assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 ); return i1-i2; } From 1c592c960198894d7ebc7cbb119037e230ddeff1 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:31:48 -0500 Subject: [PATCH 04/47] Lemon update 2009-11-03 19:18:32 on branch trunk - Enhancements to lemon to generate more compact action tables and to avoid making array bounds tests that can never fail on action table calculations. (user: drh) - Update zcc-parse.lemon: YY_SZ_ACTTAB is now YY_ACTTAB_COUNT --- src/zscript/zcc-parse.lemon | 2 +- tools/lemon/lemon.c | 50 +++++++++++++++++++++++-------------- tools/lemon/lempar.c | 24 ++++++++++++------ 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/zscript/zcc-parse.lemon b/src/zscript/zcc-parse.lemon index 69feacdc7..e9fe7e41d 100644 --- a/src/zscript/zcc-parse.lemon +++ b/src/zscript/zcc-parse.lemon @@ -90,7 +90,7 @@ static void SetNodeLine(ZCC_TreeNode *name, int line) for (int j = 1; j < YYERRORSYMBOL; ++j) { int k = i + j; - if (k >= 0 && k < YY_SZ_ACTTAB && yy_lookahead[k] == j) + if (k >= 0 && k < YY_ACTTAB_COUNT && yy_lookahead[k] == j) { expecting << (expecting.IsEmpty() ? "Expecting " : " or ") << ZCCTokenName(j); } diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index b22db6977..5801591db 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -524,21 +524,9 @@ int acttab_insert(acttab *p){ ** ** i is the index in p->aAction[] where p->mnLookahead is inserted. */ - for(i=0; inAction+p->mnLookahead; i++){ - if( p->aAction[i].lookahead<0 ){ - for(j=0; jnLookahead; j++){ - k = p->aLookahead[j].lookahead - p->mnLookahead + i; - if( k<0 ) break; - if( p->aAction[k].lookahead>=0 ) break; - } - if( jnLookahead ) continue; - for(j=0; jnAction; j++){ - if( p->aAction[j].lookahead==j+p->mnLookahead-i ) break; - } - if( j==p->nAction ){ - break; /* Fits in empty slots */ - } - }else if( p->aAction[i].lookahead==p->mnLookahead ){ + for(i=p->nAction-1; i>=0; i--){ + /* First look for an existing action table entry that can be reused */ + if( p->aAction[i].lookahead==p->mnLookahead ){ if( p->aAction[i].action!=p->mnAction ) continue; for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; @@ -557,6 +545,25 @@ int acttab_insert(acttab *p){ } } } + if( i<0 ){ + /* If no reusable entry is found, look for an empty slot */ + for(i=0; inAction; i++){ + if( p->aAction[i].lookahead<0 ){ + for(j=0; jnLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if( k<0 ) break; + if( p->aAction[k].lookahead>=0 ) break; + } + if( jnLookahead ) continue; + for(j=0; jnAction; j++){ + if( p->aAction[j].lookahead==j+p->mnLookahead-i ) break; + } + if( j==p->nAction ){ + break; /* Fits in empty slots */ + } + } + } + } /* Insert transaction set at index i. */ for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; @@ -3612,7 +3619,7 @@ int mhflag; /* Output in makeheaders format if true */ struct action *ap; struct rule *rp; struct acttab *pActtab; - int i, j, n; + int i, j, k, n; char *name; int mnTknOfst, mxTknOfst; int mnNtOfst, mxNtOfst; @@ -3771,8 +3778,9 @@ int mhflag; /* Output in makeheaders format if true */ free(ax); /* Output the yy_action table */ - fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; n = acttab_size(pActtab); + fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; + fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; for(i=j=0; instate + lemp->nrule + 2; @@ -3807,7 +3815,9 @@ int mhflag; /* Output in makeheaders format if true */ fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++; n = lemp->nstate; while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; - fprintf(out, "#define YY_SHIFT_MAX %d\n", n-1); lineno++; + fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; + fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; fprintf(out, "static const %s yy_shift_ofst[] = {\n", minimum_size_type(mnTknOfst-1, mxTknOfst)); lineno++; for(i=j=0; instate; while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; - fprintf(out, "#define YY_REDUCE_MAX %d\n", n-1); lineno++; + fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; + fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; fprintf(out, "static const %s yy_reduce_ofst[] = {\n", minimum_size_type(mnNtOfst-1, mxNtOfst)); lineno++; for(i=j=0; iyystack[pParser->yyidx].stateno; - if( stateno>YY_SHIFT_MAX || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ + if( stateno>YY_SHIFT_COUNT + || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; } assert( iLookAhead!=YYNOCODE ); i += iLookAhead; - if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){ + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ if( iLookAhead>0 ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ @@ -417,7 +417,15 @@ static int yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( j>=0 && j=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", @@ -449,22 +457,22 @@ static int yy_find_reduce_action( ){ int i; #ifdef YYERRORSYMBOL - if( stateno>YY_REDUCE_MAX ){ + if( stateno>YY_REDUCE_COUNT ){ return yy_default[stateno]; } #else - assert( stateno<=YY_REDUCE_MAX ); + assert( stateno<=YY_REDUCE_COUNT ); #endif i = yy_reduce_ofst[stateno]; assert( i!=YY_REDUCE_USE_DFLT ); assert( iLookAhead!=YYNOCODE ); i += iLookAhead; #ifdef YYERRORSYMBOL - if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){ + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ return yy_default[stateno]; } #else - assert( i>=0 && i=0 && i Date: Sun, 20 Mar 2016 09:32:45 -0500 Subject: [PATCH 05/47] Lemon update 2010-01-06 13:07:31 on branch trunk - Fix an issue with lemon generating incorrect grammars. This issue does not effect SQLite. (user: drh) --- tools/lemon/lemon.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 5801591db..102a86c27 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -495,6 +495,7 @@ void acttab_action(acttab *p, int lookahead, int action){ */ int acttab_insert(acttab *p){ int i, j, k, n; + int nActtab; /* Number of slots in the p->aAction[] table */ assert( p->nLookahead>0 ); /* Make sure we have enough space to hold the expanded action table @@ -502,7 +503,8 @@ int acttab_insert(acttab *p){ ** must be appended to the current action table */ n = p->mxLookahead + 1; - if( p->nAction + n >= p->nActionAlloc ){ + nActtab = p->nAction + n; + if( nActtab >= p->nActionAlloc ){ int oldAlloc = p->nActionAlloc; p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; p->aAction = realloc( p->aAction, @@ -520,11 +522,11 @@ int acttab_insert(acttab *p){ /* Scan the existing action table looking for an offset where we can ** insert the current transaction set. Fall out of the loop when that ** offset is found. In the worst case, we fall out of the loop when - ** i reaches p->nAction, which means we append the new transaction set. + ** i reaches nActtab, which means we append the new transaction set. ** ** i is the index in p->aAction[] where p->mnLookahead is inserted. */ - for(i=p->nAction-1; i>=0; i--){ + for(i=nActtab-1; i>=0; i--){ /* First look for an existing action table entry that can be reused */ if( p->aAction[i].lookahead==p->mnLookahead ){ if( p->aAction[i].action!=p->mnAction ) continue; @@ -547,7 +549,7 @@ int acttab_insert(acttab *p){ } if( i<0 ){ /* If no reusable entry is found, look for an empty slot */ - for(i=0; inAction; i++){ + for(i=0; iaAction[i].lookahead<0 ){ for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; From 7e8b9219a8844896ce946f166da6f099412ecd7d Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:33:25 -0500 Subject: [PATCH 06/47] Lemon update 2010-01-07 03:53:04 on branch trunk - Another attempt at fixing the table generator in lemon. Again, this does not effect the SQLite grammar. (user: drh) --- tools/lemon/lemon.c | 56 +++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 102a86c27..ec16de534 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -412,7 +412,23 @@ char *arg; /* ** The state of the yy_action table under construction is an instance of -** the following structure +** the following structure. +** +** The yy_action table maps the pair (state_number, lookahead) into an +** action_number. The table is an array of integers pairs. The state_number +** determines an initial offset into the yy_action array. The lookahead +** value is then added to this initial offset to get an index X into the +** yy_action array. If the aAction[X].lookahead equals the value of the +** of the lookahead input, then the value of the action_number output is +** aAction[X].action. If the lookaheads do not match then the +** default action for the state_number is returned. +** +** All actions associated with a single state_number are first entered +** into aLookahead[] using multiple calls to acttab_action(). Then the +** actions for that single state_number are placed into the aAction[] +** array with a single call to acttab_insert(). The acttab_insert() call +** also resets the aLookahead[] array in preparation for the next +** state number. */ typedef struct acttab acttab; struct acttab { @@ -458,7 +474,10 @@ acttab *acttab_alloc(void){ return p; } -/* Add a new action to the current transaction set +/* Add a new action to the current transaction set. +** +** This routine is called once for each lookahead for a particular +** state. */ void acttab_action(acttab *p, int lookahead, int action){ if( p->nLookahead>=p->nLookaheadAlloc ){ @@ -495,7 +514,6 @@ void acttab_action(acttab *p, int lookahead, int action){ */ int acttab_insert(acttab *p){ int i, j, k, n; - int nActtab; /* Number of slots in the p->aAction[] table */ assert( p->nLookahead>0 ); /* Make sure we have enough space to hold the expanded action table @@ -503,8 +521,7 @@ int acttab_insert(acttab *p){ ** must be appended to the current action table */ n = p->mxLookahead + 1; - nActtab = p->nAction + n; - if( nActtab >= p->nActionAlloc ){ + if( p->nAction + n >= p->nActionAlloc ){ int oldAlloc = p->nActionAlloc; p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; p->aAction = realloc( p->aAction, @@ -519,16 +536,16 @@ int acttab_insert(acttab *p){ } } - /* Scan the existing action table looking for an offset where we can - ** insert the current transaction set. Fall out of the loop when that - ** offset is found. In the worst case, we fall out of the loop when - ** i reaches nActtab, which means we append the new transaction set. + /* Scan the existing action table looking for an offset that is a + ** duplicate of the current transaction set. Fall out of the loop + ** if and when the duplicate is found. ** ** i is the index in p->aAction[] where p->mnLookahead is inserted. */ - for(i=nActtab-1; i>=0; i--){ - /* First look for an existing action table entry that can be reused */ + for(i=p->nAction-1; i>=0; i--){ if( p->aAction[i].lookahead==p->mnLookahead ){ + /* All lookaheads and actions in the aLookahead[] transaction + ** must match against the candidate aAction[i] entry. */ if( p->aAction[i].action!=p->mnAction ) continue; for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; @@ -537,19 +554,30 @@ int acttab_insert(acttab *p){ if( p->aLookahead[j].action!=p->aAction[k].action ) break; } if( jnLookahead ) continue; + + /* No possible lookahead value that is not in the aLookahead[] + ** transaction is allowed to match aAction[i] */ n = 0; for(j=0; jnAction; j++){ if( p->aAction[j].lookahead<0 ) continue; if( p->aAction[j].lookahead==j+p->mnLookahead-i ) n++; } if( n==p->nLookahead ){ - break; /* Same as a prior transaction set */ + break; /* An exact match is found at offset i */ } } } + + /* If no existing offsets exactly match the current transaction, find an + ** an empty offset in the aAction[] table in which we can add the + ** aLookahead[] transaction. + */ if( i<0 ){ - /* If no reusable entry is found, look for an empty slot */ - for(i=0; inAction, which means the + ** transaction will be appended. */ + for(i=0; inActionAlloc - p->mxLookahead; i++){ if( p->aAction[i].lookahead<0 ){ for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; From a656bb4f72c450f8feb66411402ab4b4e07ea019 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:37:13 -0500 Subject: [PATCH 07/47] Lemon update 2010-02-14 00:48:50 on branch lemon-update-2010 - Added -T option, to specify a template filename on the command line. The default is still "lempar.c", though. (user: icculus) --- tools/lemon/lemon.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index ec16de534..ca928bd35 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1446,6 +1446,14 @@ static void handle_D_option(char *z){ *z = 0; } +static char *user_templatename = NULL; +static void handle_T_option(char *z){ + user_templatename = malloc( lemonStrlen(z)+1 ); + if( user_templatename==0 ){ + memory_error(); + } + strcpy(user_templatename, z); +} /* The main program. Parse the command line and do it... */ int main(argc,argv) @@ -1464,6 +1472,7 @@ char **argv; {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."}, + {OPT_FSTR, "T", (char*)handle_T_option, "Specify a template file."}, {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, {OPT_FLAG, "m", (char*)&mhflag, "Output a makeheaders compatible file."}, {OPT_FLAG, "l", (char*)&nolinenosflag, "Do not print #line statements."}, @@ -3147,6 +3156,23 @@ struct lemon *lemp; char *cp; Boolean tpltnameinbuf; + /* first, see if user specified a template filename on the command line. */ + if (user_templatename != 0) { + if( access(user_templatename,004)==-1 ){ + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(user_templatename,"rb"); + if( in==0 ){ + fprintf(stderr,"Can't open the template file \"%s\".\n",user_templatename); + lemp->errorcnt++; + return 0; + } + return in; + } + cp = strrchr(lemp->filename,'.'); if( cp ){ sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); From 2e8d5e763b1d700685c7112e3650dfe905da7fee Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:48:32 -0500 Subject: [PATCH 08/47] Lemon update 2010-02-14 17:14:23 on branch lemon-update-2010 - Make Lemon able to compile as C++ code. (user: icculus) --- tools/lemon/lemon.c | 750 ++++++++++++++++++++------------------------ 1 file changed, 345 insertions(+), 405 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index ca928bd35..99ca6bf9d 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -24,7 +24,13 @@ #endif #ifdef __WIN32__ -extern int access(); +#ifdef __cplusplus +extern "C" { +#endif +extern int access(char *path, int mode); +#ifdef __cplusplus +} +#endif #else #include #endif @@ -47,6 +53,11 @@ static void *msort(void *list, void *next, int (*cmp)()); */ #define lemonStrlen(X) ((int)strlen(X)) +/* a few forward declarations... */ +struct rule; +struct lemon; +struct action; + /******** From the file "action.h" *************************************/ static struct action *Action_new(void); static struct action *Action_sort(struct action *); @@ -60,59 +71,60 @@ void FindFollowSets(); void FindActions(); /********* From the file "configlist.h" *********************************/ -void Configlist_init(/* void */); -struct config *Configlist_add(/* struct rule *, int */); -struct config *Configlist_addbasis(/* struct rule *, int */); -void Configlist_closure(/* void */); -void Configlist_sort(/* void */); -void Configlist_sortbasis(/* void */); -struct config *Configlist_return(/* void */); -struct config *Configlist_basis(/* void */); -void Configlist_eat(/* struct config * */); -void Configlist_reset(/* void */); +void Configlist_init(void); +struct config *Configlist_add(struct rule *, int); +struct config *Configlist_addbasis(struct rule *, int); +void Configlist_closure(struct lemon *); +void Configlist_sort(void); +void Configlist_sortbasis(void); +struct config *Configlist_return(void); +struct config *Configlist_basis(void); +void Configlist_eat(struct config *); +void Configlist_reset(void); /********* From the file "error.h" ***************************************/ void ErrorMsg(const char *, int,const char *, ...); /****** From the file "option.h" ******************************************/ +enum option_type { OPT_FLAG=1, OPT_INT, OPT_DBL, OPT_STR, + OPT_FFLAG, OPT_FINT, OPT_FDBL, OPT_FSTR}; struct s_options { - enum { OPT_FLAG=1, OPT_INT, OPT_DBL, OPT_STR, - OPT_FFLAG, OPT_FINT, OPT_FDBL, OPT_FSTR} type; - char *label; + enum option_type type; + const char *label; char *arg; - char *message; + const char *message; }; -int OptInit(/* char**,struct s_options*,FILE* */); -int OptNArgs(/* void */); -char *OptArg(/* int */); -void OptErr(/* int */); -void OptPrint(/* void */); +int OptInit(char**,struct s_options*,FILE*); +int OptNArgs(void); +char *OptArg(int); +void OptErr(int); +void OptPrint(void); /******** From the file "parse.h" *****************************************/ -void Parse(/* struct lemon *lemp */); +void Parse(struct lemon *lemp); /********* From the file "plink.h" ***************************************/ -struct plink *Plink_new(/* void */); -void Plink_add(/* struct plink **, struct config * */); -void Plink_copy(/* struct plink **, struct plink * */); -void Plink_delete(/* struct plink * */); +struct plink *Plink_new(void); +void Plink_add(struct plink **, struct config *); +void Plink_copy(struct plink **, struct plink *); +void Plink_delete(struct plink *); /********** From the file "report.h" *************************************/ -void Reprint(/* struct lemon * */); -void ReportOutput(/* struct lemon * */); -void ReportTable(/* struct lemon * */); -void ReportHeader(/* struct lemon * */); -void CompressTables(/* struct lemon * */); -void ResortStates(/* struct lemon * */); +void Reprint(struct lemon *); +void ReportOutput(struct lemon *); +void ReportTable(struct lemon *, int); +void ReportHeader(struct lemon *); +void CompressTables(struct lemon *); +void ResortStates(struct lemon *); /********** From the file "set.h" ****************************************/ -void SetSize(/* int N */); /* All sets will be of size N */ -char *SetNew(/* void */); /* A new set for element 0..N */ -void SetFree(/* char* */); /* Deallocate a set */ - -int SetAdd(/* char*,int */); /* Add element to a set */ -int SetUnion(/* char *A,char *B */); /* A <- A U B, thru element N */ +void SetSize(int); /* All sets will be of size N */ +char *SetNew(void); /* A new set for element 0..N */ +void SetFree(char*); /* Deallocate a set */ +char *SetNew(void); /* A new set for element 0..N */ +int SetAdd(char*,int); /* Add element to a set */ +int SetUnion(char *,char *); /* A <- A U B, thru element N */ #define SetFind(X,Y) (X[Y]) /* True if Y is in set X */ /********** From the file "struct.h" *************************************/ @@ -124,23 +136,25 @@ typedef enum {LEMON_FALSE=0, LEMON_TRUE} Boolean; /* Symbols (terminals and nonterminals) of the grammar are stored ** in the following: */ -struct symbol { - char *name; /* Name of the symbol */ - int index; /* Index number for this symbol */ - enum { - TERMINAL, - NONTERMINAL, - MULTITERMINAL - } type; /* Symbols are all either TERMINALS or NTs */ - struct rule *rule; /* Linked list of rules of this (if an NT) */ - struct symbol *fallback; /* fallback token in case this token doesn't parse */ - int prec; /* Precedence if defined (-1 otherwise) */ - enum e_assoc { +enum symbol_type { + TERMINAL, + NONTERMINAL, + MULTITERMINAL +}; +enum e_assoc { LEFT, RIGHT, NONE, UNK - } assoc; /* Associativity if precedence is defined */ +}; +struct symbol { + const char *name; /* Name of the symbol */ + int index; /* Index number for this symbol */ + enum symbol_type type; /* Symbols are all either TERMINALS or NTs */ + struct rule *rule; /* Linked list of rules of this (if an NT) */ + struct symbol *fallback; /* fallback token in case this token doesn't parse */ + int prec; /* Precedence if defined (-1 otherwise) */ + enum e_assoc assoc; /* Associativity if precedence is defined */ char *firstset; /* First-set for all rules of this symbol */ Boolean lambda; /* True if NT and can generate an empty string */ int useCnt; /* Number of times used */ @@ -161,14 +175,14 @@ struct symbol { ** structure. */ struct rule { struct symbol *lhs; /* Left-hand side of the rule */ - char *lhsalias; /* Alias for the LHS (NULL if none) */ + const char *lhsalias; /* Alias for the LHS (NULL if none) */ int lhsStart; /* True if left-hand side is the start symbol */ int ruleline; /* Line number for the rule */ int nrhs; /* Number of RHS symbols */ struct symbol **rhs; /* The RHS symbols */ - char **rhsalias; /* An alias for each RHS symbol (NULL if none) */ + const char **rhsalias; /* An alias for each RHS symbol (NULL if none) */ int line; /* Line number at which code begins */ - char *code; /* The code executed when this rule is reduced */ + const char *code; /* The code executed when this rule is reduced */ struct symbol *precsym; /* Precedence symbol for this rule */ int index; /* An index number for this rule */ Boolean canReduce; /* True if this rule is ever reduced */ @@ -181,6 +195,10 @@ struct rule { ** Configurations also contain a follow-set which is a list of terminal ** symbols which are allowed to immediately follow the end of the rule. ** Every configuration is recorded as an instance of the following: */ +enum cfgstatus { + COMPLETE, + INCOMPLETE +}; struct config { struct rule *rp; /* The rule upon which the configuration is based */ int dot; /* The parse point */ @@ -188,29 +206,28 @@ struct config { struct plink *fplp; /* Follow-set forward propagation links */ struct plink *bplp; /* Follow-set backwards propagation links */ struct state *stp; /* Pointer to state which contains this */ - enum { - COMPLETE, /* The status is used during followset and */ - INCOMPLETE /* shift computations */ - } status; + enum cfgstatus status; /* used during followset and shift computations */ struct config *next; /* Next configuration in the state */ struct config *bp; /* The next basis configuration */ }; +enum e_action { + SHIFT, + ACCEPT, + REDUCE, + ERROR, + SSCONFLICT, /* A shift/shift conflict */ + SRCONFLICT, /* Was a reduce, but part of a conflict */ + RRCONFLICT, /* Was a reduce, but part of a conflict */ + SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ + RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ + NOT_USED /* Deleted by compression */ +}; + /* Every shift or reduce operation is stored as one of the following */ struct action { struct symbol *sp; /* The look-ahead symbol */ - enum e_action { - SHIFT, - ACCEPT, - REDUCE, - ERROR, - SSCONFLICT, /* A shift/shift conflict */ - SRCONFLICT, /* Was a reduce, but part of a conflict */ - RRCONFLICT, /* Was a reduce, but part of a conflict */ - SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ - RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ - NOT_USED /* Deleted by compression */ - } type; + enum e_action type; union { struct state *stp; /* The new state, if a shift */ struct rule *rp; /* The rule, if a reduce */ @@ -297,41 +314,41 @@ struct lemon { /* ** Code for processing tables in the LEMON parser generator. */ - /* Routines for handling a strings */ -char *Strsafe(); +const char *Strsafe(const char *); -void Strsafe_init(/* void */); -int Strsafe_insert(/* char * */); -char *Strsafe_find(/* char * */); +void Strsafe_init(void); +int Strsafe_insert(const char *); +const char *Strsafe_find(const char *); /* Routines for handling symbols of the grammar */ -struct symbol *Symbol_new(); -int Symbolcmpp(/* struct symbol **, struct symbol ** */); -void Symbol_init(/* void */); -int Symbol_insert(/* struct symbol *, char * */); -struct symbol *Symbol_find(/* char * */); -struct symbol *Symbol_Nth(/* int */); -int Symbol_count(/* */); -struct symbol **Symbol_arrayof(/* */); +struct symbol *Symbol_new(const char *); +int Symbolcmpp(const void *, const void *); +void Symbol_init(void); +int Symbol_insert(struct symbol *, const char *); +struct symbol *Symbol_find(const char *); +struct symbol *Symbol_Nth(int); +int Symbol_count(void); +struct symbol **Symbol_arrayof(void); /* Routines to manage the state table */ -int Configcmp(/* struct config *, struct config * */); -struct state *State_new(); -void State_init(/* void */); -int State_insert(/* struct state *, struct config * */); -struct state *State_find(/* struct config * */); +int Configcmp(const char *, const char *); +struct state *State_new(void); +void State_init(void); +int State_insert(struct state *, struct config *); +struct state *State_find(struct config *); struct state **State_arrayof(/* */); /* Routines used for efficiency in Configlist_add */ -void Configtable_init(/* void */); -int Configtable_insert(/* struct config * */); -struct config *Configtable_find(/* struct config * */); -void Configtable_clear(/* int(*)(struct config *) */); +void Configtable_init(void); +int Configtable_insert(struct config *); +struct config *Configtable_find(struct config *); +void Configtable_clear(int(*)(struct config *)); + /****************** From the file "action.c" *******************************/ /* ** Routines processing parser actions in the LEMON parser generator. @@ -340,7 +357,7 @@ void Configtable_clear(/* int(*)(struct config *) */); /* Allocate a new parser action */ static struct action *Action_new(void){ static struct action *freelist = 0; - struct action *new; + struct action *newaction; if( freelist==0 ){ int i; @@ -353,9 +370,9 @@ static struct action *Action_new(void){ for(i=0; inext; - return new; + return newaction; } /* Compare two actions for sorting purposes. Return negative, zero, or @@ -387,22 +404,22 @@ static struct action *Action_sort(struct action *ap) return ap; } -void Action_add(app,type,sp,arg) -struct action **app; -enum e_action type; -struct symbol *sp; -char *arg; -{ - struct action *new; - new = Action_new(); - new->next = *app; - *app = new; - new->type = type; - new->sp = sp; +void Action_add( + struct action **app, + enum e_action type, + struct symbol *sp, + char *arg +){ + struct action *newaction; + newaction = Action_new(); + newaction->next = *app; + *app = newaction; + newaction->type = type; + newaction->sp = sp; if( type==SHIFT ){ - new->x.stp = (struct state *)arg; + newaction->x.stp = (struct state *)arg; }else{ - new->x.rp = (struct rule *)arg; + newaction->x.rp = (struct rule *)arg; } } /********************** New code to implement the "acttab" module ***********/ @@ -430,14 +447,16 @@ char *arg; ** also resets the aLookahead[] array in preparation for the next ** state number. */ +struct lookahead_action { + int lookahead; /* Value of the lookahead token */ + int action; /* Action to take on the given lookahead */ +}; typedef struct acttab acttab; struct acttab { int nAction; /* Number of used slots in aAction[] */ int nActionAlloc; /* Slots allocated for aAction[] */ - struct { - int lookahead; /* Value of the lookahead token */ - int action; /* Action to take on the given lookahead */ - } *aAction, /* The yy_action[] table under construction */ + struct lookahead_action + *aAction, /* The yy_action[] table under construction */ *aLookahead; /* A single new transaction set */ int mnLookahead; /* Minimum aLookahead[].lookahead */ int mnAction; /* Action associated with mnLookahead */ @@ -465,7 +484,7 @@ void acttab_free(acttab **pp){ /* Allocate a new acttab structure */ acttab *acttab_alloc(void){ - acttab *p = calloc( 1, sizeof(*p) ); + acttab *p = (acttab *) calloc( 1, sizeof(*p) ); if( p==0 ){ fprintf(stderr,"Unable to allocate memory for a new acttab."); exit(1); @@ -482,7 +501,7 @@ acttab *acttab_alloc(void){ void acttab_action(acttab *p, int lookahead, int action){ if( p->nLookahead>=p->nLookaheadAlloc ){ p->nLookaheadAlloc += 25; - p->aLookahead = realloc( p->aLookahead, + p->aLookahead = (struct lookahead_action *) realloc( p->aLookahead, sizeof(p->aLookahead[0])*p->nLookaheadAlloc ); if( p->aLookahead==0 ){ fprintf(stderr,"malloc failed\n"); @@ -524,7 +543,7 @@ int acttab_insert(acttab *p){ if( p->nAction + n >= p->nActionAlloc ){ int oldAlloc = p->nActionAlloc; p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; - p->aAction = realloc( p->aAction, + p->aAction = (struct lookahead_action *) realloc( p->aAction, sizeof(p->aAction[0])*p->nActionAlloc); if( p->aAction==0 ){ fprintf(stderr,"malloc failed\n"); @@ -622,8 +641,7 @@ int acttab_insert(acttab *p){ ** are not RHS symbols with a defined precedence, the precedence ** symbol field is left blank. */ -void FindRulePrecedences(xp) -struct lemon *xp; +void FindRulePrecedences(struct lemon *xp) { struct rule *rp; for(rp=xp->rule; rp; rp=rp->next){ @@ -652,8 +670,7 @@ struct lemon *xp; ** The first set is the set of all terminal symbols which can begin ** a string generated by that nonterminal. */ -void FindFirstSets(lemp) -struct lemon *lemp; +void FindFirstSets(struct lemon *lemp) { int i, j; struct rule *rp; @@ -715,9 +732,8 @@ struct lemon *lemp; ** are added to between some states so that the LR(1) follow sets ** can be computed later. */ -PRIVATE struct state *getstate(/* struct lemon * */); /* forward reference */ -void FindStates(lemp) -struct lemon *lemp; +PRIVATE struct state *getstate(struct lemon *); /* forward reference */ +void FindStates(struct lemon *lemp) { struct symbol *sp; struct rule *rp; @@ -775,9 +791,8 @@ does not work properly.",sp->name); /* Return a pointer to a state which is described by the configuration ** list which has been built from calls to Configlist_add. */ -PRIVATE void buildshifts(/* struct lemon *, struct state * */); /* Forwd ref */ -PRIVATE struct state *getstate(lemp) -struct lemon *lemp; +PRIVATE void buildshifts(struct lemon *, struct state *); /* Forwd ref */ +PRIVATE struct state *getstate(struct lemon *lemp) { struct config *cfp, *bp; struct state *stp; @@ -821,9 +836,7 @@ struct lemon *lemp; /* ** Return true if two symbols are the same. */ -int same_symbol(a,b) -struct symbol *a; -struct symbol *b; +int same_symbol(struct symbol *a, struct symbol *b) { int i; if( a==b ) return 1; @@ -839,13 +852,11 @@ struct symbol *b; /* Construct all successor states to the given state. A "successor" ** state is any state which can be reached by a shift action. */ -PRIVATE void buildshifts(lemp,stp) -struct lemon *lemp; -struct state *stp; /* The state from which successors are computed */ +PRIVATE void buildshifts(struct lemon *lemp, struct state *stp) { struct config *cfp; /* For looping thru the config closure of "stp" */ struct config *bcfp; /* For the inner loop on config closure of "stp" */ - struct config *new; /* */ + struct config *newcfg; /* */ struct symbol *sp; /* Symbol following the dot in configuration "cfp" */ struct symbol *bsp; /* Symbol following the dot in configuration "bcfp" */ struct state *newstp; /* A pointer to a successor state */ @@ -870,8 +881,8 @@ struct state *stp; /* The state from which successors are computed */ bsp = bcfp->rp->rhs[bcfp->dot]; /* Get symbol after dot */ if( !same_symbol(bsp,sp) ) continue; /* Must be same as for "cfp" */ bcfp->status = COMPLETE; /* Mark this config as used */ - new = Configlist_addbasis(bcfp->rp,bcfp->dot+1); - Plink_add(&new->bplp,bcfp); + newcfg = Configlist_addbasis(bcfp->rp,bcfp->dot+1); + Plink_add(&newcfg->bplp,bcfp); } /* Get a pointer to the state described by the basis configuration set @@ -894,8 +905,7 @@ struct state *stp; /* The state from which successors are computed */ /* ** Construct the propagation links */ -void FindLinks(lemp) -struct lemon *lemp; +void FindLinks(struct lemon *lemp) { int i; struct config *cfp, *other; @@ -930,8 +940,7 @@ struct lemon *lemp; ** A followset is the set of all symbols which can come immediately ** after a configuration. */ -void FindFollowSets(lemp) -struct lemon *lemp; +void FindFollowSets(struct lemon *lemp) { int i; struct config *cfp; @@ -963,12 +972,11 @@ struct lemon *lemp; }while( progress ); } -static int resolve_conflict(); +static int resolve_conflict(struct action *,struct action *, struct symbol *); /* Compute the reduce actions, and resolve conflicts. */ -void FindActions(lemp) -struct lemon *lemp; +void FindActions(struct lemon *lemp) { int i,j; struct config *cfp; @@ -1051,11 +1059,11 @@ struct lemon *lemp; ** If either action is a SHIFT, then it must be apx. This ** function won't work if apx->type==REDUCE and apy->type==SHIFT. */ -static int resolve_conflict(apx,apy,errsym) -struct action *apx; -struct action *apy; -struct symbol *errsym; /* The error symbol (if defined. NULL otherwise) */ -{ +static int resolve_conflict( + struct action *apx, + struct action *apy, + struct symbol *errsym /* The error symbol (if defined. NULL otherwise) */ +){ struct symbol *spx, *spy; int errcnt = 0; assert( apx->sp==apy->sp ); /* Otherwise there would be no conflict */ @@ -1128,7 +1136,7 @@ static struct config **basisend = 0; /* End of list of basis configs */ /* Return a pointer to a new configuration */ PRIVATE struct config *newconfig(){ - struct config *new; + struct config *newcfg; if( freelist==0 ){ int i; int amt = 3; @@ -1140,14 +1148,13 @@ PRIVATE struct config *newconfig(){ for(i=0; inext; - return new; + return newcfg; } /* The configuration "old" is no longer used */ -PRIVATE void deleteconfig(old) -struct config *old; +PRIVATE void deleteconfig(struct config *old) { old->next = freelist; freelist = old; @@ -1174,10 +1181,10 @@ void Configlist_reset(){ } /* Add another configuration to the configuration list */ -struct config *Configlist_add(rp,dot) -struct rule *rp; /* The rule */ -int dot; /* Index into the RHS of the rule where the dot goes */ -{ +struct config *Configlist_add( + struct rule *rp, /* The rule */ + int dot /* Index into the RHS of the rule where the dot goes */ +){ struct config *cfp, model; assert( currentend!=0 ); @@ -1201,9 +1208,7 @@ int dot; /* Index into the RHS of the rule where the dot goes */ } /* Add a basis configuration to the configuration list */ -struct config *Configlist_addbasis(rp,dot) -struct rule *rp; -int dot; +struct config *Configlist_addbasis(struct rule *rp, int dot) { struct config *cfp, model; @@ -1231,8 +1236,7 @@ int dot; } /* Compute the closure of the configuration list */ -void Configlist_closure(lemp) -struct lemon *lemp; +void Configlist_closure(struct lemon *lemp) { struct config *cfp, *newcfp; struct rule *rp, *newrp; @@ -1311,8 +1315,7 @@ struct config *Configlist_basis(){ } /* Free all elements of the given configuration list */ -void Configlist_eat(cfp) -struct config *cfp; +void Configlist_eat(struct config *cfp) { struct config *nextcfp; for(; cfp; cfp=nextcfp){ @@ -1332,10 +1335,7 @@ struct config *cfp; /* Find a good place to break "msg" so that its length is at least "min" ** but no more than "max". Make the point as close to max as possible. */ -static int findbreak(msg,min,max) -char *msg; -int min; -int max; +static int findbreak(char *msg, int min, int max) { int i,spot; char c; @@ -1430,13 +1430,13 @@ static char **azDefine = 0; /* Name of the -D macros */ static void handle_D_option(char *z){ char **paz; nDefine++; - azDefine = realloc(azDefine, sizeof(azDefine[0])*nDefine); + azDefine = (char **) realloc(azDefine, sizeof(azDefine[0])*nDefine); if( azDefine==0 ){ fprintf(stderr,"out of memory\n"); exit(1); } paz = &azDefine[nDefine-1]; - *paz = malloc( lemonStrlen(z)+1 ); + *paz = (char *) malloc( lemonStrlen(z)+1 ); if( *paz==0 ){ fprintf(stderr,"out of memory\n"); exit(1); @@ -1448,7 +1448,7 @@ static void handle_D_option(char *z){ static char *user_templatename = NULL; static void handle_T_option(char *z){ - user_templatename = malloc( lemonStrlen(z)+1 ); + user_templatename = (char *) malloc( lemonStrlen(z)+1 ); if( user_templatename==0 ){ memory_error(); } @@ -1456,9 +1456,7 @@ static void handle_T_option(char *z){ } /* The main program. Parse the command line and do it... */ -int main(argc,argv) -int argc; -char **argv; +int main(int argc, char **argv) { static int version = 0; static int rpflag = 0; @@ -1522,8 +1520,7 @@ char **argv; Symbol_new("{default}"); lem.symbols = Symbol_arrayof(); for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; - qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), - (int(*)(const void*, const void*))Symbolcmpp); + qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp); for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; for(i=1; isupper(lem.symbols[i]->name[0]); i++); lem.nterminal = i; @@ -1712,10 +1709,7 @@ static FILE *errstream; ** Print the command line with a carrot pointing to the k-th character ** of the n-th field. */ -static void errline(n,k,err) -int n; -int k; -FILE *err; +static void errline(int n, int k, FILE *err) { int i; size_t spcnt; @@ -1738,8 +1732,7 @@ FILE *err; ** Return the index of the N-th non-switch argument. Return -1 ** if N is out of range. */ -static int argindex(n) -int n; +static int argindex(int n) { int i; int dashdash = 0; @@ -1760,9 +1753,7 @@ static char emsg[] = "Command line syntax error: "; /* ** Process a flag command line argument. */ -static int handleflags(i,err) -int i; -FILE *err; +static int handleflags(int i, FILE *err) { int v; int errcnt = 0; @@ -1780,9 +1771,9 @@ FILE *err; }else if( op[j].type==OPT_FLAG ){ *((int*)op[j].arg) = v; }else if( op[j].type==OPT_FFLAG ){ - (*(void(*)())(op[j].arg))(v); + (*(void(*)(int))(op[j].arg))(v); }else if( op[j].type==OPT_FSTR ){ - (*(void(*)())(op[j].arg))(&argv[i][2]); + (*(void(*)(char *))(op[j].arg))(&argv[i][2]); }else{ if( err ){ fprintf(err,"%smissing argument on switch.\n",emsg); @@ -1796,9 +1787,7 @@ FILE *err; /* ** Process a command line switch which has an argument. */ -static int handleswitch(i,err) -int i; -FILE *err; +static int handleswitch(int i, FILE *err) { int lv = 0; double dv = 0.0; @@ -1865,29 +1854,26 @@ FILE *err; *(double*)(op[j].arg) = dv; break; case OPT_FDBL: - (*(void(*)())(op[j].arg))(dv); + (*(void(*)(double))(op[j].arg))(dv); break; case OPT_INT: *(int*)(op[j].arg) = lv; break; case OPT_FINT: - (*(void(*)())(op[j].arg))((int)lv); + (*(void(*)(int))(op[j].arg))((int)lv); break; case OPT_STR: *(char**)(op[j].arg) = sv; break; case OPT_FSTR: - (*(void(*)())(op[j].arg))(sv); + (*(void(*)(char *))(op[j].arg))(sv); break; } } return errcnt; } -int OptInit(a,o,err) -char **a; -struct s_options *o; -FILE *err; +int OptInit(char **a, struct s_options *o, FILE *err) { int errcnt = 0; argv = a; @@ -1924,16 +1910,14 @@ int OptNArgs(){ return cnt; } -char *OptArg(n) -int n; +char *OptArg(int n) { int i; i = argindex(n); return i>=0 ? argv[i] : 0; } -void OptErr(n) -int n; +void OptErr(int n) { int i; i = argindex(n); @@ -1995,42 +1979,43 @@ void OptPrint(){ */ /* The state of the parser */ +enum e_state { + INITIALIZE, + WAITING_FOR_DECL_OR_RULE, + WAITING_FOR_DECL_KEYWORD, + WAITING_FOR_DECL_ARG, + WAITING_FOR_PRECEDENCE_SYMBOL, + WAITING_FOR_ARROW, + IN_RHS, + LHS_ALIAS_1, + LHS_ALIAS_2, + LHS_ALIAS_3, + RHS_ALIAS_1, + RHS_ALIAS_2, + PRECEDENCE_MARK_1, + PRECEDENCE_MARK_2, + RESYNC_AFTER_RULE_ERROR, + RESYNC_AFTER_DECL_ERROR, + WAITING_FOR_DESTRUCTOR_SYMBOL, + WAITING_FOR_DATATYPE_SYMBOL, + WAITING_FOR_FALLBACK_ID, + WAITING_FOR_WILDCARD_ID +}; struct pstate { char *filename; /* Name of the input file */ int tokenlineno; /* Linenumber at which current token starts */ int errorcnt; /* Number of errors so far */ char *tokenstart; /* Text of current token */ struct lemon *gp; /* Global state vector */ - enum e_state { - INITIALIZE, - WAITING_FOR_DECL_OR_RULE, - WAITING_FOR_DECL_KEYWORD, - WAITING_FOR_DECL_ARG, - WAITING_FOR_PRECEDENCE_SYMBOL, - WAITING_FOR_ARROW, - IN_RHS, - LHS_ALIAS_1, - LHS_ALIAS_2, - LHS_ALIAS_3, - RHS_ALIAS_1, - RHS_ALIAS_2, - PRECEDENCE_MARK_1, - PRECEDENCE_MARK_2, - RESYNC_AFTER_RULE_ERROR, - RESYNC_AFTER_DECL_ERROR, - WAITING_FOR_DESTRUCTOR_SYMBOL, - WAITING_FOR_DATATYPE_SYMBOL, - WAITING_FOR_FALLBACK_ID, - WAITING_FOR_WILDCARD_ID - } state; /* The state of the parser */ + enum e_state state; /* The state of the parser */ struct symbol *fallback; /* The fallback token */ struct symbol *lhs; /* Left-hand side of current rule */ - char *lhsalias; /* Alias for the LHS */ + const char *lhsalias; /* Alias for the LHS */ int nrhs; /* Number of right-hand side symbols seen */ struct symbol *rhs[MAXRHS]; /* RHS symbols */ - char *alias[MAXRHS]; /* Aliases for each RHS symbol (or NULL) */ + const char *alias[MAXRHS]; /* Aliases for each RHS symbol (or NULL) */ struct rule *prevrule; /* Previous rule parsed */ - char *declkeyword; /* Keyword of a declaration */ + const char *declkeyword; /* Keyword of a declaration */ char **declargslot; /* Where the declaration argument should be put */ int insertLineMacro; /* Add #line before declaration insert */ int *decllinenoslot; /* Where to write declaration line number */ @@ -2041,10 +2026,9 @@ struct pstate { }; /* Parse a single token */ -static void parseonetoken(psp) -struct pstate *psp; +static void parseonetoken(struct pstate *psp) { - char *x; + const char *x; x = Strsafe(psp->tokenstart); /* Save the token permanently */ #if 0 printf("%s:%d: Token=[%s] state=%d\n",psp->filename,psp->tokenlineno, @@ -2176,7 +2160,7 @@ to follow the previous rule."); int i; rp->ruleline = psp->tokenlineno; rp->rhs = (struct symbol**)&rp[1]; - rp->rhsalias = (char**)&(rp->rhs[psp->nrhs]); + rp->rhsalias = (const char**)&(rp->rhs[psp->nrhs]); for(i=0; inrhs; i++){ rp->rhs[i] = psp->rhs[i]; rp->rhsalias[i] = psp->alias[i]; @@ -2215,16 +2199,17 @@ to follow the previous rule."); struct symbol *msp = psp->rhs[psp->nrhs-1]; if( msp->type!=MULTITERMINAL ){ struct symbol *origsp = msp; - msp = calloc(1,sizeof(*msp)); + msp = (struct symbol *) calloc(1,sizeof(*msp)); msp->type = MULTITERMINAL; msp->nsubsym = 1; - msp->subsym = calloc(1,sizeof(struct symbol*)); + msp->subsym = (struct symbol **) calloc(1,sizeof(struct symbol*)); msp->subsym[0] = origsp; msp->name = origsp->name; psp->rhs[psp->nrhs-1] = msp; } msp->nsubsym++; - msp->subsym = realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); if( islower(x[1]) || islower(msp->subsym[0]->name[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, @@ -2389,7 +2374,8 @@ to follow the previous rule."); break; case WAITING_FOR_DECL_ARG: if( x[0]=='{' || x[0]=='\"' || isalnum(x[0]) ){ - char *zOld, *zNew, *zBuf, *z; + const char *zOld, *zNew; + char *zBuf, *z; int nOld, n, nLine, nNew, nBack; int addLineMacro; char zLine[50]; @@ -2413,8 +2399,8 @@ to follow the previous rule."); nLine = lemonStrlen(zLine); n += nLine + lemonStrlen(psp->filename) + nBack; } - *psp->declargslot = zBuf = realloc(*psp->declargslot, n); - zBuf += nOld; + *psp->declargslot = (char *) realloc(*psp->declargslot, n); + zBuf = *psp->declargslot + nOld; if( addLineMacro ){ if( nOld && zBuf[-1]!='\n' ){ *(zBuf++) = '\n'; @@ -2567,8 +2553,7 @@ int filesize; ** token is passed to the function "parseonetoken" which builds all ** the appropriate data structures in the global state vector "gp". */ -void Parse(gp) -struct lemon *gp; +void Parse(struct lemon *gp) { struct pstate ps; FILE *fp; @@ -2725,7 +2710,7 @@ static struct plink *plink_freelist = 0; /* Allocate a new plink */ struct plink *Plink_new(){ - struct plink *new; + struct plink *newlink; if( plink_freelist==0 ){ int i; @@ -2739,27 +2724,23 @@ struct plink *Plink_new(){ for(i=0; inext; - return new; + return newlink; } /* Add a plink to a plink list */ -void Plink_add(plpp,cfp) -struct plink **plpp; -struct config *cfp; +void Plink_add(struct plink **plpp, struct config *cfp) { - struct plink *new; - new = Plink_new(); - new->next = *plpp; - *plpp = new; - new->cfp = cfp; + struct plink *newlink; + newlink = Plink_new(); + newlink->next = *plpp; + *plpp = newlink; + newlink->cfp = cfp; } /* Transfer every plink on the list "from" to the list "to" */ -void Plink_copy(to,from) -struct plink **to; -struct plink *from; +void Plink_copy(struct plink **to, struct plink *from) { struct plink *nextpl; while( from ){ @@ -2771,8 +2752,7 @@ struct plink *from; } /* Delete every plink on the list */ -void Plink_delete(plp) -struct plink *plp; +void Plink_delete(struct plink *plp) { struct plink *nextpl; @@ -2792,14 +2772,12 @@ struct plink *plp; ** name comes from malloc() and must be freed by the calling ** function. */ -PRIVATE char *file_makename(lemp,suffix) -struct lemon *lemp; -char *suffix; +PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) { char *name; char *cp; - name = malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 ); + name = (char*)malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 ); if( name==0 ){ fprintf(stderr,"Can't allocate space for a filename.\n"); exit(1); @@ -2814,11 +2792,11 @@ char *suffix; /* Open a file with a name based on the name of the input file, ** but with a different (specified) suffix, and return a pointer ** to the stream */ -PRIVATE FILE *file_open(lemp,suffix,mode) -struct lemon *lemp; -char *suffix; -char *mode; -{ +PRIVATE FILE *file_open( + struct lemon *lemp, + const char *suffix, + const char *mode +){ FILE *fp; if( lemp->outname ) free(lemp->outname); @@ -2834,8 +2812,7 @@ char *mode; /* Duplicate the input file without comments and without actions ** on rules */ -void Reprint(lemp) -struct lemon *lemp; +void Reprint(struct lemon *lemp) { struct rule *rp; struct symbol *sp; @@ -2880,9 +2857,7 @@ struct lemon *lemp; } } -void ConfigPrint(fp,cfp) -FILE *fp; -struct config *cfp; +void ConfigPrint(FILE *fp, struct config *cfp) { struct rule *rp; struct symbol *sp; @@ -2975,8 +2950,7 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ } /* Generate the "y.output" log file */ -void ReportOutput(lemp) -struct lemon *lemp; +void ReportOutput(struct lemon *lemp) { int i; struct state *stp; @@ -3042,12 +3016,11 @@ struct lemon *lemp; /* Search for the file "name" which is in the same directory as ** the exacutable */ -PRIVATE char *pathsearch(argv0,name,modemask) -char *argv0; -char *name; -int modemask; +PRIVATE char *pathsearch(char *argv0, char *name, int modemask) { - char *pathlist; + const char *pathlist; + char *pathbufptr; + char *pathbuf; char *path,*cp; char c; @@ -3067,22 +3040,25 @@ int modemask; if( path ) sprintf(path,"%s/%s",argv0,name); *cp = c; }else{ - extern char *getenv(); pathlist = getenv("PATH"); if( pathlist==0 ) pathlist = ".:/bin:/usr/bin"; + pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1 ); path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); - if( path!=0 ){ - while( *pathlist ){ - cp = strchr(pathlist,':'); - if( cp==0 ) cp = &pathlist[lemonStrlen(pathlist)]; + if( (pathbuf != 0) && (path!=0) ){ + pathbufptr = pathbuf; + strcpy(pathbuf, pathlist); + while( *pathbuf ){ + cp = strchr(pathbuf,':'); + if( cp==0 ) cp = &pathbuf[lemonStrlen(pathbuf)]; c = *cp; *cp = 0; - sprintf(path,"%s/%s",pathlist,name); + sprintf(path,"%s/%s",pathbuf,name); *cp = c; - if( c==0 ) pathlist = ""; - else pathlist = &cp[1]; + if( c==0 ) pathbuf[0] = 0; + else pathbuf = &cp[1]; if( access(path,modemask)==0 ) break; } + free(pathbufptr); } } return path; @@ -3092,9 +3068,7 @@ int modemask; ** which is to be put in the action table of the generated machine. ** Return negative if no action should be generated. */ -PRIVATE int compute_action(lemp,ap) -struct lemon *lemp; -struct action *ap; +PRIVATE int compute_action(struct lemon *lemp, struct action *ap) { int act; switch( ap->type ){ @@ -3117,11 +3091,7 @@ struct action *ap; ** if name!=0, then any word that begin with "Parse" is changed to ** begin with *name instead. */ -PRIVATE void tplt_xfer(name,in,out,lineno) -char *name; -FILE *in; -FILE *out; -int *lineno; +PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) { int i, iStart; char line[LINESIZE]; @@ -3146,8 +3116,7 @@ int *lineno; /* The next function finds the template file and opens it, returning ** a pointer to the opened file. */ -PRIVATE FILE *tplt_open(lemp) -struct lemon *lemp; +PRIVATE FILE *tplt_open(struct lemon *lemp) { static char templatename[] = "lempar.c"; char buf[1000]; @@ -3207,10 +3176,7 @@ struct lemon *lemp; } /* Print a #line directive line to the output file. */ -PRIVATE void tplt_linedir(out,lineno,filename) -FILE *out; -int lineno; -char *filename; +PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) { fprintf(out,"#line %d \"",lineno); while( *filename ){ @@ -3222,11 +3188,7 @@ char *filename; } /* Print a string to the file and keep the linenumber up to date */ -PRIVATE void tplt_print(out,lemp,str,lineno) -FILE *out; -struct lemon *lemp; -char *str; -int *lineno; +PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) { if( str==0 ) return; while( *str ){ @@ -3249,12 +3211,12 @@ int *lineno; ** The following routine emits code for the destructor for the ** symbol sp */ -void emit_destructor_code(out,sp,lemp,lineno) -FILE *out; -struct symbol *sp; -struct lemon *lemp; -int *lineno; -{ +void emit_destructor_code( + FILE *out, + struct symbol *sp, + struct lemon *lemp, + int *lineno +){ char *cp = 0; if( sp->type==TERMINAL ){ @@ -3292,9 +3254,7 @@ int *lineno; /* ** Return TRUE (non-zero) if the given symbol has a destructor. */ -int has_destructor(sp, lemp) -struct symbol *sp; -struct lemon *lemp; +int has_destructor(struct symbol *sp, struct lemon *lemp) { int ret; if( sp->type==TERMINAL ){ @@ -3317,13 +3277,13 @@ struct lemon *lemp; ** ** If n==-1, then the previous character is overwritten. */ -PRIVATE char *append_str(char *zText, int n, int p1, int p2, int bNoSubst){ +PRIVATE char *append_str(const char *zText, int n, int p1, int p2, int bNoSubst){ + static char empty[1] = { 0 }; static char *z = 0; static int alloced = 0; static int used = 0; int c; char zInt[40]; - if( zText==0 ){ used = 0; return z; @@ -3337,9 +3297,9 @@ PRIVATE char *append_str(char *zText, int n, int p1, int p2, int bNoSubst){ } if( n+sizeof(zInt)*2+used >= (size_t)alloced ){ alloced = n + sizeof(zInt)*2 + used + 200; - z = realloc(z, alloced); + z = (char *) realloc(z, alloced); } - if( z==0 ) return ""; + if( z==0 ) return empty; while( n-- > 0 ){ c = *(zText++); if( !bNoSubst && c=='%' && n>0 && zText[0]=='d' ){ @@ -3372,12 +3332,15 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ lhsused = 0; if( rp->code==0 ){ - rp->code = "\n"; - rp->line = rp->ruleline; + static char newlinestr[2] = { '\n', '\0' }; + rp->code = newlinestr; + rp->line = rp->ruleline; } append_str(0,0,0,0,0); - for(cp=rp->code; *cp; cp++){ + + /* This const cast is wrong but harmless, if we're careful. */ + for(cp=(char *)rp->code; *cp; cp++){ if( isalpha(*cp) && (cp==rp->code || (!isalnum(cp[-1]) && cp[-1]!='_')) ){ char saved; for(xp= &cp[1]; isalnum(*xp) || *xp=='_'; xp++); @@ -3450,13 +3413,13 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ ** Generate code which executes when the rule "rp" is reduced. Write ** the code to "out". Make sure lineno stays up-to-date. */ -PRIVATE void emit_code(out,rp,lemp,lineno) -FILE *out; -struct rule *rp; -struct lemon *lemp; -int *lineno; -{ - char *cp; +PRIVATE void emit_code( + FILE *out, + struct rule *rp, + struct lemon *lemp, + int *lineno +){ + const char *cp; /* Generate code to do the reduce action */ if( rp->code ){ @@ -3479,12 +3442,12 @@ int *lineno; ** union, also set the ".dtnum" field of every terminal and nonterminal ** symbol. */ -void print_stack_union(out,lemp,plineno,mhflag) -FILE *out; /* The output stream */ -struct lemon *lemp; /* The main info structure for this parser */ -int *plineno; /* Pointer to the line number */ -int mhflag; /* True if generating makeheaders output */ -{ +void print_stack_union( + FILE *out, /* The output stream */ + struct lemon *lemp, /* The main info structure for this parser */ + int *plineno, /* Pointer to the line number */ + int mhflag /* True if generating makeheaders output */ +){ int lineno = *plineno; /* The line number of the output */ char **types; /* A hash table of datatypes */ int arraysize; /* Size of the "types" array */ @@ -3492,7 +3455,7 @@ int mhflag; /* True if generating makeheaders output */ char *stddt; /* Standardized name for a datatype */ int i,j; /* Loop counters */ int hash; /* For hashing the name of a type */ - char *name; /* Name of the parser */ + const char *name; /* Name of the parser */ /* Allocate and initialize types[] and allocate stddt[] */ arraysize = lemp->nsymbol * 2; @@ -3664,10 +3627,10 @@ static void writeRuleText(FILE *out, struct rule *rp){ /* Generate C source code for the parser */ -void ReportTable(lemp, mhflag) -struct lemon *lemp; -int mhflag; /* Output in makeheaders format if true */ -{ +void ReportTable( + struct lemon *lemp, + int mhflag /* Output in makeheaders format if true */ +){ FILE *out, *in; char line[LINESIZE]; int lineno; @@ -3676,7 +3639,7 @@ int mhflag; /* Output in makeheaders format if true */ struct rule *rp; struct acttab *pActtab; int i, j, k, n; - char *name; + const char *name; int mnTknOfst, mxTknOfst; int mnNtOfst, mxNtOfst; struct axset *ax; @@ -3702,7 +3665,7 @@ int mhflag; /* Output in makeheaders format if true */ /* Generate #defines for all tokens */ if( mhflag ){ - char *prefix; + const char *prefix; fprintf(out,"#if INTERFACE\n"); lineno++; if( lemp->tokenprefix ) prefix = lemp->tokenprefix; else prefix = ""; @@ -3780,7 +3743,7 @@ int mhflag; /* Output in makeheaders format if true */ */ /* Compute the actions on all states and count them up */ - ax = calloc(lemp->nstate*2 , sizeof(ax[0])); + ax = (struct axset *) calloc(lemp->nstate*2 , sizeof(ax[0])); if( ax==0 ){ fprintf(stderr,"malloc failed\n"); exit(1); @@ -4109,11 +4072,10 @@ int mhflag; /* Output in makeheaders format if true */ } /* Generate a header file for the parser */ -void ReportHeader(lemp) -struct lemon *lemp; +void ReportHeader(struct lemon *lemp) { FILE *out, *in; - char *prefix; + const char *prefix; char line[LINESIZE]; char pattern[LINESIZE]; int i; @@ -4152,8 +4114,7 @@ struct lemon *lemp; ** it the default. Except, there is no default if the wildcard token ** is a possible look-ahead. */ -void CompressTables(lemp) -struct lemon *lemp; +void CompressTables(struct lemon *lemp) { struct state *stp; struct action *ap, *ap2; @@ -4237,8 +4198,7 @@ static int stateResortCompare(const void *a, const void *b){ ** Renumber and resort states so that states with fewer choices ** occur at the end. Except, keep state 0 as the first state. */ -void ResortStates(lemp) -struct lemon *lemp; +void ResortStates(struct lemon *lemp) { int i; struct state *stp; @@ -4278,8 +4238,7 @@ struct lemon *lemp; static int size = 0; /* Set the set size */ -void SetSize(n) -int n; +void SetSize(int n) { size = n+1; } @@ -4296,17 +4255,14 @@ char *SetNew(){ } /* Deallocate a set */ -void SetFree(s) -char *s; +void SetFree(char *s) { free(s); } /* Add a new element to the set. Return TRUE if the element was added ** and FALSE if it was already there. */ -int SetAdd(s,e) -char *s; -int e; +int SetAdd(char *s, int e) { int rv; assert( e>=0 && e'Z'); int i2 = (**b).index + 10000000*((**b).name[0]>'Z'); assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 ); @@ -4552,8 +4506,8 @@ struct s_x2 { ** in an associative array of type "x2". */ typedef struct s_x2node { - struct symbol *data; /* The data */ - char *key; /* The key */ + struct symbol *data; /* The data */ + const char *key; /* The key */ struct s_x2node *next; /* Next entry with the same hash */ struct s_x2node **from; /* Previous link */ } x2node; @@ -4582,9 +4536,7 @@ void Symbol_init(){ } /* Insert a new record into the array. Return TRUE if successful. ** Prior data with the same key is NOT overwritten */ -int Symbol_insert(data,key) -struct symbol *data; -char *key; +int Symbol_insert(struct symbol *data, const char *key) { x2node *np; int h; @@ -4642,8 +4594,7 @@ char *key; /* Return a pointer to data assigned to the given key. Return NULL ** if no such key. */ -struct symbol *Symbol_find(key) -char *key; +struct symbol *Symbol_find(const char *key) { int h; x2node *np; @@ -4659,8 +4610,7 @@ char *key; } /* Return the n-th data. Return NULL if n is out of range. */ -struct symbol *Symbol_Nth(n) -int n; +struct symbol *Symbol_Nth(int n) { struct symbol *data; if( x2a && n>0 && n<=x2a->count ){ @@ -4694,10 +4644,10 @@ struct symbol **Symbol_arrayof() } /* Compare two configurations */ -int Configcmp(a,b) -struct config *a; -struct config *b; +int Configcmp(const char *_a,const char *_b) { + const struct config *a = (struct config *) _a; + const struct config *b = (struct config *) _b; int x; x = a->rp->index - b->rp->index; if( x==0 ) x = a->dot - b->dot; @@ -4705,9 +4655,7 @@ struct config *b; } /* Compare two states */ -PRIVATE int statecmp(a,b) -struct config *a; -struct config *b; +PRIVATE int statecmp(struct config *a, struct config *b) { int rc; for(rc=0; rc==0 && a && b; a=a->bp, b=b->bp){ @@ -4722,8 +4670,7 @@ struct config *b; } /* Hash a state */ -PRIVATE int statehash(a) -struct config *a; +PRIVATE int statehash(struct config *a) { int h=0; while( a ){ @@ -4736,10 +4683,10 @@ struct config *a; /* Allocate a new state structure */ struct state *State_new() { - struct state *new; - new = (struct state *)calloc(1, sizeof(struct state) ); - MemoryCheck(new); - return new; + struct state *newstate; + newstate = (struct state *)calloc(1, sizeof(struct state) ); + MemoryCheck(newstate); + return newstate; } /* There is one instance of the following structure for each @@ -4788,9 +4735,7 @@ void State_init(){ } /* Insert a new record into the array. Return TRUE if successful. ** Prior data with the same key is NOT overwritten */ -int State_insert(data,key) -struct state *data; -struct config *key; +int State_insert(struct state *data, struct config *key) { x3node *np; int h; @@ -4848,8 +4793,7 @@ struct config *key; /* Return a pointer to data assigned to the given key. Return NULL ** if no such key. */ -struct state *State_find(key) -struct config *key; +struct state *State_find(struct config *key) { int h; x3node *np; @@ -4881,8 +4825,7 @@ struct state **State_arrayof() } /* Hash a configuration */ -PRIVATE int confighash(a) -struct config *a; +PRIVATE int confighash(struct config *a) { int h=0; h = h*571 + a->rp->index*37 + a->dot; @@ -4934,8 +4877,7 @@ void Configtable_init(){ } /* Insert a new record into the array. Return TRUE if successful. ** Prior data with the same key is NOT overwritten */ -int Configtable_insert(data) -struct config *data; +int Configtable_insert(struct config *data) { x4node *np; int h; @@ -4946,7 +4888,7 @@ struct config *data; h = ph & (x4a->size-1); np = x4a->ht[h]; while( np ){ - if( Configcmp(np->data,data)==0 ){ + if( Configcmp((const char *) np->data,(const char *) data)==0 ){ /* An existing entry with the same key is found. */ /* Fail because overwrite is not allows. */ return 0; @@ -4991,8 +4933,7 @@ struct config *data; /* Return a pointer to data assigned to the given key. Return NULL ** if no such key. */ -struct config *Configtable_find(key) -struct config *key; +struct config *Configtable_find(struct config *key) { int h; x4node *np; @@ -5001,7 +4942,7 @@ struct config *key; h = confighash(key) & (x4a->size-1); np = x4a->ht[h]; while( np ){ - if( Configcmp(np->data,key)==0 ) break; + if( Configcmp((const char *) np->data,(const char *) key)==0 ) break; np = np->next; } return np ? np->data : 0; @@ -5009,8 +4950,7 @@ struct config *key; /* Remove all data from the table. Pass each data to the function "f" ** as it is removed. ("f" may be null to avoid this step.) */ -void Configtable_clear(f) -int(*f)(/* struct config * */); +void Configtable_clear(int(*f)(struct config *)) { int i; if( x4a==0 || x4a->count==0 ) return; From 031cebb1384f079464ec19eb4ab523a4d740bc69 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:53:54 -0500 Subject: [PATCH 09/47] Lemon update 2010-02-15 00:01:04 on branch lemon-update-2010 - Removed the 80-char line formatting for ErrorMsg(), on dhr's suggestion. (user: icculus) --- tools/lemon/lemon.c | 69 +++++---------------------------------------- 1 file changed, 7 insertions(+), 62 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 99ca6bf9d..75bb4da53 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1332,81 +1332,26 @@ void Configlist_eat(struct config *cfp) ** Code for printing error message. */ -/* Find a good place to break "msg" so that its length is at least "min" -** but no more than "max". Make the point as close to max as possible. -*/ -static int findbreak(char *msg, int min, int max) -{ - int i,spot; - char c; - for(i=spot=min; i<=max; i++){ - c = msg[i]; - if( c=='\t' ) msg[i] = ' '; - if( c=='\n' ){ msg[i] = ' '; spot = i; break; } - if( c==0 ){ spot = i; break; } - if( c=='-' && i0 ){ - sprintf(prefix,"%.*s(%d) : error : ",PREFIXLIMIT-10,filename,lineno); + fprintf(stderr,"%s(%d) : error : ",filename,lineno); }else{ - sprintf(prefix,"%.*s : error : ",PREFIXLIMIT-10,filename); + fprintf(stderr,"%s : error : ",filename); } #else if( lineno>0 ){ - sprintf(prefix,"%.*s:%d: ",PREFIXLIMIT-10,filename,lineno); + fprintf(stderr,"%s:%d: ",filename,lineno); }else{ - sprintf(prefix,"%.*s: ",PREFIXLIMIT-10,filename); + fprintf(stderr,"%s: ",filename); } #endif - prefixsize = lemonStrlen(prefix); - availablewidth = LINEWIDTH - prefixsize; - - /* Generate the error message */ - vsprintf(errmsg,format,ap); + va_start(ap, format); + vfprintf(stderr,format,ap); va_end(ap); - errmsgsize = lemonStrlen(errmsg); - /* Remove trailing '\n's from the error message. */ - while( errmsgsize>0 && errmsg[errmsgsize-1]=='\n' ){ - errmsg[--errmsgsize] = 0; - } - - /* Print the error message */ - base = 0; - while( errmsg[base]!=0 ){ - end = restart = findbreak(&errmsg[base],0,availablewidth); - restart += base; - while( errmsg[restart]==' ' ) restart++; - fprintf(stdout,"%s%.*s\n",prefix,end,&errmsg[base]); - base = restart; - } + fprintf(stderr, "\n"); } /**************** From the file "main.c" ************************************/ /* From 3d7a8e8348bc20e3b487b91ae9ebf079456bd950 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:55:01 -0500 Subject: [PATCH 10/47] Lemon update 2010-02-17 20:19:51 on branch lemon-update-2010 - Corrected error message (cut-and-paste bug). (user: icculus) --- tools/lemon/lemon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 75bb4da53..f0a2e06e5 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -2287,7 +2287,7 @@ to follow the previous rule."); case WAITING_FOR_DATATYPE_SYMBOL: if( !isalpha(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, - "Symbol name missing after %%destructor keyword"); + "Symbol name missing after %%type keyword"); psp->errorcnt++; psp->state = RESYNC_AFTER_DECL_ERROR; }else{ From 54ff482d8fb76e8facd8b4b1fb871de9440b0608 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:56:17 -0500 Subject: [PATCH 11/47] Lemon update 2010-02-17 20:31:32 on branch lemon-update-2010 - Report error if the grammar has multiple %type lines for the same nonterminal. --- tools/lemon/lemon.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index f0a2e06e5..e21a4cffc 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -2291,10 +2291,20 @@ to follow the previous rule."); psp->errorcnt++; psp->state = RESYNC_AFTER_DECL_ERROR; }else{ - struct symbol *sp = Symbol_new(x); - psp->declargslot = &sp->datatype; - psp->insertLineMacro = 0; - psp->state = WAITING_FOR_DECL_ARG; + struct symbol *sp = Symbol_find(x); + if((sp) && (sp->datatype)){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol %%type \"%s\" already defined", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + if (!sp){ + sp = Symbol_new(x); + } + psp->declargslot = &sp->datatype; + psp->insertLineMacro = 0; + psp->state = WAITING_FOR_DECL_ARG; + } } break; case WAITING_FOR_PRECEDENCE_SYMBOL: From e20dc913b77451027e86bd6c29755122820843ed Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:57:51 -0500 Subject: [PATCH 12/47] Lemon update 2010-07-18 11:35:53 on branch trunk - Add the -p option to lemon to cause conflicts resolved by precedence rules to appear in the parse.out file. (user: drh) --- tools/lemon/lemon.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index e21a4cffc..2dedf16dc 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -44,6 +44,7 @@ extern int access(char *path, int mode); #define MAXRHS 1000 #endif +static int showPrecedenceConflict = 0; static void *msort(void *list, void *next, int (*cmp)()); /* @@ -1419,6 +1420,8 @@ int main(int argc, char **argv) {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, {OPT_FLAG, "m", (char*)&mhflag, "Output a makeheaders compatible file."}, {OPT_FLAG, "l", (char*)&nolinenosflag, "Do not print #line statements."}, + {OPT_FLAG, "p", (char*)&showPrecedenceConflict, + "Show conflicts resolved by precedence rules"}, {OPT_FLAG, "q", (char*)&quiet, "(Quiet) Don't print the report file."}, {OPT_FLAG, "s", (char*)&statistics, "Print parser stats to standard output."}, @@ -2896,7 +2899,21 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: + if( showPrecedenceConflict ){ + fprintf(fp,"%*s shift %d -- dropped by precedence", + indent,ap->sp->name,ap->x.stp->statenum); + }else{ + result = 0; + } + break; case RD_RESOLVED: + if( showPrecedenceConflict ){ + fprintf(fp,"%*s reduce %d -- dropped by precedence", + indent,ap->sp->name,ap->x.rp->index); + }else{ + result = 0; + } + break; case NOT_USED: result = 0; break; From cb7eb1abc5293aef64e8bb14f0a25fbf87f419c3 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 09:58:45 -0500 Subject: [PATCH 13/47] Lemon update 2010-07-19 01:52:07 on branch trunk - Improvements to the formatting of parse.out file from Lemon. Add the -r option to Lemon to disable the state sorting, making debugging easier. (user: drh) --- tools/lemon/lemon.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 2dedf16dc..dd3966219 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1079,7 +1079,7 @@ static int resolve_conflict( /* Not enough precedence information. */ apy->type = SRCONFLICT; errcnt++; - }else if( spx->prec>spy->prec ){ /* Lower precedence wins */ + }else if( spx->prec>spy->prec ){ /* higher precedence wins */ apy->type = RD_RESOLVED; }else if( spx->precprec ){ apx->type = SH_RESOLVED; @@ -1412,6 +1412,7 @@ int main(int argc, char **argv) static int statistics = 0; static int mhflag = 0; static int nolinenosflag = 0; + static int noResort = 0; static struct s_options options[] = { {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, @@ -1423,6 +1424,7 @@ int main(int argc, char **argv) {OPT_FLAG, "p", (char*)&showPrecedenceConflict, "Show conflicts resolved by precedence rules"}, {OPT_FLAG, "q", (char*)&quiet, "(Quiet) Don't print the report file."}, + {OPT_FLAG, "r", (char*)&noResort, "Do not sort or renumber states"}, {OPT_FLAG, "s", (char*)&statistics, "Print parser stats to standard output."}, {OPT_FLAG, "x", (char*)&version, "Print the version number."}, @@ -1506,8 +1508,9 @@ int main(int argc, char **argv) if( compress==0 ) CompressTables(&lem); /* Reorder and renumber the states so that states with fewer choices - ** occur at the end. */ - ResortStates(&lem); + ** occur at the end. This is an optimization that helps make the + ** generated parser tables smaller. */ + if( noResort==0 ) ResortStates(&lem); /* Generate a report of the parser generated. (the "y.output" file) */ if( !quiet ) ReportOutput(&lem); @@ -2895,12 +2898,12 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ indent,ap->sp->name,ap->x.rp->index); break; case SSCONFLICT: - fprintf(fp,"%*s shift %d ** Parsing conflict **", + fprintf(fp,"%*s shift %-3d ** Parsing conflict **", indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s shift %d -- dropped by precedence", + fprintf(fp,"%*s shift %-3d -- dropped by precedence", indent,ap->sp->name,ap->x.stp->statenum); }else{ result = 0; @@ -2908,7 +2911,7 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ break; case RD_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s reduce %d -- dropped by precedence", + fprintf(fp,"%*s reduce %-3d -- dropped by precedence", indent,ap->sp->name,ap->x.rp->index); }else{ result = 0; From c5e4f05fe28a8b72314fc8c66667469c9f06bd5c Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:01:41 -0500 Subject: [PATCH 14/47] Lemon update 2012-01-09 14:19:05 on branch trunk - Cosmetic changes to lemon. No changes to core functionality nor impact on SQLite. (user: drh) --- tools/lemon/lemon.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index dd3966219..a79b3224e 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -122,8 +122,6 @@ void ResortStates(struct lemon *); void SetSize(int); /* All sets will be of size N */ char *SetNew(void); /* A new set for element 0..N */ void SetFree(char*); /* Deallocate a set */ - -char *SetNew(void); /* A new set for element 0..N */ int SetAdd(char*,int); /* Add element to a set */ int SetUnion(char *,char *); /* A <- A U B, thru element N */ #define SetFind(X,Y) (X[Y]) /* True if Y is in set X */ @@ -973,7 +971,7 @@ void FindFollowSets(struct lemon *lemp) }while( progress ); } -static int resolve_conflict(struct action *,struct action *, struct symbol *); +static int resolve_conflict(struct action *,struct action *); /* Compute the reduce actions, and resolve conflicts. */ @@ -1027,7 +1025,7 @@ void FindActions(struct lemon *lemp) for(nap=ap->next; nap && nap->sp==ap->sp; nap=nap->next){ /* The two actions "ap" and "nap" have the same lookahead. ** Figure out which one should be used */ - lemp->nconflict += resolve_conflict(ap,nap,lemp->errsym); + lemp->nconflict += resolve_conflict(ap,nap); } } } @@ -1062,8 +1060,7 @@ void FindActions(struct lemon *lemp) */ static int resolve_conflict( struct action *apx, - struct action *apy, - struct symbol *errsym /* The error symbol (if defined. NULL otherwise) */ + struct action *apy ){ struct symbol *spx, *spy; int errcnt = 0; From 249aed455bd1a9b0c19a73a85a967efa04a703d5 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:02:29 -0500 Subject: [PATCH 15/47] Lemon update 2013-10-02 20:46:30 on branch trunk - In the lemon parser generator, change all hashes to unsigned to avoid potential problems with signed integer overflow. (user: drh) --- tools/lemon/lemon.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index a79b3224e..ae10a6447 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3426,7 +3426,7 @@ void print_stack_union( int maxdtlength; /* Maximum length of any ".datatype" field. */ char *stddt; /* Standardized name for a datatype */ int i,j; /* Loop counters */ - int hash; /* For hashing the name of a type */ + unsigned hash; /* For hashing the name of a type */ const char *name; /* Name of the parser */ /* Allocate and initialize types[] and allocate stddt[] */ @@ -4270,10 +4270,10 @@ int SetUnion(char *s1, char *s2) ** Code for processing tables in the LEMON parser generator. */ -PRIVATE int strhash(const char *x) +PRIVATE unsigned strhash(const char *x) { - int h = 0; - while( *x) h = h*13 + *(x++); + unsigned h = 0; + while( *x ) h = h*13 + *(x++); return h; } @@ -4345,8 +4345,8 @@ void Strsafe_init(){ int Strsafe_insert(const char *data) { x1node *np; - int h; - int ph; + unsigned h; + unsigned ph; if( x1a==0 ) return 0; ph = strhash(data); @@ -4400,7 +4400,7 @@ int Strsafe_insert(const char *data) ** if no such key. */ const char *Strsafe_find(const char *key) { - int h; + unsigned h; x1node *np; if( x1a==0 ) return 0; @@ -4511,8 +4511,8 @@ void Symbol_init(){ int Symbol_insert(struct symbol *data, const char *key) { x2node *np; - int h; - int ph; + unsigned h; + unsigned ph; if( x2a==0 ) return 0; ph = strhash(key); @@ -4568,7 +4568,7 @@ int Symbol_insert(struct symbol *data, const char *key) ** if no such key. */ struct symbol *Symbol_find(const char *key) { - int h; + unsigned h; x2node *np; if( x2a==0 ) return 0; @@ -4642,9 +4642,9 @@ PRIVATE int statecmp(struct config *a, struct config *b) } /* Hash a state */ -PRIVATE int statehash(struct config *a) +PRIVATE unsigned statehash(struct config *a) { - int h=0; + unsigned h=0; while( a ){ h = h*571 + a->rp->index*37 + a->dot; a = a->bp; @@ -4710,8 +4710,8 @@ void State_init(){ int State_insert(struct state *data, struct config *key) { x3node *np; - int h; - int ph; + unsigned h; + unsigned ph; if( x3a==0 ) return 0; ph = statehash(key); @@ -4767,7 +4767,7 @@ int State_insert(struct state *data, struct config *key) ** if no such key. */ struct state *State_find(struct config *key) { - int h; + unsigned h; x3node *np; if( x3a==0 ) return 0; @@ -4797,9 +4797,9 @@ struct state **State_arrayof() } /* Hash a configuration */ -PRIVATE int confighash(struct config *a) +PRIVATE unsigned confighash(struct config *a) { - int h=0; + unsigned h=0; h = h*571 + a->rp->index*37 + a->dot; return h; } @@ -4852,8 +4852,8 @@ void Configtable_init(){ int Configtable_insert(struct config *data) { x4node *np; - int h; - int ph; + unsigned h; + unsigned ph; if( x4a==0 ) return 0; ph = confighash(data); From e59ef08cc83491eeff0dd67eec72d0c8f920b894 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:08:44 -0500 Subject: [PATCH 16/47] Lemon update 2014-01-11 03:06:18 on branch lemon-updates - Add the new "%token_class" directive for defining symbolic names that stand any one of a collection of tokens. (user: drh) --- tools/lemon/lemon.c | 90 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index ae10a6447..4a0eb488e 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1463,12 +1463,15 @@ int main(int argc, char **argv) } /* Count and index the symbols of the grammar */ - lem.nsymbol = Symbol_count(); Symbol_new("{default}"); + lem.nsymbol = Symbol_count(); lem.symbols = Symbol_arrayof(); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; - qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; + for(i=0; iindex = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp); + for(i=0; iindex = i; + while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); + lem.nsymbol = i - 1; for(i=1; isupper(lem.symbols[i]->name[0]); i++); lem.nterminal = i; @@ -1947,7 +1950,9 @@ enum e_state { WAITING_FOR_DESTRUCTOR_SYMBOL, WAITING_FOR_DATATYPE_SYMBOL, WAITING_FOR_FALLBACK_ID, - WAITING_FOR_WILDCARD_ID + WAITING_FOR_WILDCARD_ID, + WAITING_FOR_CLASS_ID, + WAITING_FOR_CLASS_TOKEN }; struct pstate { char *filename; /* Name of the input file */ @@ -1957,6 +1962,7 @@ struct pstate { struct lemon *gp; /* Global state vector */ enum e_state state; /* The state of the parser */ struct symbol *fallback; /* The fallback token */ + struct symbol *tkclass; /* Token class symbol */ struct symbol *lhs; /* Left-hand side of current rule */ const char *lhsalias; /* Alias for the LHS */ int nrhs; /* Number of right-hand side symbols seen */ @@ -2260,6 +2266,8 @@ to follow the previous rule."); psp->state = WAITING_FOR_FALLBACK_ID; }else if( strcmp(x,"wildcard")==0 ){ psp->state = WAITING_FOR_WILDCARD_ID; + }else if( strcmp(x,"token_class")==0 ){ + psp->state = WAITING_FOR_CLASS_ID; }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Unknown declaration keyword: \"%%%s\".",x); @@ -2428,6 +2436,40 @@ to follow the previous rule."); } } break; + case WAITING_FOR_CLASS_ID: + if( !islower(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: ", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else if( Symbol_find(x) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){ + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + if( !isupper(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; case RESYNC_AFTER_RULE_ERROR: /* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; ** break; */ @@ -2800,11 +2842,13 @@ void Reprint(struct lemon *lemp) printf(" ::="); for(i=0; inrhs; i++){ sp = rp->rhs[i]; - printf(" %s", sp->name); if( sp->type==MULTITERMINAL ){ + printf(" %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ printf("|%s", sp->subsym[j]->name); } + }else{ + printf(" %s", sp->name); } /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */ } @@ -2826,11 +2870,13 @@ void ConfigPrint(FILE *fp, struct config *cfp) if( i==cfp->dot ) fprintf(fp," *"); if( i==rp->nrhs ) break; sp = rp->rhs[i]; - fprintf(fp," %s", sp->name); if( sp->type==MULTITERMINAL ){ + fprintf(fp," %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ fprintf(fp,"|%s",sp->subsym[j]->name); } + }else{ + fprintf(fp," %s", sp->name); } } } @@ -3587,9 +3633,11 @@ static void writeRuleText(FILE *out, struct rule *rp){ fprintf(out,"%s ::=", rp->lhs->name); for(j=0; jnrhs; j++){ struct symbol *sp = rp->rhs[j]; - fprintf(out," %s", sp->name); - if( sp->type==MULTITERMINAL ){ + if( sp->type!=MULTITERMINAL ){ + fprintf(out," %s", sp->name); + }else{ int k; + fprintf(out," %s", sp->subsym[0]->name); for(k=1; knsubsym; k++){ fprintf(out,"|%s",sp->subsym[k]->name); } @@ -4058,7 +4106,8 @@ void ReportHeader(struct lemon *lemp) if( in ){ int nextChar; for(i=1; interminal && fgets(line,LINESIZE,in); i++){ - sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + sprintf(pattern,"#define %s%-30s %2d\n", + prefix,lemp->symbols[i]->name,i); if( strcmp(line,pattern) ) break; } nextChar = fgetc(in); @@ -4072,7 +4121,7 @@ void ReportHeader(struct lemon *lemp) out = file_open(lemp,".h","wb"); if( out ){ for(i=1; interminal; i++){ - fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); } fclose(out); } @@ -4442,11 +4491,15 @@ struct symbol *Symbol_new(const char *x) return sp; } -/* Compare two symbols for working purposes +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. ** ** Symbols that begin with upper case letters (terminals or tokens) ** must sort before symbols that begin with lower case letters -** (non-terminals). Other than that, the order does not matter. +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. ** ** We find experimentally that leaving the symbols in their original ** order (the order they appeared in the grammar file) gives the @@ -4454,12 +4507,11 @@ struct symbol *Symbol_new(const char *x) */ int Symbolcmpp(const void *_a, const void *_b) { - const struct symbol **a = (const struct symbol **) _a; - const struct symbol **b = (const struct symbol **) _b; - int i1 = (**a).index + 10000000*((**a).name[0]>'Z'); - int i2 = (**b).index + 10000000*((**b).name[0]>'Z'); - assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 ); - return i1-i2; + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; } /* There is one instance of the following structure for each From dd0d0e036c059392e24f1a2cbbca3e4c10e9ffd8 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:14:22 -0500 Subject: [PATCH 17/47] Lemon update 2014-01-11 12:52:25 on branch trunk - In LEMON, limit the size of the grammar file to 100MB. This ensures that the program will never experience integer overflow. To be doubly sure, use calloc() instead of malloc() when allocating arrays. (user: drh) --- tools/lemon/lemon.c | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 4a0eb488e..6f32074e3 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -2581,11 +2581,10 @@ void Parse(struct lemon *gp) filesize = ftell(fp); rewind(fp); filebuf = (char *)malloc( filesize+1 ); - if( filebuf==0 ){ - ErrorMsg(ps.filename,0,"Can't allocate %d of memory to hold this file.", - filesize+1); + if( filesize>100000000 || filebuf==0 ){ + ErrorMsg(ps.filename,0,"Input file too large."); gp->errorcnt++; - fclose(fp); + fclose(fp); return; } if( fread(filebuf,1,filesize,fp)!=filesize ){ @@ -2593,7 +2592,7 @@ void Parse(struct lemon *gp) filesize); free(filebuf); gp->errorcnt++; - fclose(fp); + fclose(fp); return; } fclose(fp); @@ -4377,8 +4376,7 @@ void Strsafe_init(){ if( x1a ){ x1a->size = 1024; x1a->count = 0; - x1a->tbl = (x1node*)malloc( - (sizeof(x1node) + sizeof(x1node*))*1024 ); + x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*)); if( x1a->tbl==0 ){ free(x1a); x1a = 0; @@ -4415,8 +4413,7 @@ int Strsafe_insert(const char *data) struct s_x1 array; array.size = size = x1a->size*2; array.count = x1a->count; - array.tbl = (x1node*)malloc( - (sizeof(x1node) + sizeof(x1node*))*size ); + array.tbl = (x1node*)calloc(size, sizeof(x1node) + sizeof(x1node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x1node**)&(array.tbl[size]); for(i=0; isize = 128; x2a->count = 0; - x2a->tbl = (x2node*)malloc( - (sizeof(x2node) + sizeof(x2node*))*128 ); + x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*)); if( x2a->tbl==0 ){ free(x2a); x2a = 0; @@ -4584,8 +4580,7 @@ int Symbol_insert(struct symbol *data, const char *key) struct s_x2 array; array.size = size = x2a->size*2; array.count = x2a->count; - array.tbl = (x2node*)malloc( - (sizeof(x2node) + sizeof(x2node*))*size ); + array.tbl = (x2node*)calloc(size, sizeof(x2node) + sizeof(x2node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x2node**)&(array.tbl[size]); for(i=0; isize = 128; x3a->count = 0; - x3a->tbl = (x3node*)malloc( - (sizeof(x3node) + sizeof(x3node*))*128 ); + x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*)); if( x3a->tbl==0 ){ free(x3a); x3a = 0; @@ -4783,8 +4777,7 @@ int State_insert(struct state *data, struct config *key) struct s_x3 array; array.size = size = x3a->size*2; array.count = x3a->count; - array.tbl = (x3node*)malloc( - (sizeof(x3node) + sizeof(x3node*))*size ); + array.tbl = (x3node*)calloc(size, sizeof(x3node) + sizeof(x3node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x3node**)&(array.tbl[size]); for(i=0; icount; - array = (struct state **)malloc( sizeof(struct state *)*size ); + array = (struct state **)calloc(size, sizeof(struct state *)); if( array ){ for(i=0; itbl[i].data; } @@ -4887,8 +4880,7 @@ void Configtable_init(){ if( x4a ){ x4a->size = 64; x4a->count = 0; - x4a->tbl = (x4node*)malloc( - (sizeof(x4node) + sizeof(x4node*))*64 ); + x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*)); if( x4a->tbl==0 ){ free(x4a); x4a = 0; @@ -4925,8 +4917,7 @@ int Configtable_insert(struct config *data) struct s_x4 array; array.size = size = x4a->size*2; array.count = x4a->count; - array.tbl = (x4node*)malloc( - (sizeof(x4node) + sizeof(x4node*))*size ); + array.tbl = (x4node*)calloc(size, sizeof(x4node) + sizeof(x4node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x4node**)&(array.tbl[size]); for(i=0; i Date: Sun, 20 Mar 2016 10:15:18 -0500 Subject: [PATCH 18/47] Lemon update 2014-06-09 13:11:40 on branch trunk - Modify the %nonassoc directive in lemon so that it generates a run-time error rather than a parsing conflict. This changes is due to a bug report on the mailing list. SQLite does not use the %nonassoc directive in its grammar so this change does not affect SQLite. (user: drh) --- tools/lemon/lemon.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 6f32074e3..5deb3493b 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1086,8 +1086,7 @@ static int resolve_conflict( apx->type = SH_RESOLVED; }else{ assert( spx->prec==spy->prec && spx->assoc==NONE ); - apy->type = SRCONFLICT; - errcnt++; + apy->type = ERROR; } }else if( apx->type==REDUCE && apy->type==REDUCE ){ spx = apx->x.rp->precsym; From 9413ea15b6d58bb6aaba2633168cc3c07b8ab5b0 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:16:16 -0500 Subject: [PATCH 19/47] Lemon update 2015-01-01 19:11:22 on branch trunk - Enhance the "lemon" executable so that it ignores -f, -W, -O, and -I command-line options. This permits most of the same options that are passed to the compiler to also be harmlessly passed to lemon, and thus simplifies makefiles. (user: drh) --- tools/lemon/lemon.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 5deb3493b..7e75298c5 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1413,10 +1413,12 @@ int main(int argc, char **argv) {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."}, - {OPT_FSTR, "T", (char*)handle_T_option, "Specify a template file."}, + {OPT_FSTR, "f", 0, "Ignored. (Placeholder for -f compiler options.)"}, {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, + {OPT_FSTR, "I", 0, "Ignored. (Placeholder for '-I' compiler options.)"}, {OPT_FLAG, "m", (char*)&mhflag, "Output a makeheaders compatible file."}, {OPT_FLAG, "l", (char*)&nolinenosflag, "Do not print #line statements."}, + {OPT_FSTR, "O", 0, "Ignored. (Placeholder for '-O' compiler options.)"}, {OPT_FLAG, "p", (char*)&showPrecedenceConflict, "Show conflicts resolved by precedence rules"}, {OPT_FLAG, "q", (char*)&quiet, "(Quiet) Don't print the report file."}, @@ -1424,6 +1426,8 @@ int main(int argc, char **argv) {OPT_FLAG, "s", (char*)&statistics, "Print parser stats to standard output."}, {OPT_FLAG, "x", (char*)&version, "Print the version number."}, + {OPT_FSTR, "T", (char*)handle_T_option, "Specify a template file."}, + {OPT_FSTR, "W", 0, "Ignored. (Placeholder for '-W' compiler options.)"}, {OPT_FLAG,0,0,0} }; int i; @@ -1718,6 +1722,8 @@ static int handleflags(int i, FILE *err) errline(i,1,err); } errcnt++; + }else if( op[j].arg==0 ){ + /* Ignore this option */ }else if( op[j].type==OPT_FLAG ){ *((int*)op[j].arg) = v; }else if( op[j].type==OPT_FFLAG ){ @@ -1907,17 +1913,17 @@ void OptPrint(){ break; case OPT_INT: case OPT_FINT: - fprintf(errstream," %s=%*s %s\n",op[i].label, + fprintf(errstream," -%s%*s %s\n",op[i].label, (int)(max-lemonStrlen(op[i].label)-9),"",op[i].message); break; case OPT_DBL: case OPT_FDBL: - fprintf(errstream," %s=%*s %s\n",op[i].label, + fprintf(errstream," -%s%*s %s\n",op[i].label, (int)(max-lemonStrlen(op[i].label)-6),"",op[i].message); break; case OPT_STR: case OPT_FSTR: - fprintf(errstream," %s=%*s %s\n",op[i].label, + fprintf(errstream," -%s%*s %s\n",op[i].label, (int)(max-lemonStrlen(op[i].label)-8),"",op[i].message); break; } From 23f69b6fbc37a94120ff2fd30b0bd0a770464d0f Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:19:55 -0500 Subject: [PATCH 20/47] Lemon update 2015-09-04 18:03:45 on branch trunk - Fix over-length source code lines in Lemon. (user: drh) --- tools/lemon/lemon.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 7e75298c5..3b900adcd 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -1279,14 +1279,16 @@ void Configlist_closure(struct lemon *lemp) /* Sort the configuration list */ void Configlist_sort(){ - current = (struct config *)msort(current,&(current->next),Configcmp); + current = (struct config*)msort((char*)current,(char**)&(current->next), + Configcmp); currentend = 0; return; } /* Sort the basis configuration list */ void Configlist_sortbasis(){ - basis = (struct config *)msort(current,&(current->bp),Configcmp); + basis = (struct config *)msort((char*)current,(char**)&(current->bp), + Configcmp); basisend = 0; return; } @@ -1529,7 +1531,8 @@ int main(int argc, char **argv) if( statistics ){ printf("Parser statistics: %d terminals, %d nonterminals, %d rules\n", lem.nterminal, lem.nsymbol - lem.nterminal, lem.nrule); - printf(" %d states, %d parser table entries, %d conflicts\n", + printf(" %d states, %d parser table entries," + " %d conflicts\n", lem.nstate, lem.tablesize, lem.nconflict); } if( lem.nconflict ){ @@ -1780,7 +1783,8 @@ static int handleswitch(int i, FILE *err) dv = strtod(cp,&end); if( *end ){ if( err ){ - fprintf(err,"%sillegal character in floating-point argument.\n",emsg); + fprintf(err, + "%sillegal character in floating-point argument.\n",emsg); errline(i,((size_t)end)-(size_t)argv[i],err); } errcnt++; @@ -3157,7 +3161,8 @@ PRIVATE FILE *tplt_open(struct lemon *lemp) } in = fopen(user_templatename,"rb"); if( in==0 ){ - fprintf(stderr,"Can't open the template file \"%s\".\n",user_templatename); + fprintf(stderr,"Can't open the template file \"%s\".\n", + user_templatename); lemp->errorcnt++; return 0; } @@ -3248,7 +3253,10 @@ void emit_destructor_code( }else if( sp->destructor ){ cp = sp->destructor; fprintf(out,"{\n"); (*lineno)++; - if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,sp->destLineno,lemp->filename); } + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,sp->destLineno,lemp->filename); + } }else if( lemp->vardest ){ cp = lemp->vardest; if( cp==0 ) return; @@ -3445,13 +3453,19 @@ PRIVATE void emit_code( /* Generate code to do the reduce action */ if( rp->code ){ - if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,rp->line,lemp->filename); } + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,rp->line,lemp->filename); + } fprintf(out,"{%s",rp->code); for(cp=rp->code; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } /* End loop */ fprintf(out,"}\n"); (*lineno)++; - if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,*lineno,lemp->outname); + } } /* End if( rp->code ) */ return; From 00de97aca2ef41fe0c74a6b9638300ea171ecf04 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:23:21 -0500 Subject: [PATCH 21/47] Lemon update 2015-09-07 02:23:02 on branch trunk - Improved "Parser Statistics" output (the -s option) for the Lemon parser generator. (user: drh) --- tools/lemon/lemon.c | 77 ++++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 3b900adcd..79bdcd7a7 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -289,7 +289,8 @@ struct lemon { char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ int nconflict; /* Number of parsing conflicts */ - int tablesize; /* Size of the parse tables */ + int nactiontab; /* Number of entries in the yy_action[] table */ + int tablesize; /* Total table size of all tables in bytes */ int basisflag; /* Print only basis configurations */ int has_fallback; /* True if any %fallback is seen in the grammar */ int nolinenosflag; /* True if #line statements should not be printed */ @@ -1399,6 +1400,18 @@ static void handle_T_option(char *z){ strcpy(user_templatename, z); } +/* forward reference */ +static const char *minimum_size_type(int lwr, int upr, int *pnByte); + +/* Print a single line of the "Parser Stats" output +*/ +static void stats_line(const char *zLabel, int iValue){ + int nLabel = lemonStrlen(zLabel); + printf(" %s%.*s %5d\n", zLabel, + 35-nLabel, "................................", + iValue); +} + /* The main program. Parse the command line and do it... */ int main(int argc, char **argv) { @@ -1529,11 +1542,15 @@ int main(int argc, char **argv) if( !mhflag ) ReportHeader(&lem); } if( statistics ){ - printf("Parser statistics: %d terminals, %d nonterminals, %d rules\n", - lem.nterminal, lem.nsymbol - lem.nterminal, lem.nrule); - printf(" %d states, %d parser table entries," - " %d conflicts\n", - lem.nstate, lem.tablesize, lem.nconflict); + printf("Parser statistics:\n"); + stats_line("terminal symbols", lem.nterminal); + stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); + stats_line("total symbols", lem.nsymbol); + stats_line("rules", lem.nrule); + stats_line("states", lem.nstate); + stats_line("conflicts", lem.nconflict); + stats_line("action table entries", lem.nactiontab); + stats_line("total table size (bytes)", lem.tablesize); } if( lem.nconflict ){ fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict); @@ -3595,24 +3612,32 @@ void print_stack_union( /* ** Return the name of a C datatype able to represent values between -** lwr and upr, inclusive. +** lwr and upr, inclusive. If pnByte!=NULL then also write the sizeof +** for that type (1, 2, or 4) into *pnByte. */ -static const char *minimum_size_type(int lwr, int upr){ +static const char *minimum_size_type(int lwr, int upr, int *pnByte){ + const char *zType = "int"; + int nByte = 4; if( lwr>=0 ){ if( upr<=255 ){ - return "unsigned char"; + zType = "unsigned char"; + nByte = 1; }else if( upr<65535 ){ - return "unsigned short int"; + zType = "unsigned short int"; + nByte = 2; }else{ - return "unsigned int"; + zType = "unsigned int"; + nByte = 4; } }else if( lwr>=-127 && upr<=127 ){ - return "signed char"; + zType = "signed char"; + nByte = 1; }else if( lwr>=-32767 && upr<32767 ){ - return "short"; - }else{ - return "int"; + zType = "short"; + nByte = 2; } + if( pnByte ) *pnByte = nByte; + return zType; } /* @@ -3676,7 +3701,9 @@ void ReportTable( struct action *ap; struct rule *rp; struct acttab *pActtab; - int i, j, k, n; + int i, j, k, n, sz; + int szActionType; /* sizeof(YYACTIONTYPE) */ + int szCodeType; /* sizeof(YYCODETYPE) */ const char *name; int mnTknOfst, mxTknOfst; int mnNtOfst, mxNtOfst; @@ -3717,10 +3744,10 @@ void ReportTable( /* Generate the defines */ fprintf(out,"#define YYCODETYPE %s\n", - minimum_size_type(0, lemp->nsymbol+1)); lineno++; + minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++; fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; fprintf(out,"#define YYACTIONTYPE %s\n", - minimum_size_type(0, lemp->nstate+lemp->nrule+5)); lineno++; + minimum_size_type(0, lemp->nstate+lemp->nrule+5, &szActionType)); lineno++; if( lemp->wildcard ){ fprintf(out,"#define YYWILDCARD %d\n", lemp->wildcard->index); lineno++; @@ -3835,7 +3862,8 @@ void ReportTable( free(ax); /* Output the yy_action table */ - n = acttab_size(pActtab); + lemp->nactiontab = n = acttab_size(pActtab); + lemp->tablesize += n*szActionType; fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; for(i=j=0; itablesize += n*szCodeType; fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++; for(i=j=0; itablesize += n*sz; for(i=j=0; isorted[i]; @@ -3901,7 +3931,8 @@ void ReportTable( fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; fprintf(out, "static const %s yy_reduce_ofst[] = {\n", - minimum_size_type(mnNtOfst-1, mxNtOfst)); lineno++; + minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++; + lemp->tablesize += n*sz; for(i=j=0; isorted[i]; @@ -3921,6 +3952,7 @@ void ReportTable( /* Output the default action table */ fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; n = lemp->nstate; + lemp->tablesize += n*szActionType; for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); @@ -3939,7 +3971,8 @@ void ReportTable( */ if( lemp->has_fallback ){ int mx = lemp->nterminal - 1; - while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } + while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } + lemp->tablesize += (mx+1)*szCodeType; for(i=0; i<=mx; i++){ struct symbol *p = lemp->symbols[i]; if( p->fallback==0 ){ From 2b5cef0c177f7900c68c4dee512da8009308f949 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:24:15 -0500 Subject: [PATCH 22/47] Lemon update 2015-09-07 14:22:24 on branch trunk - In the "parse.out" output file from Lemon, show addition the complete text of rules on reduce actions. (user: drh) --- tools/lemon/lemon.c | 68 ++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 79bdcd7a7..e59cb4030 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -244,7 +244,7 @@ struct state { struct action *ap; /* Array of actions for this state */ int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ - int iDflt; /* Default action */ + int iDflt; /* Default action is reduce by this rule */ }; #define NO_OFFSET (-2147483647) @@ -2884,15 +2884,14 @@ void Reprint(struct lemon *lemp) } } -void ConfigPrint(FILE *fp, struct config *cfp) -{ - struct rule *rp; +/* Print a single rule. +*/ +void RulePrint(FILE *fp, struct rule *rp, int iCursor){ struct symbol *sp; int i, j; - rp = cfp->rp; fprintf(fp,"%s ::=",rp->lhs->name); for(i=0; i<=rp->nrhs; i++){ - if( i==cfp->dot ) fprintf(fp," *"); + if( i==iCursor ) fprintf(fp," *"); if( i==rp->nrhs ) break; sp = rp->rhs[i]; if( sp->type==MULTITERMINAL ){ @@ -2906,6 +2905,12 @@ void ConfigPrint(FILE *fp, struct config *cfp) } } +/* Print the rule for a configuration. +*/ +void ConfigPrint(FILE *fp, struct config *cfp){ + RulePrint(fp, cfp->rp, cfp->dot); +} + /* #define TEST */ #if 0 /* Print a set */ @@ -2945,15 +2950,29 @@ char *tag; /* Print an action to the given file descriptor. Return FALSE if ** nothing was actually printed. */ -int PrintAction(struct action *ap, FILE *fp, int indent){ +int PrintAction( + struct action *ap, /* The action to print */ + FILE *fp, /* Print the action here */ + int indent, /* Indent by this amount */ + struct rule **apRule /* All rules by index */ +){ int result = 1; switch( ap->type ){ - case SHIFT: - fprintf(fp,"%*s shift %d",indent,ap->sp->name,ap->x.stp->statenum); + case SHIFT: { + struct state *stp = ap->x.stp; + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); + if( stp->nTknAct==0 && stp->nNtAct==0 && apRule ){ + fprintf(fp,"then reduce %d: ", stp->iDflt); + RulePrint(fp, apRule[stp->iDflt], -1); + } break; - case REDUCE: - fprintf(fp,"%*s reduce %d",indent,ap->sp->name,ap->x.rp->index); + } + case REDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); + if( apRule ) RulePrint(fp, apRule[rp->index], -1); break; + } case ACCEPT: fprintf(fp,"%*s accept",indent,ap->sp->name); break; @@ -2962,16 +2981,16 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ break; case SRCONFLICT: case RRCONFLICT: - fprintf(fp,"%*s reduce %-3d ** Parsing conflict **", + fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.rp->index); break; case SSCONFLICT: - fprintf(fp,"%*s shift %-3d ** Parsing conflict **", + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s shift %-3d -- dropped by precedence", + fprintf(fp,"%*s shift %-7d -- dropped by precedence", indent,ap->sp->name,ap->x.stp->statenum); }else{ result = 0; @@ -2979,7 +2998,7 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ break; case RD_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s reduce %-3d -- dropped by precedence", + fprintf(fp,"%*s reduce %-7d -- dropped by precedence", indent,ap->sp->name,ap->x.rp->index); }else{ result = 0; @@ -3000,7 +3019,17 @@ void ReportOutput(struct lemon *lemp) struct config *cfp; struct action *ap; FILE *fp; + struct rule **apRule; + apRule = malloc( sizeof(apRule[0])*(lemp->nrule+1) ); + if( apRule ){ + struct rule *x; + memset(apRule, 0, sizeof(apRule[0])*(lemp->nrule+1) ); + for(x=lemp->rule; x; x=x->next){ + assert( x->index>=0 && x->index<(lemp->nrule+1) ); + apRule[x->index] = x; + } + } fp = file_open(lemp,".out","wb"); if( fp==0 ) return; for(i=0; instate; i++){ @@ -3028,7 +3057,7 @@ void ReportOutput(struct lemon *lemp) } fprintf(fp,"\n"); for(ap=stp->ap; ap; ap=ap->next){ - if( PrintAction(ap,fp,30) ) fprintf(fp,"\n"); + if( PrintAction(ap,fp,30,apRule) ) fprintf(fp,"\n"); } fprintf(fp,"\n"); } @@ -3054,6 +3083,7 @@ void ReportOutput(struct lemon *lemp) fprintf(fp, "\n"); } fclose(fp); + free(apRule); return; } @@ -3956,7 +3986,7 @@ void ReportTable( for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); - fprintf(out, " %4d,", stp->iDflt); + fprintf(out, " %4d,", stp->iDflt+n); if( j==9 || i==n-1 ){ fprintf(out, "\n"); lineno++; j = 0; @@ -4279,7 +4309,7 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ stp = lemp->sorted[i]; stp->nTknAct = stp->nNtAct = 0; - stp->iDflt = lemp->nstate + lemp->nrule; + stp->iDflt = lemp->nrule; stp->iTknOfst = NO_OFFSET; stp->iNtOfst = NO_OFFSET; for(ap=stp->ap; ap; ap=ap->next){ @@ -4289,7 +4319,7 @@ void ResortStates(struct lemon *lemp) }else if( ap->sp->indexnsymbol ){ stp->nNtAct++; }else{ - stp->iDflt = compute_action(lemp, ap); + stp->iDflt = compute_action(lemp, ap) - lemp->nstate; } } } From 3d5867d29e8932d238084e5c66dead1b4502f82f Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:27:12 -0500 Subject: [PATCH 23/47] Lemon update 2015-09-07 18:23:37 on branch lemon-update - For the Lemon-generated parser, add a new action type SHIFTREDUCE and use it to further compress the parser tables and improve parser performance. (user: drh) --- tools/lemon/lemon.c | 180 ++++++++++++++++++++++++++----------------- tools/lemon/lempar.c | 99 +++++++++++++++--------- 2 files changed, 173 insertions(+), 106 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index e59cb4030..6d7ae8acb 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -220,7 +220,8 @@ enum e_action { RRCONFLICT, /* Was a reduce, but part of a conflict */ SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ - NOT_USED /* Deleted by compression */ + NOT_USED, /* Deleted by compression */ + SHIFTREDUCE /* Shift first, then reduce */ }; /* Every shift or reduce operation is stored as one of the following */ @@ -244,7 +245,9 @@ struct state { struct action *ap; /* Array of actions for this state */ int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ - int iDflt; /* Default action is reduce by this rule */ + int iDfltReduce; /* Default action is to REDUCE by this rule */ + struct rule *pDfltReduce;/* The default REDUCE rule. */ + int autoReduce; /* True if this is an auto-reduce state */ }; #define NO_OFFSET (-2147483647) @@ -264,6 +267,7 @@ struct lemon { struct state **sorted; /* Table of states sorted by state number */ struct rule *rule; /* List of all rules */ int nstate; /* Number of states */ + int nxstate; /* nstate with tail degenerate states removed */ int nrule; /* Number of rules */ int nsymbol; /* Number of terminal and nonterminal symbols */ int nterminal; /* Number of terminal symbols */ @@ -388,7 +392,7 @@ struct action *ap2; if( rc==0 ){ rc = (int)ap1->type - (int)ap2->type; } - if( rc==0 && ap1->type==REDUCE ){ + if( rc==0 && (ap1->type==REDUCE || ap1->type==SHIFTREDUCE) ){ rc = ap1->x.rp->index - ap2->x.rp->index; } if( rc==0 ){ @@ -1547,7 +1551,7 @@ int main(int argc, char **argv) stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); stats_line("total symbols", lem.nsymbol); stats_line("rules", lem.nrule); - stats_line("states", lem.nstate); + stats_line("states", lem.nxstate); stats_line("conflicts", lem.nconflict); stats_line("action table entries", lem.nactiontab); stats_line("total table size (bytes)", lem.tablesize); @@ -2953,24 +2957,25 @@ char *tag; int PrintAction( struct action *ap, /* The action to print */ FILE *fp, /* Print the action here */ - int indent, /* Indent by this amount */ - struct rule **apRule /* All rules by index */ + int indent /* Indent by this amount */ ){ int result = 1; switch( ap->type ){ case SHIFT: { struct state *stp = ap->x.stp; - fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); - if( stp->nTknAct==0 && stp->nNtAct==0 && apRule ){ - fprintf(fp,"then reduce %d: ", stp->iDflt); - RulePrint(fp, apRule[stp->iDflt], -1); - } + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); break; } case REDUCE: { struct rule *rp = ap->x.rp; - fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); - if( apRule ) RulePrint(fp, apRule[rp->index], -1); + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); + RulePrint(fp, rp, -1); + break; + } + case SHIFTREDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index); + RulePrint(fp, rp, -1); break; } case ACCEPT: @@ -2981,16 +2986,16 @@ int PrintAction( break; case SRCONFLICT: case RRCONFLICT: - fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", + fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.rp->index); break; case SSCONFLICT: - fprintf(fp,"%*s shift %-7d ** Parsing conflict **", + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s shift %-7d -- dropped by precedence", + fprintf(fp,"%*s shift %-7d -- dropped by precedence", indent,ap->sp->name,ap->x.stp->statenum); }else{ result = 0; @@ -3011,7 +3016,7 @@ int PrintAction( return result; } -/* Generate the "y.output" log file */ +/* Generate the "*.out" log file */ void ReportOutput(struct lemon *lemp) { int i; @@ -3019,20 +3024,10 @@ void ReportOutput(struct lemon *lemp) struct config *cfp; struct action *ap; FILE *fp; - struct rule **apRule; - apRule = malloc( sizeof(apRule[0])*(lemp->nrule+1) ); - if( apRule ){ - struct rule *x; - memset(apRule, 0, sizeof(apRule[0])*(lemp->nrule+1) ); - for(x=lemp->rule; x; x=x->next){ - assert( x->index>=0 && x->index<(lemp->nrule+1) ); - apRule[x->index] = x; - } - } fp = file_open(lemp,".out","wb"); if( fp==0 ) return; - for(i=0; instate; i++){ + for(i=0; inxstate; i++){ stp = lemp->sorted[i]; fprintf(fp,"State %d:\n",stp->statenum); if( lemp->basisflag ) cfp=stp->bp; @@ -3057,7 +3052,7 @@ void ReportOutput(struct lemon *lemp) } fprintf(fp,"\n"); for(ap=stp->ap; ap; ap=ap->next){ - if( PrintAction(ap,fp,30,apRule) ) fprintf(fp,"\n"); + if( PrintAction(ap,fp,30) ) fprintf(fp,"\n"); } fprintf(fp,"\n"); } @@ -3083,7 +3078,6 @@ void ReportOutput(struct lemon *lemp) fprintf(fp, "\n"); } fclose(fp); - free(apRule); return; } @@ -3145,10 +3139,11 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) { int act; switch( ap->type ){ - case SHIFT: act = ap->x.stp->statenum; break; - case REDUCE: act = ap->x.rp->index + lemp->nstate; break; - case ERROR: act = lemp->nstate + lemp->nrule; break; - case ACCEPT: act = lemp->nstate + lemp->nrule + 1; break; + case SHIFT: act = ap->x.stp->statenum; break; + case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate; break; + case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break; + case ERROR: act = lemp->nstate + lemp->nrule*2; break; + case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1; break; default: act = -1; break; } return act; @@ -3777,7 +3772,7 @@ void ReportTable( minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++; fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; fprintf(out,"#define YYACTIONTYPE %s\n", - minimum_size_type(0, lemp->nstate+lemp->nrule+5, &szActionType)); lineno++; + minimum_size_type(0,lemp->nstate+lemp->nrule*2+5,&szActionType)); lineno++; if( lemp->wildcard ){ fprintf(out,"#define YYWILDCARD %d\n", lemp->wildcard->index); lineno++; @@ -3814,36 +3809,24 @@ void ReportTable( if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } - fprintf(out,"#define YYNSTATE %d\n",lemp->nstate); lineno++; - fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; if( lemp->errsym->useCnt ){ - fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; - fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; + fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; + fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; } if( lemp->has_fallback ){ fprintf(out,"#define YYFALLBACK 1\n"); lineno++; } - tplt_xfer(lemp->name,in,out,&lineno); - /* Generate the action table and its associates: - ** - ** yy_action[] A single table containing all actions. - ** yy_lookahead[] A table containing the lookahead for each entry in - ** yy_action. Used to detect hash collisions. - ** yy_shift_ofst[] For each state, the offset into yy_action for - ** shifting terminals. - ** yy_reduce_ofst[] For each state, the offset into yy_action for - ** shifting non-terminals after a reduce. - ** yy_default[] Default action for each state. + /* Compute the action table, but do not output it yet. The action + ** table must be computed before generating the YYNSTATE macro because + ** we need to know how many states can be eliminated. */ - - /* Compute the actions on all states and count them up */ - ax = (struct axset *) calloc(lemp->nstate*2 , sizeof(ax[0])); + ax = (struct axset *) calloc(lemp->nxstate*2 , sizeof(ax[0])); if( ax==0 ){ fprintf(stderr,"malloc failed\n"); exit(1); } - for(i=0; instate; i++){ + for(i=0; inxstate; i++){ stp = lemp->sorted[i]; ax[i*2].stp = stp; ax[i*2].isTkn = 1; @@ -3854,15 +3837,12 @@ void ReportTable( } mxTknOfst = mnTknOfst = 0; mxNtOfst = mnNtOfst = 0; - - /* Compute the action table. In order to try to keep the size of the - ** action table to a minimum, the heuristic of placing the largest action - ** sets first is used. - */ - for(i=0; instate*2; i++) ax[i].iOrder = i; - qsort(ax, lemp->nstate*2, sizeof(ax[0]), axset_compare); + /* In an effort to minimize the action table size, use the heuristic + ** of placing the largest action sets first */ + for(i=0; inxstate*2; i++) ax[i].iOrder = i; + qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare); pActtab = acttab_alloc(); - for(i=0; instate*2 && ax[i].nAction>0; i++){ + for(i=0; inxstate*2 && ax[i].nAction>0; i++){ stp = ax[i].stp; if( ax[i].isTkn ){ for(ap=stp->ap; ap; ap=ap->next){ @@ -3891,6 +3871,34 @@ void ReportTable( } free(ax); + /* Finish rendering the constants now that the action table has + ** been computed */ + fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; + fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nstate-1); lineno++; + fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",lemp->nstate); lineno++; + i = lemp->nstate + lemp->nrule; + fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_MIN_REDUCE %d\n", i); lineno++; + i = lemp->nstate + lemp->nrule*2; + fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_ERROR_ACTION %d\n", i); lineno++; + fprintf(out,"#define YY_ACCEPT_ACTION %d\n", i+1); lineno++; + fprintf(out,"#define YY_NO_ACTION %d\n", i+2); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Now output the action table and its associates: + ** + ** yy_action[] A single table containing all actions. + ** yy_lookahead[] A table containing the lookahead for each entry in + ** yy_action. Used to detect hash collisions. + ** yy_shift_ofst[] For each state, the offset into yy_action for + ** shifting terminals. + ** yy_reduce_ofst[] For each state, the offset into yy_action for + ** shifting non-terminals after a reduce. + ** yy_default[] Default action for each state. + */ + /* Output the yy_action table */ lemp->nactiontab = n = acttab_size(pActtab); lemp->tablesize += n*szActionType; @@ -3929,7 +3937,7 @@ void ReportTable( /* Output the yy_shift_ofst[] table */ fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++; - n = lemp->nstate; + n = lemp->nxstate; while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; @@ -3955,7 +3963,7 @@ void ReportTable( /* Output the yy_reduce_ofst[] table */ fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++; - n = lemp->nstate; + n = lemp->nxstate; while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; @@ -3981,12 +3989,12 @@ void ReportTable( /* Output the default action table */ fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; - n = lemp->nstate; + n = lemp->nxstate; lemp->tablesize += n*szActionType; for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); - fprintf(out, " %4d,", stp->iDflt+n); + fprintf(out, " %4d,", stp->iDfltReduce+lemp->nstate+lemp->nrule); if( j==9 || i==n-1 ){ fprintf(out, "\n"); lineno++; j = 0; @@ -4221,7 +4229,7 @@ void CompressTables(struct lemon *lemp) struct state *stp; struct action *ap, *ap2; struct rule *rp, *rp2, *rbest; - int nbest, n; + int nbest, n, nshift; int i; int usesWildcard; @@ -4269,6 +4277,32 @@ void CompressTables(struct lemon *lemp) if( ap->type==REDUCE && ap->x.rp==rbest ) ap->type = NOT_USED; } stp->ap = Action_sort(stp->ap); + + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==SHIFT ) break; + if( ap->type==REDUCE && ap->x.rp!=rbest ) break; + } + if( ap==0 ){ + stp->autoReduce = 1; + stp->pDfltReduce = rbest; + } + } + + /* Make a second pass over all states and actions. Convert + ** every action that is a SHIFT to an autoReduce state into + ** a SHIFTREDUCE action. + */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=ap->next){ + struct state *pNextState; + if( ap->type!=SHIFT ) continue; + pNextState = ap->x.stp; + if( pNextState->autoReduce && pNextState->pDfltReduce!=0 ){ + ap->type = SHIFTREDUCE; + ap->x.rp = pNextState->pDfltReduce; + } + } } } @@ -4309,17 +4343,19 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ stp = lemp->sorted[i]; stp->nTknAct = stp->nNtAct = 0; - stp->iDflt = lemp->nrule; + stp->iDfltReduce = lemp->nrule; /* Init dflt action to "syntax error" */ stp->iTknOfst = NO_OFFSET; stp->iNtOfst = NO_OFFSET; for(ap=stp->ap; ap; ap=ap->next){ - if( compute_action(lemp,ap)>=0 ){ + int iAction = compute_action(lemp,ap); + if( iAction>=0 ){ if( ap->sp->indexnterminal ){ stp->nTknAct++; }else if( ap->sp->indexnsymbol ){ stp->nNtAct++; }else{ - stp->iDflt = compute_action(lemp, ap) - lemp->nstate; + assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp ); + stp->iDfltReduce = iAction - lemp->nstate - lemp->nrule; } } } @@ -4329,6 +4365,10 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ lemp->sorted[i]->statenum = i; } + lemp->nxstate = lemp->nstate; + while( lemp->nxstate>1 && lemp->sorted[lemp->nxstate-1]->autoReduce ){ + lemp->nxstate--; + } } diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 7979a7d2f..90c410e62 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -59,15 +59,19 @@ ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ %% -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -94,16 +98,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -497,9 +505,9 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ } /* -** Perform a shift action. +** Perform a shift action. Return the number of errors. */ -static void yy_shift( +static int yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -515,14 +523,14 @@ static void yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return; + return 1; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return; + return 1; } } #endif @@ -533,13 +541,18 @@ static void yy_shift( #ifndef NDEBUG if( yyTraceFILE && yypParser->yyidx>0 ){ int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," (%d)%s",yypParser->yystack[i].stateno,yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } } #endif + return 0; } /* The following table contains information about every rule that @@ -572,8 +585,9 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -609,9 +623,8 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least + if( yyact < YY_MAX_SHIFTREDUCE ){ + /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). ** That gives a significant speed improvement. */ @@ -621,13 +634,19 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; - }else +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); + } #endif - { - yy_shift(yypParser,yyact,yygoto,&yygotominor); + }else{ + if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; + } + if( yyact>=YY_MIN_SHIFTREDUCE ){ + yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); } }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); yy_accept(yypParser); } } @@ -747,13 +766,16 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyactyyerrcnt--; - yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ + yypParser->yyerrcnt--; + yymajor = YYNOCODE; + if( yyact > YY_MAX_SHIFT ){ + yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); + } + } + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ #ifdef YYERRORSYMBOL int yymx; @@ -803,7 +825,7 @@ void Parse( yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -853,5 +875,10 @@ void Parse( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + } +#endif return; } From fd3507c4c351d4cc15e0ee06f0e6e78a9e309a82 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:02:31 -0500 Subject: [PATCH 24/47] Lemon update 2015-09-07 19:52:55 on branch lemon-update - Change the parser engine so that it (once again) waits for a lookahead token before reducing, even in a SHIFTREDUCE action. (user: drh) --- tools/lemon/lempar.c | 78 +++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 90c410e62..561fbdc87 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -170,9 +170,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -400,7 +404,8 @@ static int yy_find_shift_action( ){ int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - + + if( stateno>=YY_MIN_REDUCE ) return stateno; if( stateno>YY_SHIFT_COUNT || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; @@ -504,10 +509,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + /* ** Perform a shift action. Return the number of errors. */ -static int yy_shift( +static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -523,14 +550,14 @@ static int yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } } #endif @@ -538,21 +565,7 @@ static int yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); - } - } -#endif - return 0; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -623,7 +636,8 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YY_MAX_SHIFTREDUCE ){ + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). @@ -634,16 +648,9 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); - } -#endif + yyTraceShift(yypParser, yyact); }else{ - if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; - } - if( yyact>=YY_MIN_SHIFTREDUCE ){ - yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); + yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ assert( yyact == YY_ACCEPT_ACTION ); @@ -767,13 +774,10 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ - yypParser->yyerrcnt--; - yymajor = YYNOCODE; - if( yyact > YY_MAX_SHIFT ){ - yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); - } - } + if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,&yyminorunion); + yypParser->yyerrcnt--; + yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ From 123db1926c65ac6b8c9a25f02f02510af963c6d1 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:04:21 -0500 Subject: [PATCH 25/47] Lemon update 2015-09-07 20:02:39 on branch lemon-update - Fix an unreachable branch in the new parse automaton. (user: drh) --- tools/lemon/lempar.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 561fbdc87..13ea60068 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -406,10 +406,9 @@ static int yy_find_shift_action( int stateno = pParser->yystack[pParser->yyidx].stateno; if( stateno>=YY_MIN_REDUCE ) return stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } + assert( stateno <= YY_SHIFT_COUNT ); + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ From daa68a0c88c8fd120fe169193387fe01841624d4 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:28:48 -0500 Subject: [PATCH 26/47] Lemon update 2015-09-07 23:40:42 on branch trunk - Minor tweaks to Lemon. (user: drh) --- tools/lemon/lemon.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 6d7ae8acb..f560f1096 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3687,7 +3687,7 @@ static int axset_compare(const void *a, const void *b){ int c; c = p2->nAction - p1->nAction; if( c==0 ){ - c = p2->iOrder - p1->iOrder; + c = p1->iOrder - p2->iOrder; } assert( c!=0 || p1==p2 ); return c; @@ -3868,6 +3868,16 @@ void ReportTable( if( stp->iNtOfstiNtOfst; if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst; } +#if 0 /* Uncomment for a trace of how the yy_action[] table fills out */ + { int jj, nn; + for(jj=nn=0; jjnAction; jj++){ + if( pActtab->aAction[jj].action<0 ) nn++; + } + printf("%4d: State %3d %s n: %2d size: %5d freespace: %d\n", + i, stp->statenum, ax[i].isTkn ? "Token" : "Var ", + ax[i].nAction, pActtab->nAction, nn); + } +#endif } free(ax); @@ -3875,7 +3885,7 @@ void ReportTable( ** been computed */ fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; - fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nstate-1); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",lemp->nstate); lineno++; i = lemp->nstate + lemp->nrule; fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; From 5286a7fef3c900c06e26a9b7ae9e5e32c6b91d5f Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:31:37 -0500 Subject: [PATCH 27/47] Lemon update 2015-10-29 13:48:15 on branch trunk - Fix uses of ctype functions (ex: isspace()) on signed characters in test programs and in some obscure extensions. No changes to the core. (user: drh) --- tools/lemon/lemon.c | 82 +++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index f560f1096..a70220572 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -17,6 +17,14 @@ #include #include +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISALNUM(X) isalnum((unsigned char)(X)) +#define ISALPHA(X) isalpha((unsigned char)(X)) +#define ISUPPER(X) isupper((unsigned char)(X)) +#define ISLOWER(X) islower((unsigned char)(X)) + + #ifndef __WIN32__ # if defined(_WIN32) || defined(WIN32) # define __WIN32__ @@ -1494,7 +1502,7 @@ int main(int argc, char **argv) while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); lem.nsymbol = i - 1; - for(i=1; isupper(lem.symbols[i]->name[0]); i++); + for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++); lem.nterminal = i; /* Generate a reprint of the grammar, if requested on the command line */ @@ -2028,7 +2036,7 @@ static void parseonetoken(struct pstate *psp) case WAITING_FOR_DECL_OR_RULE: if( x[0]=='%' ){ psp->state = WAITING_FOR_DECL_KEYWORD; - }else if( islower(x[0]) ){ + }else if( ISLOWER(x[0]) ){ psp->lhs = Symbol_new(x); psp->nrhs = 0; psp->lhsalias = 0; @@ -2058,7 +2066,7 @@ to follow the previous rule."); } break; case PRECEDENCE_MARK_1: - if( !isupper(x[0]) ){ + if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "The precedence symbol must be a terminal."); psp->errorcnt++; @@ -2098,7 +2106,7 @@ to follow the previous rule."); } break; case LHS_ALIAS_1: - if( isalpha(x[0]) ){ + if( ISALPHA(x[0]) ){ psp->lhsalias = x; psp->state = LHS_ALIAS_2; }else{ @@ -2163,11 +2171,11 @@ to follow the previous rule."); }else{ psp->lastrule->next = rp; psp->lastrule = rp; - } + } psp->prevrule = rp; - } + } psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( isalpha(x[0]) ){ + }else if( ISALPHA(x[0]) ){ if( psp->nrhs>=MAXRHS ){ ErrorMsg(psp->filename,psp->tokenlineno, "Too many symbols on RHS of rule beginning at \"%s\".", @@ -2195,7 +2203,7 @@ to follow the previous rule."); msp->subsym = (struct symbol **) realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); - if( islower(x[1]) || islower(msp->subsym[0]->name[0]) ){ + if( ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "Cannot form a compound containing a non-terminal"); psp->errorcnt++; @@ -2210,7 +2218,7 @@ to follow the previous rule."); } break; case RHS_ALIAS_1: - if( isalpha(x[0]) ){ + if( ISALPHA(x[0]) ){ psp->alias[psp->nrhs-1] = x; psp->state = RHS_ALIAS_2; }else{ @@ -2232,7 +2240,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DECL_KEYWORD: - if( isalpha(x[0]) ){ + if( ISALPHA(x[0]) ){ psp->declkeyword = x; psp->declargslot = 0; psp->decllinenoslot = 0; @@ -2312,7 +2320,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DESTRUCTOR_SYMBOL: - if( !isalpha(x[0]) ){ + if( !ISALPHA(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "Symbol name missing after %%destructor keyword"); psp->errorcnt++; @@ -2326,7 +2334,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DATATYPE_SYMBOL: - if( !isalpha(x[0]) ){ + if( !ISALPHA(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "Symbol name missing after %%type keyword"); psp->errorcnt++; @@ -2351,7 +2359,7 @@ to follow the previous rule."); case WAITING_FOR_PRECEDENCE_SYMBOL: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( isupper(x[0]) ){ + }else if( ISUPPER(x[0]) ){ struct symbol *sp; sp = Symbol_new(x); if( sp->prec>=0 ){ @@ -2369,7 +2377,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DECL_ARG: - if( x[0]=='{' || x[0]=='\"' || isalnum(x[0]) ){ + if( x[0]=='{' || x[0]=='\"' || ISALNUM(x[0]) ){ const char *zOld, *zNew; char *zBuf, *z; int nOld, n, nLine, nNew, nBack; @@ -2430,7 +2438,7 @@ to follow the previous rule."); case WAITING_FOR_FALLBACK_ID: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( !isupper(x[0]) ){ + }else if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%fallback argument \"%s\" should be a token", x); psp->errorcnt++; @@ -2451,7 +2459,7 @@ to follow the previous rule."); case WAITING_FOR_WILDCARD_ID: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( !isupper(x[0]) ){ + }else if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%wildcard argument \"%s\" should be a token", x); psp->errorcnt++; @@ -2467,7 +2475,7 @@ to follow the previous rule."); } break; case WAITING_FOR_CLASS_ID: - if( !islower(x[0]) ){ + if( !ISLOWER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%token_class must be followed by an identifier: ", x); psp->errorcnt++; @@ -2486,12 +2494,12 @@ to follow the previous rule."); case WAITING_FOR_CLASS_TOKEN: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){ + }else if( ISUPPER(x[0]) || ((x[0]=='|' || x[0]=='/') && ISUPPER(x[1])) ){ struct symbol *msp = psp->tkclass; msp->nsubsym++; msp->subsym = (struct symbol **) realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); - if( !isupper(x[0]) ) x++; + if( !ISUPPER(x[0]) ) x++; msp->subsym[msp->nsubsym-1] = Symbol_new(x); }else{ ErrorMsg(psp->filename, psp->tokenlineno, @@ -2524,7 +2532,7 @@ static void preprocess_input(char *z){ for(i=0; z[i]; i++){ if( z[i]=='\n' ) lineno++; if( z[i]!='%' || (i>0 && z[i-1]!='\n') ) continue; - if( strncmp(&z[i],"%endif",6)==0 && isspace(z[i+6]) ){ + if( strncmp(&z[i],"%endif",6)==0 && ISSPACE(z[i+6]) ){ if( exclude ){ exclude--; if( exclude==0 ){ @@ -2532,13 +2540,13 @@ static void preprocess_input(char *z){ } } for(j=i; z[j] && z[j]!='\n'; j++) z[j] = ' '; - }else if( (strncmp(&z[i],"%ifdef",6)==0 && isspace(z[i+6])) - || (strncmp(&z[i],"%ifndef",7)==0 && isspace(z[i+7])) ){ + }else if( (strncmp(&z[i],"%ifdef",6)==0 && ISSPACE(z[i+6])) + || (strncmp(&z[i],"%ifndef",7)==0 && ISSPACE(z[i+7])) ){ if( exclude ){ exclude++; }else{ - for(j=i+7; isspace(z[j]); j++){} - for(n=0; z[j+n] && !isspace(z[j+n]); n++){} + for(j=i+7; ISSPACE(z[j]); j++){} + for(n=0; z[j+n] && !ISSPACE(z[j+n]); n++){} exclude = 1; for(k=0; kiStart ) fprintf(out,"%.*s",i-iStart,&line[iStart]); fprintf(out,"%s",name); @@ -3413,9 +3421,9 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ /* This const cast is wrong but harmless, if we're careful. */ for(cp=(char *)rp->code; *cp; cp++){ - if( isalpha(*cp) && (cp==rp->code || (!isalnum(cp[-1]) && cp[-1]!='_')) ){ + if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){ char saved; - for(xp= &cp[1]; isalnum(*xp) || *xp=='_'; xp++); + for(xp= &cp[1]; ISALNUM(*xp) || *xp=='_'; xp++); saved = *xp; *xp = 0; if( rp->lhsalias && strcmp(cp,rp->lhsalias)==0 ){ @@ -3579,9 +3587,9 @@ void print_stack_union( cp = sp->datatype; if( cp==0 ) cp = lemp->vartype; j = 0; - while( isspace(*cp) ) cp++; + while( ISSPACE(*cp) ) cp++; while( *cp ) stddt[j++] = *cp++; - while( j>0 && isspace(stddt[j-1]) ) j--; + while( j>0 && ISSPACE(stddt[j-1]) ) j--; stddt[j] = 0; if( lemp->tokentype && strcmp(stddt, lemp->tokentype)==0 ){ sp->dtnum = 0; @@ -3792,8 +3800,8 @@ void ReportTable( if( lemp->arg && lemp->arg[0] ){ size_t i; i = lemonStrlen(lemp->arg); - while( i>=1 && isspace(lemp->arg[i-1]) ) i--; - while( i>=1 && (isalnum(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; + while( i>=1 && ISSPACE(lemp->arg[i-1]) ) i--; + while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++; fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++; fprintf(out,"#define %sARG_FETCH %s = yypParser->%s\n", @@ -4603,7 +4611,7 @@ struct symbol *Symbol_new(const char *x) sp = (struct symbol *)calloc(1, sizeof(struct symbol) ); MemoryCheck(sp); sp->name = Strsafe(x); - sp->type = isupper(*x) ? TERMINAL : NONTERMINAL; + sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; sp->rule = 0; sp->fallback = 0; sp->prec = -1; From bcdef59c3c8c222c23b8184c48cedcc5cc88357c Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:05:07 -0500 Subject: [PATCH 28/47] Lemon update 2015-11-09 14:11:37 on branch trunk - Size reduction and performance improvement in the stack-popping logic of the Lemon-generated parser. (user: drh) --- tools/lemon/lempar.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 13ea60068..dbdccc0e7 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -334,25 +334,19 @@ static void yy_destructor( ** ** If there is a destructor routine associated with the token which ** is popped from the stack, then call it. -** -** Return the major token number for the symbol popped. */ -static int yy_pop_parser_stack(yyParser *pParser){ - YYCODETYPE yymajor; - yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; - - if( pParser->yyidx<0 ) return 0; +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yyidx>=0 ); + yytos = &pParser->yystack[pParser->yyidx--]; #ifndef NDEBUG - if( yyTraceFILE && pParser->yyidx>=0 ){ + if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sPopping %s\n", yyTracePrompt, yyTokenName[yytos->major]); } #endif - yymajor = yytos->major; - yy_destructor(pParser, yymajor, &yytos->minor); - pParser->yyidx--; - return yymajor; + yy_destructor(pParser, yytos->major, &yytos->minor); } /* From 59b14415ffe5cf62c3b1ed2e88714d9d378960f9 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:08:36 -0500 Subject: [PATCH 29/47] Lemon update 2015-11-09 15:06:26 on branch trunk - Avoid recursion in the yy_find_shift_action() routine of the Lemon-generated parser, so that routine can be inlined, for a size reduction and performance increase. (user: drh) --- tools/lemon/lempar.c | 92 ++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index dbdccc0e7..799eb24b1 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -398,55 +398,60 @@ static int yy_find_shift_action( ){ int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - - if( stateno>=YY_MIN_REDUCE ) return stateno; + + if( stateno>=YY_MIN_REDUCE ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); - i = yy_shift_ofst[stateno]; - if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ + do{ + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ #ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - return yy_find_shift_action(pParser, iFallback); - } -#endif -#ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); + fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; } -#endif /* NDEBUG */ - return yy_action[j]; } - } #endif /* YYWILDCARD */ + } + return yy_default[stateno]; + }else{ + return yy_action[i]; } - return yy_default[stateno]; - }else{ - return yy_action[i]; - } + }while(1); } /* @@ -625,6 +630,7 @@ static void yy_reduce( */ %% }; + assert( yyruleno>=0 && yyrulenoyyidx -= yysize; @@ -732,7 +738,9 @@ void Parse( ){ YYMINORTYPE yyminorunion; int yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ +#endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif @@ -755,7 +763,9 @@ void Parse( yypParser->yystack[0].major = 0; } yyminorunion.yy0 = yyminor; +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); +#endif ParseARG_STORE; #ifndef NDEBUG From e9501e8688f1d54a9a96ec50b829fd2eaec08b31 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:19:06 -0500 Subject: [PATCH 30/47] Lemon update 2015-11-09 19:33:42 on branch parser-enhancements - Change the parser to use the standard "lempar.c" template over in the tool/ folder rather than the customized "lempar.c" found in src/. (user: drh) --- tools/lemon/lempar.c | 188 +++++++++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 80 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 799eb24b1..d3f09a7ba 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -1,8 +1,27 @@ -/* Driver template for the LEMON parser generator. -** The author disclaims copyright to this source code. +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: */ -/* First off, code is included that follows the "include" declaration -** in the input grammar file. */ #include #include #include @@ -13,46 +32,46 @@ #define CDECL #endif +/************ Begin %include sections from the grammar ************************/ %% -/* Next is all token values, in a form suitable for use by makeheaders. -** This section will be null unless lemon is run with the -m switch. -*/ -/* -** These constants (all generated automatically by the parser generator) -** specify the various kinds of tokens (terminals) that the parser -** understands. -** -** Each symbol here is a terminal symbol in the grammar. -*/ +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ %% -/* Make sure the INTERFACE macro is defined. -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/* The next thing included is series of defines which control +/**************** End makeheaders token definitions ***************************/ +/* The next section is a series of control #defines. ** various aspects of the generated parser. -** YYCODETYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 terminals -** and nonterminals. "int" is used otherwise. -** YYNOCODE is a number of type YYCODETYPE which corresponds -** to no legal terminal or nonterminal number. This -** number is used to fill in empty slots of the hash -** table. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. ** YYFALLBACK If defined, this indicates that one or more tokens ** have fall-back values which should be used if the ** original value of the token will not parse. -** YYACTIONTYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 rules and -** states combined. "int" is used otherwise. -** ParseTOKENTYPE is the data type used for minor tokens given -** directly to the parser from the tokenizer. -** YYMINORTYPE is the data type used for all minor tokens. +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** ParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of ** which is ParseTOKENTYPE. The entry in the union -** for base tokens is called "yy0". +** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If ** zero the stack is dynamically sized using realloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument @@ -71,7 +90,12 @@ ** YY_ACCEPT_ACTION The yy_action[] code for accept ** YY_NO_ACTION The yy_action[] code for no-op */ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ %% +/************* End control #defines *******************************************/ /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -140,11 +164,13 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. -*/ +** +*********** Begin parsing tables **********************************************/ %% +/********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens into fallback tokens. If a construct -** like the following: +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: ** ** %fallback ID X Y Z. ** @@ -152,6 +178,10 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { @@ -273,6 +303,15 @@ static void yyGrowStack(yyParser *p){ } #endif +/* Datatype of the argument to the memory allocated passed as the +** second argument to ParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -285,9 +324,9 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(CDECL *mallocProc)(size_t)){ +void *ParseAlloc(void *(CDECL *mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH @@ -302,10 +341,12 @@ void *ParseAlloc(void *(CDECL *mallocProc)(size_t)){ return pParser; } -/* The following function deletes the value associated with a -** symbol. The symbol can be either a terminal or nonterminal. -** "yymajor" is the symbol code, and "yypminor" is a pointer to -** the value. +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. */ static void yy_destructor( yyParser *yypParser, /* The parser */ @@ -321,10 +362,12 @@ static void yy_destructor( ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are not used + ** which appear on the RHS of the rule, but which are *not* used ** inside the C code. */ +/********* Begin destructor definitions ***************************************/ %% +/********* End destructor definitions *****************************************/ default: break; /* If no destructor action specified: do nothing */ } } @@ -349,24 +392,22 @@ static void yy_pop_parser_stack(yyParser *pParser){ yy_destructor(pParser, yytos->major, &yytos->minor); } -/* -** Deallocate and destroy a parser. Destructors are all called for +/* +** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. -** -** Inputs: -**
    -**
  • A pointer to the parser. This should be a pointer -** obtained from ParseAlloc. -**
  • A pointer to a function used to reclaim memory obtained -** from malloc. -**
+* +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. */ void ParseFree( void *p, /* The parser to be deleted */ void (CDECL *freeProc)(void*) /* Function used to reclaim memory */ ){ yyParser *pParser = (yyParser*)p; +#ifndef YYPARSEFREENEVERNULL if( pParser==0 ) return; +#endif while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 free(pParser->yystack); @@ -387,10 +428,6 @@ int ParseStackPeak(void *p){ /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ static int yy_find_shift_action( yyParser *pParser, /* The parser */ @@ -457,10 +494,6 @@ static int yy_find_shift_action( /* ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ static int yy_find_reduce_action( int stateno, /* Current state number */ @@ -503,7 +536,9 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack ever overflows */ +/******** Begin %stack_overflow code ******************************************/ %% +/******** End %stack_overflow code ********************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } @@ -530,7 +565,7 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){ #endif /* -** Perform a shift action. Return the number of errors. +** Perform a shift action. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ @@ -602,21 +637,6 @@ static void yy_reduce( } #endif /* NDEBUG */ - /* Silence complaints from purify about yygotominor being uninitialized - ** in some cases when it is copied into the stack after the following - ** switch. yygotominor is uninitialized when a rule reduces that does - ** not set the value of its left-hand side nonterminal. Leaving the - ** value of the nonterminal uninitialized is utterly harmless as long - ** as the value is never used. So really the only thing this code - ** accomplishes is to quieten purify. - ** - ** 2007-01-16: The wireshark project (www.wireshark.org) reports that - ** without this code, their parser segfaults. I'm not sure what there - ** parser is doing to make this happen. This is the second bug report - ** from wireshark this week. Clearly they are stressing Lemon in ways - ** that it has not been previously stressed... (SQLite ticket #2172) - */ - /*memset(&yygotominor, 0, sizeof(yygotominor));*/ yygotominor = yyzerominor; switch( yyruleno ){ @@ -628,7 +648,9 @@ static void yy_reduce( ** #line ** break; */ +/********** Begin reduce actions **********************************************/ %% +/********** End reduce actions ************************************************/ }; assert( yyruleno>=0 && yyrulenoyyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ +/************ Begin %parse_failure code ***************************************/ %% +/************ End %parse_failure code *****************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ @@ -688,7 +712,9 @@ static void yy_syntax_error( ){ ParseARG_FETCH; #define TOKEN (yyminor.yy0) +/************ Begin %syntax_error code ****************************************/ %% +/************ End %syntax_error code ******************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -707,7 +733,9 @@ static void yy_accept( while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ %% +/*********** End %parse_accept code *******************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } From be870502fd36215f5298fc87d80fe908664e75e1 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:20:38 -0500 Subject: [PATCH 31/47] Lemon update 2015-11-10 14:51:22 on branch trunk - Improved output formatting for "PRAGMA parser_trace=ON;". (user: drh) --- tools/lemon/lempar.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index d3f09a7ba..cf0d38171 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -548,15 +548,13 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ #ifndef NDEBUG static void yyTraceShift(yyParser *yypParser, int yyNewState){ if( yyTraceFILE ){ - int i; if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); + fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n", + yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major], + yyNewState); }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + fprintf(yyTraceFILE,"%sShift '%s'\n", + yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]); } } } @@ -632,7 +630,7 @@ static void yy_reduce( if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ yysize = yyRuleInfo[yyruleno].nrhs; - fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -789,6 +787,12 @@ void Parse( yypParser->yyerrcnt = -1; yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n", + yyTracePrompt); + } +#endif } yyminorunion.yy0 = yyminor; #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) @@ -798,7 +802,7 @@ void Parse( #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); + fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); } #endif @@ -912,7 +916,12 @@ void Parse( }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + int i; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=1; i<=yypParser->yyidx; i++) + fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', + yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"]\n"); } #endif return; From 7306279a87cd08143f09129663d9af8b12df12b9 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:21:41 -0500 Subject: [PATCH 32/47] Lemon update 2016-02-16 01:01:43 on branch trunk - Improve the Lemon parser template (lempar.c) so that it avoids unnecessary work when the grammer defines YYNOERRORRECOVERY (as SQLite does). Slightly smaller and faster code results. (user: drh) --- tools/lemon/lempar.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index cf0d38171..d7cb78496 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -221,7 +221,9 @@ struct yyParser { #ifdef YYTRACKMAXSTACKDEPTH int yyidxMax; /* Maximum value of yyidx */ #endif +#ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ +#endif ParseARG_SDECL /* A place to hold %extra_argument */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ @@ -784,7 +786,9 @@ void Parse( } #endif yypParser->yyidx = 0; +#ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; +#endif yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; #ifndef NDEBUG @@ -811,7 +815,9 @@ void Parse( if( yyact <= YY_MAX_SHIFTREDUCE ){ if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; yy_shift(yypParser,yyact,yymajor,&yyminorunion); +#ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; +#endif yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); From 75f6d3a4388a3397cc2ea10b3d4d203b42c9739d Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:27:16 -0500 Subject: [PATCH 33/47] Lemon update 2016-02-16 21:19:49 on branch parser-performance - Experimental changes to Lemon for improved parser performance. (user: drh) --- tools/lemon/lempar.c | 87 +++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index d7cb78496..3a440983d 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -527,7 +527,7 @@ static int yy_find_reduce_action( /* ** The following routine is called if the stack overflows. */ -static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ +static void yyStackOverflow(yyParser *yypParser){ ParseARG_FETCH; yypParser->yyidx--; #ifndef NDEBUG @@ -571,7 +571,7 @@ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ - YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ + ParseTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; yypParser->yyidx++; @@ -582,14 +582,14 @@ static void yy_shift( #endif #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ - yyStackOverflow(yypParser, yypMinor); + yyStackOverflow(yypParser); return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ - yyStackOverflow(yypParser, yypMinor); + yyStackOverflow(yypParser); return; } } @@ -597,7 +597,7 @@ static void yy_shift( yytos = &yypParser->yystack[yypParser->yyidx]; yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; - yytos->minor = *yypMinor; + yytos->minor.yy0 = yyMinor; yyTraceShift(yypParser, yyNewState); } @@ -637,7 +637,32 @@ static void yy_reduce( } #endif /* NDEBUG */ - yygotominor = yyzerominor; + /* yygotominor = yyzerominor; */ + + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( yyRuleInfo[yyruleno].nrhs==0 ){ +#ifdef YYTRACKMAXSTACKDEPTH + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yyidx>=YYSTACKDEPTH ){ + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yyidx>=yypParser->yystksz ){ + yyGrowStack(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyStackOverflow(yypParser); + return; + } + } +#endif + } switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -655,26 +680,18 @@ static void yy_reduce( assert( yyruleno>=0 && yyrulenoyyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); if( yyact <= YY_MAX_SHIFTREDUCE ){ if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - /* If the reduce action popped at least - ** one element off the stack, then we can push the new element back - ** onto the stack here, and skip the stack overflow test in yy_shift(). - ** That gives a significant speed improvement. */ - if( yysize ){ - yypParser->yyidx++; - yymsp -= yysize-1; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yymsp->minor = yygotominor; - yyTraceShift(yypParser, yyact); - }else{ - yy_shift(yypParser,yyact,yygoto,&yygotominor); - } + yypParser->yyidx -= yysize - 1; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yymsp->minor = yygotominor; + yyTraceShift(yypParser, yyact); }else{ assert( yyact == YY_ACCEPT_ACTION ); + yypParser->yyidx -= yysize; yy_accept(yypParser); } } @@ -708,10 +725,10 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - YYMINORTYPE yyminor /* The minor type of the error token */ + ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ ParseARG_FETCH; -#define TOKEN (yyminor.yy0) +#define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ %% /************ End %syntax_error code ******************************************/ @@ -779,9 +796,7 @@ void Parse( if( yypParser->yyidx<0 ){ #if YYSTACKDEPTH<=0 if( yypParser->yystksz <=0 ){ - /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ - yyminorunion = yyzerominor; - yyStackOverflow(yypParser, &yyminorunion); + yyStackOverflow(yypParser); return; } #endif @@ -798,7 +813,6 @@ void Parse( } #endif } - yyminorunion.yy0 = yyminor; #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif @@ -814,7 +828,7 @@ void Parse( yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - yy_shift(yypParser,yyact,yymajor,&yyminorunion); + yy_shift(yypParser,yyact,yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif @@ -826,6 +840,7 @@ void Parse( int yymx; #endif assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); @@ -834,7 +849,7 @@ void Parse( #ifdef YYERRORSYMBOL /* A syntax error has occurred. ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". + ** grammar defines an error token "ERROR". ** ** This is what we do if the grammar does define ERROR: ** @@ -852,7 +867,7 @@ void Parse( ** */ if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor,yyminor); } yymx = yypParser->yystack[yypParser->yyidx].major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ @@ -862,10 +877,10 @@ void Parse( yyTracePrompt,yyTokenName[yymajor]); } #endif - yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); yymajor = YYNOCODE; }else{ - while( + while( yypParser->yyidx >= 0 && yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( @@ -879,9 +894,7 @@ void Parse( yy_parse_failed(yypParser); yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ - YYMINORTYPE u2; - u2.YYERRSYMDT = 0; - yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); } } yypParser->yyerrcnt = 3; @@ -894,7 +907,7 @@ void Parse( ** Applications can set this macro (for example inside %include) if ** they intend to abandon the parse upon the first syntax error seen. */ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; @@ -909,7 +922,7 @@ void Parse( ** three input tokens have been successfully shifted. */ if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor, yyminor); } yypParser->yyerrcnt = 3; yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); From 8a18e5c7eb57c8e56bd91815078229a46219060c Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:50:28 -0500 Subject: [PATCH 34/47] Lemon update 2016-02-17 01:18:33 on branch parser-performance - In Lemon, add the ability for the left-most RHS label to be the same as the LHS label, causing the LHS values to be written directly into the stack. (user: drh) --- tools/lemon/lemon.c | 133 ++++++++++++++++++++++++++++++++++++------- tools/lemon/lempar.c | 2 - 2 files changed, 112 insertions(+), 23 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index a70220572..7eb72b652 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -190,6 +190,8 @@ struct rule { const char **rhsalias; /* An alias for each RHS symbol (NULL if none) */ int line; /* Line number at which code begins */ const char *code; /* The code executed when this rule is reduced */ + const char *codePrefix; /* Setup code before code[] above */ + const char *codeSuffix; /* Breakdown code after code[] above */ struct symbol *precsym; /* Precedence symbol for this rule */ int index; /* An index number for this rule */ Boolean canReduce; /* True if this rule is ever reduced */ @@ -3365,6 +3367,7 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2, int bNoSubst) int c; char zInt[40]; if( zText==0 ){ + if( used==0 && z!=0 ) z[0] = 0; used = 0; return z; } @@ -3406,7 +3409,9 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ char *cp, *xp; int i; char lhsused = 0; /* True if the LHS element has been used */ + char lhsdirect; /* True if LHS writes directly into stack */ char used[MAXRHS]; /* True for each RHS element which is used */ + char zLhs[50]; /* Convert the LHS symbol into this string */ for(i=0; inrhs; i++) used[i] = 0; lhsused = 0; @@ -3417,6 +3422,47 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ rp->line = rp->ruleline; } + if( rp->lhsalias==0 ){ + /* There is no LHS value symbol. */ + lhsdirect = 1; + }else if( rp->nrhs==0 ){ + /* If there are no RHS symbols, then writing directly to the LHS is ok */ + lhsdirect = 1; + }else if( rp->rhsalias[0]==0 ){ + /* The left-most RHS symbol has not value. LHS direct is ok. But + ** we have to call the distructor on the RHS symbol first. */ + lhsdirect = 1; + if( has_destructor(rp->rhs[0],lemp) ){ + append_str(0,0,0,0,0); + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[0]->index,1-rp->nrhs,0); + rp->codePrefix = Strsafe(append_str(0,0,0,0,0)); + } + }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){ + /* The LHS symbol and the left-most RHS symbol are the same, so + ** direct writing is allowed */ + lhsdirect = 1; + lhsused = 1; + used[0] = 1; + if( rp->lhs->dtnum!=rp->rhs[0]->dtnum ){ + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) and %s(%s) share the same label but have " + "different datatypes.", + rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]); + lemp->errorcnt++; + } + }else{ + lhsdirect = 0; + } + if( lhsdirect ){ + sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); + }else{ + append_str(0,0,0,0,0); + append_str(" YYMINORTYPE yylhsminor;\n", 0, 0, 0, 0); + rp->codePrefix = Strsafe(append_str(0,0,0,0,0)); + sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); + } + append_str(0,0,0,0,0); /* This const cast is wrong but harmless, if we're careful. */ @@ -3427,7 +3473,7 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ saved = *xp; *xp = 0; if( rp->lhsalias && strcmp(cp,rp->lhsalias)==0 ){ - append_str("yygotominor.yy%d",0,rp->lhs->dtnum,0,0); + append_str(zLhs,0,0,0,0); cp = xp; lhsused = 1; }else{ @@ -3458,6 +3504,11 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ append_str(cp, 1, 0, 0, 1); } /* End loop */ + /* Main code generation completed */ + cp = append_str(0,0,0,0,0); + if( cp && cp[0] ) rp->code = Strsafe(cp); + append_str(0,0,0,0,0); + /* Check to make sure the LHS has been used */ if( rp->lhsalias && !lhsused ){ ErrorMsg(lemp->filename,rp->ruleline, @@ -3466,27 +3517,53 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ lemp->errorcnt++; } - /* Generate destructor code for RHS symbols which are not used in the - ** reduce code */ + /* Generate destructor code for RHS minor values which are not referenced. + ** Generate error messages for unused labels and duplicate labels. + */ for(i=0; inrhs; i++){ - if( rp->rhsalias[i] && !used[i] ){ - ErrorMsg(lemp->filename,rp->ruleline, - "Label %s for \"%s(%s)\" is never used.", - rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]); - lemp->errorcnt++; - }else if( rp->rhsalias[i]==0 ){ - if( has_destructor(rp->rhs[i],lemp) ){ - append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, - rp->rhs[i]->index,i-rp->nrhs+1,0); - }else{ - /* No destructor defined for this term */ + if( rp->rhsalias[i] ){ + if( i>0 ){ + int j; + if( rp->lhsalias && strcmp(rp->lhsalias,rp->rhsalias[i])==0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) has the same label as the LHS but is not the left-most " + "symbol on the RHS.", + rp->rhs[i]->name, rp->rhsalias); + lemp->errorcnt++; + } + for(j=0; jrhsalias[j] && strcmp(rp->rhsalias[j],rp->rhsalias[i])==0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used for multiple symbols on the RHS of a rule.", + rp->rhsalias[i]); + lemp->errorcnt++; + break; + } + } } + if( !used[i] ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s for \"%s(%s)\" is never used.", + rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]); + lemp->errorcnt++; + } + }else if( i>0 && has_destructor(rp->rhs[i],lemp) ){ + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[i]->index,i-rp->nrhs+1,0); } } - if( rp->code ){ - cp = append_str(0,0,0,0,0); - rp->code = Strsafe(cp?cp:""); + + /* If unable to write LHS values directly into the stack, write the + ** saved LHS value now. */ + if( lhsdirect==0 ){ + append_str(" yymsp[%d].minor.yy%d = ", 0, 1-rp->nrhs, rp->lhs->dtnum, 0); + append_str(zLhs, 0, 0, 0, 0); + append_str(";\n", 0, 0, 0, 0); } + + /* Suffix code generation complete */ + cp = append_str(0,0,0,0,0); + if( cp ) rp->codeSuffix = Strsafe(cp); } /* @@ -3501,6 +3578,12 @@ PRIVATE void emit_code( ){ const char *cp; + /* Setup code prior to the #line directive */ + if( rp->codePrefix && rp->codePrefix[0] ){ + fprintf(out, "{%s", rp->codePrefix); + for(cp=rp->codePrefix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + } + /* Generate code to do the reduce action */ if( rp->code ){ if( !lemp->nolinenosflag ){ @@ -3508,15 +3591,23 @@ PRIVATE void emit_code( tplt_linedir(out,rp->line,lemp->filename); } fprintf(out,"{%s",rp->code); - for(cp=rp->code; *cp; cp++){ - if( *cp=='\n' ) (*lineno)++; - } /* End loop */ + for(cp=rp->code; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } fprintf(out,"}\n"); (*lineno)++; if( !lemp->nolinenosflag ){ (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } - } /* End if( rp->code ) */ + } + + /* Generate breakdown code that occurs after the #line directive */ + if( rp->codeSuffix && rp->codeSuffix[0] ){ + fprintf(out, "%s", rp->codeSuffix); + for(cp=rp->codeSuffix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + } + + if( rp->codePrefix ){ + fprintf(out, "}\n"); (*lineno)++; + } return; } diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 3a440983d..b3ab0da06 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -623,7 +623,6 @@ static void yy_reduce( ){ int yygoto; /* The next state */ int yyact; /* The next action */ - YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ ParseARG_FETCH; @@ -687,7 +686,6 @@ static void yy_reduce( yymsp -= yysize-1; yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; - yymsp->minor = yygotominor; yyTraceShift(yypParser, yyact); }else{ assert( yyact == YY_ACCEPT_ACTION ); From ab4c3a0c5ee5b18a41bdc2ba39ca9d057398b5b2 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:53:42 -0500 Subject: [PATCH 35/47] Lemon update 2016-02-17 01:46:19 on branch parser-performance - Further improvements to the Lemon-generated code for yy_reduce(). (user: drh) --- tools/lemon/lemon.c | 18 +++++++++++++----- tools/lemon/lempar.c | 2 -- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 7eb72b652..b32c032b6 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3404,10 +3404,14 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2, int bNoSubst) ** zCode is a string that is the action associated with a rule. Expand ** the symbols in this string so that the refer to elements of the parser ** stack. +** +** Return 1 if the expanded code requires that "yylhsminor" local variable +** to be defined. */ -PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ +PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ char *cp, *xp; int i; + int rc = 0; /* True if yylhsminor is used */ char lhsused = 0; /* True if the LHS element has been used */ char lhsdirect; /* True if LHS writes directly into stack */ char used[MAXRHS]; /* True for each RHS element which is used */ @@ -3457,9 +3461,7 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ if( lhsdirect ){ sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); }else{ - append_str(0,0,0,0,0); - append_str(" YYMINORTYPE yylhsminor;\n", 0, 0, 0, 0); - rp->codePrefix = Strsafe(append_str(0,0,0,0,0)); + rc = 1; sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); } @@ -3564,6 +3566,8 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ /* Suffix code generation complete */ cp = append_str(0,0,0,0,0); if( cp ) rp->codeSuffix = Strsafe(cp); + + return rc; } /* @@ -4232,9 +4236,13 @@ void ReportTable( tplt_xfer(lemp->name,in,out,&lineno); /* Generate code which executes during each REDUCE action */ + i = 0; for(rp=lemp->rule; rp; rp=rp->next){ - translate_code(lemp, rp); + i += translate_code(lemp, rp); } + if( i ){ + fprintf(out," YYMINORTYPE yylhsminor;\n"); lineno++; + } /* First output rules other than the default: rule */ for(rp=lemp->rule; rp; rp=rp->next){ struct rule *rp2; /* Other rules with the same action */ diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index b3ab0da06..25ac97036 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -636,8 +636,6 @@ static void yy_reduce( } #endif /* NDEBUG */ - /* yygotominor = yyzerominor; */ - /* Check that the stack is large enough to grow by a single entry ** if the RHS of the rule is empty. This ensures that there is room ** enough on the stack to push the LHS value */ From 25972d79a791dbe73f4ea9a6165fb72244f990ac Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:57:03 -0500 Subject: [PATCH 36/47] Lemon update 2016-02-17 04:33:10 on branch parser-performance - Enhance Lemon so that if reduce code contains a comment of the form "/*A-overwrites-X*/" then a LHS label A is allowed to overwrite the RHS label X. (user: drh) --- tools/lemon/lemon.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index b32c032b6..264809896 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3411,11 +3411,13 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2, int bNoSubst) PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ char *cp, *xp; int i; - int rc = 0; /* True if yylhsminor is used */ - char lhsused = 0; /* True if the LHS element has been used */ - char lhsdirect; /* True if LHS writes directly into stack */ - char used[MAXRHS]; /* True for each RHS element which is used */ - char zLhs[50]; /* Convert the LHS symbol into this string */ + int rc = 0; /* True if yylhsminor is used */ + const char *zSkip = 0; /* The zOvwrt comment within rp->code, or NULL */ + char lhsused = 0; /* True if the LHS element has been used */ + char lhsdirect; /* True if LHS writes directly into stack */ + char used[MAXRHS]; /* True for each RHS element which is used */ + char zLhs[50]; /* Convert the LHS symbol into this string */ + char zOvwrt[900]; /* Comment that to allow LHS to overwrite RHS */ for(i=0; inrhs; i++) used[i] = 0; lhsused = 0; @@ -3456,7 +3458,15 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ lemp->errorcnt++; } }else{ - lhsdirect = 0; + sprintf(zOvwrt, "/*%s-overwrites-%s*/", rp->lhsalias, rp->rhsalias[0]); + zSkip = strstr(rp->code, zOvwrt); + if( zSkip!=0 ){ + /* The code contains a special comment that indicates that it is safe + ** for the LHS label to overwrite left-most RHS label. */ + lhsdirect = 1; + }else{ + lhsdirect = 0; + } } if( lhsdirect ){ sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); @@ -3469,6 +3479,11 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ /* This const cast is wrong but harmless, if we're careful. */ for(cp=(char *)rp->code; *cp; cp++){ + if( cp==zSkip ){ + append_str(zOvwrt,0,0,0,0); + cp += lemonStrlen(zOvwrt)-1; + continue; + } if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){ char saved; for(xp= &cp[1]; ISALNUM(*xp) || *xp=='_'; xp++); From 4a24f7adcf9e0e7b362c8771a256b690b0b16297 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 10:58:56 -0500 Subject: [PATCH 37/47] Lemon update 2016-02-17 12:34:03 on branch parser-performance - More agressive use of /*A-overwrites-X*/ in the parser. Fix an off-by-one error in parser stack overflow detection. (user: drh) --- tools/lemon/lemon.c | 9 ++++++++- tools/lemon/lempar.c | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 264809896..34d0b1ca3 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3412,6 +3412,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ char *cp, *xp; int i; int rc = 0; /* True if yylhsminor is used */ + int dontUseRhs0 = 0; /* If true, use of left-most RHS label is illegal */ const char *zSkip = 0; /* The zOvwrt comment within rp->code, or NULL */ char lhsused = 0; /* True if the LHS element has been used */ char lhsdirect; /* True if LHS writes directly into stack */ @@ -3482,6 +3483,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ if( cp==zSkip ){ append_str(zOvwrt,0,0,0,0); cp += lemonStrlen(zOvwrt)-1; + dontUseRhs0 = 1; continue; } if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){ @@ -3496,7 +3498,12 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ }else{ for(i=0; inrhs; i++){ if( rp->rhsalias[i] && strcmp(cp,rp->rhsalias[i])==0 ){ - if( cp!=rp->code && cp[-1]=='@' ){ + if( i==0 && dontUseRhs0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used after '%s'.", + rp->rhsalias[0], zOvwrt); + lemp->errorcnt++; + }else if( cp!=rp->code && cp[-1]=='@' ){ /* If the argument is of the form @X then substituted ** the token number of X, not the value of X */ append_str("yymsp[%d].major",-1,i-rp->nrhs+1,0,0); diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 25ac97036..f17d6eea3 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -646,14 +646,14 @@ static void yy_reduce( } #endif #if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH ){ + if( yypParser->yyidx>=YYSTACKDEPTH-1 ){ yyStackOverflow(yypParser); return; } #else - if( yypParser->yyidx>=yypParser->yystksz ){ + if( yypParser->yyidx>=yypParser->yystksz-1 ){ yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz ){ + if( yypParser->yyidx>=yypParser->yystksz-1 ){ yyStackOverflow(yypParser); return; } From ec5108b2e5fde31764eff4daa02cce6ab75ee78d Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 12:33:07 -0500 Subject: [PATCH 38/47] Lemon update 2016-02-19 13:19:22 on branch trunk - Omit the unused yyzerominor constant. (user: drh) --- tools/lemon/lempar.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index f17d6eea3..4c14a8fb9 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -97,10 +97,6 @@ %% /************* End control #defines *******************************************/ -/* The yyzerominor constant is used to initialize instances of -** YYMINORTYPE objects to zero. */ -static const YYMINORTYPE yyzerominor = { 0 }; - /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. ** From 4641549e0e81a4595d0e8e61a5d0fd58688adb74 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 11:03:52 -0500 Subject: [PATCH 39/47] Lemon update 2016-03-16 19:45:54 on branch trunk - Enhance Lemon so that it reorders the reduce rules such that rules without actions occur at the end and so that the first rule is number 0. This reduces the size of the jump table on the reduce switch, and helps the parser to run faster. (user: drh) --- tools/lemon/lemon.c | 105 ++++++++++++++++++++++++++++++++++--------- tools/lemon/lempar.c | 11 +++-- 2 files changed, 88 insertions(+), 28 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 34d0b1ca3..e8a983943 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -194,6 +194,7 @@ struct rule { const char *codeSuffix; /* Breakdown code after code[] above */ struct symbol *precsym; /* Precedence symbol for this rule */ int index; /* An index number for this rule */ + int iRule; /* Rule number as used in the generated tables */ Boolean canReduce; /* True if this rule is ever reduced */ struct rule *nextlhs; /* Next rule with the same LHS */ struct rule *next; /* Next rule in the global list */ @@ -276,6 +277,7 @@ struct plink { struct lemon { struct state **sorted; /* Table of states sorted by state number */ struct rule *rule; /* List of all rules */ + struct rule *startRule; /* First rule */ int nstate; /* Number of states */ int nxstate; /* nstate with tail degenerate states removed */ int nrule; /* Number of rules */ @@ -761,12 +763,12 @@ void FindStates(struct lemon *lemp) ErrorMsg(lemp->filename,0, "The specified start symbol \"%s\" is not \ in a nonterminal of the grammar. \"%s\" will be used as the start \ -symbol instead.",lemp->start,lemp->rule->lhs->name); +symbol instead.",lemp->start,lemp->startRule->lhs->name); lemp->errorcnt++; - sp = lemp->rule->lhs; + sp = lemp->startRule->lhs; } }else{ - sp = lemp->rule->lhs; + sp = lemp->startRule->lhs; } /* Make sure the start symbol doesn't occur on the right-hand side of @@ -1020,9 +1022,9 @@ void FindActions(struct lemon *lemp) /* Add the accepting token */ if( lemp->start ){ sp = Symbol_find(lemp->start); - if( sp==0 ) sp = lemp->rule->lhs; + if( sp==0 ) sp = lemp->startRule->lhs; }else{ - sp = lemp->rule->lhs; + sp = lemp->startRule->lhs; } /* Add to the first state (which is always the starting state of the ** finite state machine) an action to ACCEPT if the lookahead is the @@ -1414,6 +1416,54 @@ static void handle_T_option(char *z){ strcpy(user_templatename, z); } +/* Merge together to lists of rules order by rule.iRule */ +static struct rule *Rule_merge(struct rule *pA, struct rule *pB){ + struct rule *pFirst = 0; + struct rule **ppPrev = &pFirst; + while( pA && pB ){ + if( pA->iRuleiRule ){ + *ppPrev = pA; + ppPrev = &pA->next; + pA = pA->next; + }else{ + *ppPrev = pB; + ppPrev = &pB->next; + pB = pB->next; + } + } + if( pA ){ + *ppPrev = pA; + }else{ + *ppPrev = pB; + } + return pFirst; +} + +/* +** Sort a list of rules in order of increasing iRule value +*/ +static struct rule *Rule_sort(struct rule *rp){ + int i; + struct rule *pNext; + struct rule *x[32]; + memset(x, 0, sizeof(x)); + while( rp ){ + pNext = rp->next; + rp->next = 0; + for(i=0; iname[0]); i++); lem.nterminal = i; + /* Assign sequential rule numbers */ + for(i=0, rp=lem.rule; rp; rp=rp->next){ + rp->iRule = rp->code ? i++ : -1; + } + for(rp=lem.rule; rp; rp=rp->next){ + if( rp->iRule<0 ) rp->iRule = i++; + } + lem.startRule = lem.rule; + lem.rule = Rule_sort(lem.rule); + /* Generate a reprint of the grammar, if requested on the command line */ if( rpflag ){ Reprint(&lem); @@ -2978,13 +3039,13 @@ int PrintAction( } case REDUCE: { struct rule *rp = ap->x.rp; - fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->iRule); RulePrint(fp, rp, -1); break; } case SHIFTREDUCE: { struct rule *rp = ap->x.rp; - fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index); + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule); RulePrint(fp, rp, -1); break; } @@ -2997,7 +3058,7 @@ int PrintAction( case SRCONFLICT: case RRCONFLICT: fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", - indent,ap->sp->name,ap->x.rp->index); + indent,ap->sp->name,ap->x.rp->iRule); break; case SSCONFLICT: fprintf(fp,"%*s shift %-7d ** Parsing conflict **", @@ -3014,7 +3075,7 @@ int PrintAction( case RD_RESOLVED: if( showPrecedenceConflict ){ fprintf(fp,"%*s reduce %-7d -- dropped by precedence", - indent,ap->sp->name,ap->x.rp->index); + indent,ap->sp->name,ap->x.rp->iRule); }else{ result = 0; } @@ -3045,7 +3106,7 @@ void ReportOutput(struct lemon *lemp) while( cfp ){ char buf[20]; if( cfp->dot==cfp->rp->nrhs ){ - sprintf(buf,"(%d)",cfp->rp->index); + sprintf(buf,"(%d)",cfp->rp->iRule); fprintf(fp," %5s ",buf); }else{ fprintf(fp," "); @@ -3150,8 +3211,8 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) int act; switch( ap->type ){ case SHIFT: act = ap->x.stp->statenum; break; - case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate; break; - case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break; + case SHIFTREDUCE: act = ap->x.rp->iRule + lemp->nstate; break; + case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break; case ERROR: act = lemp->nstate + lemp->nrule*2; break; case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1; break; default: act = -1; break; @@ -4174,7 +4235,7 @@ void ReportTable( ** when tracing REDUCE actions. */ for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ - assert( rp->index==i ); + assert( rp->iRule==i ); fprintf(out," /* %3d */ \"", i); writeRuleText(out, rp); fprintf(out,"\",\n"); lineno++; @@ -4269,15 +4330,15 @@ void ReportTable( for(rp=lemp->rule; rp; rp=rp->next){ struct rule *rp2; /* Other rules with the same action */ if( rp->code==0 ) continue; - if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */ - fprintf(out," case %d: /* ",rp->index); + if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */ + fprintf(out," case %d: /* ",rp->iRule); writeRuleText(out, rp); fprintf(out," */\n"); lineno++; for(rp2=rp->next; rp2; rp2=rp2->next){ if( rp2->code==rp->code ){ - fprintf(out," case %d: /*",rp2->index); + fprintf(out," case %d: /*",rp2->iRule); writeRuleText(out, rp2); - fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp2->index); lineno++; + fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; rp2->code = 0; } } @@ -4289,11 +4350,11 @@ void ReportTable( ** empty actions. */ fprintf(out," default:\n"); lineno++; for(rp=lemp->rule; rp; rp=rp->next){ - if( rp->code==0 ) continue; - assert( rp->code[0]=='\n' && rp->code[1]==0 ); - fprintf(out," /* (%d) ", rp->index); - writeRuleText(out, rp); - fprintf(out," */ yytestcase(yyruleno==%d);\n", rp->index); lineno++; + if( rp->code==0 ) continue; + assert( rp->code[0]=='\n' && rp->code[1]==0 ); + fprintf(out," /* (%d) ", rp->iRule); + writeRuleText(out, rp); + fprintf(out," */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; } fprintf(out," break;\n"); lineno++; tplt_xfer(lemp->name,in,out,&lineno); diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 4c14a8fb9..0549a9dad 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -427,7 +427,7 @@ int ParseStackPeak(void *p){ ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. */ -static int yy_find_shift_action( +static unsigned int yy_find_shift_action( yyParser *pParser, /* The parser */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ @@ -615,7 +615,7 @@ static void yy_accept(yyParser*); /* Forward Declaration */ */ static void yy_reduce( yyParser *yypParser, /* The parser */ - int yyruleno /* Number of the rule by which to reduce */ + unsigned int yyruleno /* Number of the rule by which to reduce */ ){ int yygoto; /* The next state */ int yyact; /* The next action */ @@ -624,8 +624,7 @@ static void yy_reduce( ParseARG_FETCH; yymsp = &yypParser->yystack[yypParser->yyidx]; #ifndef NDEBUG - if( yyTraceFILE && yyruleno>=0 - && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ yysize = yyRuleInfo[yyruleno].nrhs; fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, yyRuleName[yyruleno], yymsp[-yysize].stateno); @@ -670,7 +669,7 @@ static void yy_reduce( %% /********** End reduce actions ************************************************/ }; - assert( yyruleno>=0 && yyruleno Date: Sun, 20 Mar 2016 13:35:46 -0500 Subject: [PATCH 40/47] VS2015 warning fixes for Lemon --- tools/lemon/lemon.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index e8a983943..9a6614f00 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3784,7 +3784,7 @@ void print_stack_union( break; } hash++; - if( hash>=arraysize ) hash = 0; + if( hash>=(unsigned)arraysize ) hash = 0; } if( types[hash]==0 ){ sp->dtnum = hash + 1; @@ -3912,7 +3912,7 @@ void ReportTable( struct action *ap; struct rule *rp; struct acttab *pActtab; - int i, j, k, n, sz; + int i, j, n, sz; int szActionType; /* sizeof(YYACTIONTYPE) */ int szCodeType; /* sizeof(YYCODETYPE) */ const char *name; @@ -4429,7 +4429,7 @@ void CompressTables(struct lemon *lemp) struct state *stp; struct action *ap, *ap2; struct rule *rp, *rp2, *rbest; - int nbest, n, nshift; + int nbest, n; int i; int usesWildcard; From 7f3e1777a7e983461ec0b5f6acd9bccb98f63912 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 14:26:02 -0500 Subject: [PATCH 41/47] Add -C option to Lemon, to specify an output directory --- tools/lemon/lemon.c | 75 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 9a6614f00..546cf515e 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -273,7 +273,7 @@ struct plink { /* The state vector for the entire parser generator is recorded as ** follows. (LEMON uses no global variables and makes little use of ** static variables. Fields in the following structure can be thought -** of as begin global variables in the program.) */ +** of as being global variables in the program.) */ struct lemon { struct state **sorted; /* Table of states sorted by state number */ struct rule *rule; /* List of all rules */ @@ -302,6 +302,7 @@ struct lemon { char *tokendest; /* Code to execute to destroy token data */ char *vardest; /* Code for the default non-terminal destructor */ char *filename; /* Name of the input file */ + char *outbasefilename; /* Name of the input file, with the output dir's path */ char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ int nconflict; /* Number of parsing conflicts */ @@ -1416,6 +1417,72 @@ static void handle_T_option(char *z){ strcpy(user_templatename, z); } +/* Routines for routing output to a different directory than the one +** the source file resides in. +*/ +static char *output_dir = NULL; + +static inline Boolean is_seperator(int c) +{ + if (c == '/') + return LEMON_TRUE; +#if defined(_WIN32) || defined(DOS) + if (c == '\\' || c == ':') + return LEMON_TRUE; +#endif + return LEMON_FALSE; +} + +/* Returns the file part of a pathname. +*/ +const char *file_base(const char *path) +{ + const char *src = path + strlen(path) - 1; + if( src >= path ){ + // back up until a / or the start + while (src != path && !is_seperator(*(src - 1))) + src--; + + // Check for files with drive specification but no path +#if defined(_WIN32) || defined(DOS) + if( src == path && src[0] != 0 ){ + if( src[1] == ':' ) + src += 2; + } +#endif + return src; + } + return NULL; +} + +static char *stitch_outdir(char *path) +{ + if( output_dir ){ + const char *base = file_base(path); + char *newpath = (char *) malloc( lemonStrlen(output_dir) + lemonStrlen(path) + 1 ); + if( newpath==0 ){ + memory_error(); + } + strcpy(newpath, output_dir); + strcat(newpath, base); + return newpath; + } + return path; +} + +static void handle_C_option(char *z){ + int len = lemonStrlen(z); + output_dir = (char *) malloc( len+2 ); + if( output_dir==0 ){ + memory_error(); + } + strcpy(output_dir, z); + if( !is_seperator(output_dir[len-1]) ){ + output_dir[len] = '/'; + output_dir[len+1] = '\0'; + } +} + /* Merge together to lists of rules order by rule.iRule */ static struct rule *Rule_merge(struct rule *pA, struct rule *pB){ struct rule *pFirst = 0; @@ -1491,6 +1558,7 @@ int main(int argc, char **argv) static struct s_options options[] = { {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, + {OPT_FSTR, "C", (char*)handle_C_option, "Write output files to a different directory."}, {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."}, {OPT_FSTR, "f", 0, "Ignored. (Placeholder for -f compiler options.)"}, {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, @@ -1531,6 +1599,7 @@ int main(int argc, char **argv) State_init(); lem.argv0 = argv[0]; lem.filename = OptArg(0); + lem.outbasefilename = stitch_outdir(lem.filename); lem.basisflag = basisflag; lem.nolinenosflag = nolinenosflag; Symbol_new("$"); @@ -2877,12 +2946,12 @@ PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) char *name; char *cp; - name = (char*)malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 ); + name = (char*)malloc( lemonStrlen(lemp->outbasefilename) + lemonStrlen(suffix) + 5 ); if( name==0 ){ fprintf(stderr,"Can't allocate space for a filename.\n"); exit(1); } - strcpy(name,lemp->filename); + strcpy(name,lemp->outbasefilename); cp = strrchr(name,'.'); if( cp ) *cp = 0; strcat(name,suffix); From 2a394d0cb8308afa64b75b70fd32dd5330a92764 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 14:34:44 -0500 Subject: [PATCH 42/47] Do not copy lemon grammars to the output directory - Having my edits to the grammar disappear because Visual Studio had opened the copy instead of the original was super annoying. Using the -C option with Lemon, this problem is avoided because there are no copies to worry about. --- src/CMakeLists.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 797e76139..87447270b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -708,15 +708,11 @@ else() endif() add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/xlat_parser.c ${CMAKE_CURRENT_BINARY_DIR}/xlat_parser.h - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/xlat/xlat_parser.y . - COMMAND lemon xlat_parser.y - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMAND lemon -C${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/xlat/xlat_parser.y DEPENDS lemon ${CMAKE_CURRENT_SOURCE_DIR}/xlat/xlat_parser.y ) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/zcc-parse.c ${CMAKE_CURRENT_BINARY_DIR}/zcc-parse.h - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/zscript/zcc-parse.lemon . - COMMAND lemon zcc-parse.lemon - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMAND lemon -C${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/zscript/zcc-parse.lemon DEPENDS lemon ${CMAKE_CURRENT_SOURCE_DIR}/zscript/zcc-parse.lemon ) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sc_man_scanner.h From baa6dc0568a88f9a556fced497654583ddf4f004 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 15:42:13 -0500 Subject: [PATCH 43/47] Add zcc-parse.c and .h to the ZScript project folder --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87447270b..d23eb084a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1370,5 +1370,5 @@ source_group("Shared Game" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/g_sh source_group("Versioning" FILES version.h win32/zdoom.rc) source_group("Win32 Files" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/win32/.+") source_group("Xlat" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/xlat/.+" FILES ${CMAKE_CURRENT_BINARY_DIR}/xlat_parser.c ${CMAKE_CURRENT_BINARY_DIR}/xlat_parser.h) -source_group("ZScript" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/zscript/.+") +source_group("ZScript" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/zscript/.+" FILES ${CMAKE_CURRENT_BINARY_DIR}/zcc-parse.c ${CMAKE_CURRENT_BINARY_DIR}/zcc-parse.h) source_group("Source Files" FILES ${CMAKE_CURRENT_BINARY_DIR}/sc_man_scanner.h sc_man_scanner.re) From 260cf6848f95d7f34091ab0c3194ba509459dd54 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 15:10:22 -0500 Subject: [PATCH 44/47] Use LHS and RHS label matching in the grammars where possible --- src/xlat/xlat_parser.y | 5 +-- src/zscript/zcc-parse.lemon | 88 ++++++++++++------------------------- 2 files changed, 30 insertions(+), 63 deletions(-) diff --git a/src/xlat/xlat_parser.y b/src/xlat/xlat_parser.y index f0850c625..9bafb9fb8 100644 --- a/src/xlat/xlat_parser.y +++ b/src/xlat/xlat_parser.y @@ -202,10 +202,7 @@ special_args(Z) ::= . /* empty */ Z.args[3] = 0; Z.args[4] = 0; } -special_args(Z) ::= multi_special_arg(A). -{ - Z = A; -} +special_args(Z) ::= multi_special_arg(Z). //========================================================================== // diff --git a/src/zscript/zcc-parse.lemon b/src/zscript/zcc-parse.lemon index e9fe7e41d..eb9209b64 100644 --- a/src/zscript/zcc-parse.lemon +++ b/src/zscript/zcc-parse.lemon @@ -129,7 +129,7 @@ main ::= translation_unit(A). { stat->TopNode = A; stat->sc.ScriptMessage("Parse %type translation_unit {ZCC_TreeNode *} translation_unit(X) ::= . { X = NULL; } translation_unit(X) ::= translation_unit(A) external_declaration(B). { SAFE_APPEND(A,B); X = A; } -translation_unit(X) ::= translation_unit(A) EOF. { X = A; } +translation_unit(X) ::= translation_unit(X) EOF. translation_unit(X) ::= error. { X = NULL; } %type external_declaration {ZCC_TreeNode *} @@ -150,10 +150,7 @@ opt_expr(X) ::= . { X = NULL; } -opt_expr(X) ::= expr(A). -{ - X = A; -} +opt_expr(X) ::= expr(X). /************ Class Definition ************/ @@ -254,10 +251,10 @@ struct_def(X) ::= STRUCT(T) IDENTIFIER(A) LBRACE opt_struct_body(B) RBRACE opt_s } opt_struct_body(X) ::= . { X = NULL; } -opt_struct_body(X) ::= struct_body(A). { X = A; } +opt_struct_body(X) ::= struct_body(X). struct_body(X) ::= error. { X = NULL; } -struct_body(X) ::= struct_member(A). { X = A; } +struct_body(X) ::= struct_member(X). struct_body(X) ::= struct_member(A) struct_body(B). { X = A; A->AppendSibling(B); } struct_member(X) ::= declarator_no_fun(A). { X = A; } @@ -346,11 +343,11 @@ enum_type(X) ::= . { X.Int = ZCC_IntAuto; X.SourceLoc = stat->sc.GetMes enum_type(X) ::= COLON int_type(A). { X = A; } enum_list(X) ::= error. { X = NULL; } -enum_list(X) ::= enumerator(A). { X = A; } +enum_list(X) ::= enumerator(X). enum_list(X) ::= enum_list(A) COMMA enumerator(B). { X = A; A->AppendSibling(B); } opt_enum_list(X) ::= . { X = NULL; } -opt_enum_list(X) ::= enum_list(A) opt_comma. { X = A; } +opt_enum_list(X) ::= enum_list(X) opt_comma. enumerator(X) ::= IDENTIFIER(A). { @@ -416,7 +413,7 @@ state_label(X) ::= NWS(A) COLON. X = label; } -state_flow(X) ::= state_flow_type(A) scanner_mode SEMICOLON. { X = A; } +state_flow(X) ::= state_flow_type(X) scanner_mode SEMICOLON. state_flow_type(X) ::= STOP(A). { NEW_AST_NODE(StateStop, flow, A); X = flow; } state_flow_type(X) ::= WAIT(A). { NEW_AST_NODE(StateWait, flow, A); X = flow; } @@ -505,7 +502,7 @@ int_type(X) ::= INT(T). { X.Int = ZCC_SInt32; X.SourceLoc = T.SourceLoc; } int_type(X) ::= UINT(T). { X.Int = ZCC_UInt32; X.SourceLoc = T.SourceLoc; } type_name1(X) ::= BOOL(T). { X.Int = ZCC_Bool; X.SourceLoc = T.SourceLoc; } -type_name1(X) ::= int_type(A). { X = A; } +type_name1(X) ::= int_type(X). type_name1(X) ::= FLOAT(T). { X.Int = ZCC_FloatAuto; X.SourceLoc = T.SourceLoc; } type_name1(X) ::= DOUBLE(T). { X.Int = ZCC_Float64; X.SourceLoc = T.SourceLoc; } type_name1(X) ::= STRING(T). { X.Int = ZCC_String; X.SourceLoc = T.SourceLoc; } @@ -554,8 +551,8 @@ vector_size(X) ::= LT intconst(A) GT. } X.SourceLoc = A.SourceLoc; } -intconst(X) ::= INTCONST(A). { X = A; } -intconst(X) ::= UINTCONST(A). { X = A; } +intconst(X) ::= INTCONST(X). +intconst(X) ::= UINTCONST(X). /* Type names can also be used as identifiers in contexts where type names * are not normally allowed. */ @@ -599,14 +596,14 @@ class_restrictor(X) ::= LT dottable_id(A) GT. { X = A; } type(X) ::= type_name(A). { X = A; A->ArraySize = NULL; } type(X) ::= aggregate_type(A). { X = A; A->ArraySize = NULL; } -type_or_array(X) ::= type(A). { X = A; } +type_or_array(X) ::= type(X). type_or_array(X) ::= type(A) array_size(B). { X = A; A->ArraySize = B; } -type_list(X) ::= type_or_array(A). { X = A; }/* A comma-separated list of types */ +type_list(X) ::= type_or_array(X). /* A comma-separated list of types */ type_list(X) ::= type_list(A) COMMA type_or_array(B). { X = A; A->AppendSibling(B); } type_list_or_void(X) ::= VOID. { X = NULL; } -type_list_or_void(X) ::= type_list(A). { X = A; } +type_list_or_void(X) ::= type_list(X). array_size_expr(X) ::= LBRACKET opt_expr(A) RBRACKET. { @@ -622,10 +619,7 @@ array_size_expr(X) ::= LBRACKET opt_expr(A) RBRACKET. X = A; } } -array_size(X) ::= array_size_expr(A). -{ - X = A; -} +array_size(X) ::= array_size_expr(X). array_size(X) ::= array_size(A) array_size_expr(B). { A->AppendSibling(B); @@ -738,10 +732,7 @@ variable_name(X) ::= IDENTIFIER(A) array_size(B). X = var; } -variable_list(X) ::= variable_name(A). -{ - X = A; -} +variable_list(X) ::= variable_name(X). variable_list(X) ::= variable_list(A) COMMA variable_name(B). { A->AppendSibling(B); @@ -764,7 +755,7 @@ func_const(X) ::= . { X.Int = 0; X.SourceLoc = stat->sc.GetMessageLine(); func_const(X) ::= CONST(T). { X.Int = ZCC_FuncConst; X.SourceLoc = T.SourceLoc; } opt_func_body(X) ::= SEMICOLON. { X = NULL; } -opt_func_body(X) ::= function_body(A). { X = A; } +opt_func_body(X) ::= function_body(X). %type func_params {ZCC_FuncParamDecl *} %type func_param_list {ZCC_FuncParamDecl *} @@ -772,9 +763,9 @@ opt_func_body(X) ::= function_body(A). { X = A; } func_params(X) ::= . /* empty */ { X = NULL; } func_params(X) ::= VOID. { X = NULL; } -func_params(X) ::= func_param_list(A). { X = A; } +func_params(X) ::= func_param_list(X). -func_param_list(X) ::= func_param(A). { X = A; } +func_param_list(X) ::= func_param(X). func_param_list(X) ::= func_param_list(A) COMMA func_param(B). { X = A; A->AppendSibling(B); } func_param(X) ::= func_param_flags(A) type(B) IDENTIFIER(C). @@ -819,10 +810,7 @@ primary(X) ::= SUPER(T). expr->Type = NULL; X = expr; } -primary(X) ::= constant(A). -{ - X = A; -} +primary(X) ::= constant(A). { X = A; } primary(X) ::= SELF(T). { NEW_AST_NODE(Expression, expr, T); @@ -881,10 +869,7 @@ primary(X) ::= SCOPE primary(B). */ /*----- Unary Expressions -----*/ -unary_expr(X) ::= primary(A). -{ - X = A; -} +unary_expr(X) ::= primary(X). unary_expr(X) ::= SUB unary_expr(A). [UNARY] { ZCC_ExprConstant *con = static_cast(A); @@ -959,10 +944,7 @@ unary_expr(X) ::= ALIGNOF unary_expr(A). [UNARY] /*----- Binary Expressions -----*/ -expr(X) ::= unary_expr(A). -{ - X = A; -} +expr(X) ::= unary_expr(X). expr(X) ::= expr(A) ADD expr(B). /* a + b */ { BINARY_EXPR(A,B,PEX_Add); @@ -1118,10 +1100,7 @@ expr(X) ::= expr(A) QUESTION expr(B) COLON expr(C). %type expr_list{ZCC_Expression *} -expr_list(X) ::= expr(A). -{ - X = A; -} +expr_list(X) ::= expr(X). expr_list(X) ::= expr_list(A) COMMA expr(B). { X = A; @@ -1137,10 +1116,7 @@ expr_list(X) ::= expr_list(A) COMMA expr(B). %type func_expr_item{ZCC_FuncParm *} %type named_expr{ZCC_FuncParm *} -func_expr_list(X) ::= func_expr_item(A). -{ - X = A; -} +func_expr_list(X) ::= func_expr_item(X). func_expr_list(X) ::= func_expr_list(A) COMMA(T) func_expr_item(B). { // Omitted parameters still need to appear as nodes in the list. @@ -1166,10 +1142,7 @@ func_expr_item(X) ::= . { X = NULL; } -func_expr_item(X) ::= named_expr(A). -{ - X = A; -} +func_expr_item(X) ::= named_expr(X). named_expr(X) ::= IDENTIFIER(A) COLON expr(B). { @@ -1208,10 +1181,7 @@ string_constant(X) ::= string_constant(A) STRCONST(B). X = strconst; } -constant(X) ::= string_constant(A). -{ - X = A; -} +constant(X) ::= string_constant(X). constant(X) ::= INTCONST(A). { NEW_INTCONST_NODE(intconst, TypeSInt32, A.Int, A); @@ -1251,16 +1221,16 @@ constant(X) ::= TRUE(A). /************ Statements ************/ -function_body(X) ::= compound_statement(A). { X = A; } +function_body(X) ::= compound_statement(X). %type statement{ZCC_Statement *} statement(X) ::= SEMICOLON. { X = NULL; } statement(X) ::= labeled_statement(A). { X = A; } statement(X) ::= compound_statement(A). { X = A; } statement(X) ::= expression_statement(A) SEMICOLON. { X = A; } -statement(X) ::= selection_statement(A). { X = A; } -statement(X) ::= iteration_statement(A). { X = A; } -statement(X) ::= jump_statement(A). { X = A; } +statement(X) ::= selection_statement(X). +statement(X) ::= iteration_statement(X). +statement(X) ::= jump_statement(X). statement(X) ::= assign_statement(A) SEMICOLON. { X = A; } statement(X) ::= local_var(A) SEMICOLON. { X = A; } statement(X) ::= error SEMICOLON. { X = NULL; } From c0dd39ffd4037f8134e4a53465ab26ba7952c2c3 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 15:17:44 -0500 Subject: [PATCH 45/47] Use %token_class to define intconst --- src/zscript/zcc-parse.lemon | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/zscript/zcc-parse.lemon b/src/zscript/zcc-parse.lemon index eb9209b64..8c6e0bc5e 100644 --- a/src/zscript/zcc-parse.lemon +++ b/src/zscript/zcc-parse.lemon @@ -537,6 +537,7 @@ type_name(X) ::= DOT dottable_id(A). * (Well, actually, I'm not sure if 4D ones are going to happen * straight away.) */ +%token_class intconst INTCONST|UINTCONST. vector_size(X) ::= . { X.Int = ZCC_Vector3; X.SourceLoc = stat->sc.GetMessageLine(); } vector_size(X) ::= LT intconst(A) GT. { @@ -551,8 +552,6 @@ vector_size(X) ::= LT intconst(A) GT. } X.SourceLoc = A.SourceLoc; } -intconst(X) ::= INTCONST(X). -intconst(X) ::= UINTCONST(X). /* Type names can also be used as identifiers in contexts where type names * are not normally allowed. */ From 071a5718cc15829ea0a317b2320ef0c904e3f9c7 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 15:39:21 -0500 Subject: [PATCH 46/47] Use /*X-overwrites-A*/ wherever X = A; appears in the grammar --- src/zscript/zcc-parse.lemon | 132 ++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/src/zscript/zcc-parse.lemon b/src/zscript/zcc-parse.lemon index 8c6e0bc5e..37f69842f 100644 --- a/src/zscript/zcc-parse.lemon +++ b/src/zscript/zcc-parse.lemon @@ -128,15 +128,15 @@ main ::= translation_unit(A). { stat->TopNode = A; stat->sc.ScriptMessage("Parse %type translation_unit {ZCC_TreeNode *} translation_unit(X) ::= . { X = NULL; } -translation_unit(X) ::= translation_unit(A) external_declaration(B). { SAFE_APPEND(A,B); X = A; } +translation_unit(X) ::= translation_unit(A) external_declaration(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } translation_unit(X) ::= translation_unit(X) EOF. translation_unit(X) ::= error. { X = NULL; } %type external_declaration {ZCC_TreeNode *} -external_declaration(X) ::= class_definition(A). { X = A; } -external_declaration(X) ::= struct_def(A). { X = A; } -external_declaration(X) ::= enum_def(A). { X = A; } -external_declaration(X) ::= const_def(A). { X = A; } +external_declaration(X) ::= class_definition(A). { X = A; /*X-overwrites-A*/ } +external_declaration(X) ::= struct_def(A). { X = A; /*X-overwrites-A*/ } +external_declaration(X) ::= enum_def(A). { X = A; /*X-overwrites-A*/ } +external_declaration(X) ::= const_def(A). { X = A; /*X-overwrites-A*/ } /* Optional bits. */ opt_semicolon ::= . @@ -165,7 +165,7 @@ opt_expr(X) ::= expr(X). class_definition(X) ::= class_head(A) class_body(B). { A->Body = B; - X = A; + X = A; /*X-overwrites-A*/ } class_head(X) ::= CLASS(T) IDENTIFIER(A) class_ancestry(B) class_flags(C). @@ -180,7 +180,7 @@ class_head(X) ::= CLASS(T) IDENTIFIER(A) class_ancestry(B) class_flags(C). %type class_ancestry{ZCC_Identifier *} class_ancestry(X) ::= . { X = NULL; } -class_ancestry(X) ::= COLON dottable_id(A). { X = A; } +class_ancestry(X) ::= COLON dottable_id(A). { X = A; /*X-overwrites-A*/ } %type class_flags{ClassFlagsBlock} class_flags(X) ::= . { X.Flags = 0; X.Replaces = NULL; } @@ -204,7 +204,7 @@ dottable_id(X) ::= dottable_id(A) DOT IDENTIFIER(B). NEW_AST_NODE(Identifier,id2,A); id2->Id = B.Name(); A->AppendSibling(id2); - X = A; + X = A; /*X-overwrites-A*/ } /*------ Class Body ------*/ @@ -217,23 +217,23 @@ dottable_id(X) ::= dottable_id(A) DOT IDENTIFIER(B). // * constants // * defaults -class_body(X) ::= SEMICOLON class_innards(A) EOF. { X = A; } -class_body(X) ::= LBRACE class_innards(A) RBRACE. { X = A; } +class_body(X) ::= SEMICOLON class_innards(A) EOF. { X = A; /*X-overwrites-A*/ } +class_body(X) ::= LBRACE class_innards(A) RBRACE. { X = A; /*X-overwrites-A*/ } class_innards(X) ::= . { X = NULL; } -class_innards(X) ::= class_innards(A) class_member(B). { SAFE_APPEND(A,B); X = A; } +class_innards(X) ::= class_innards(A) class_member(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } %type struct_def{ZCC_Struct *} %type enum_def {ZCC_Enum *} %type states_def {ZCC_States *} %type const_def {ZCC_ConstantDef *} -class_member(X) ::= declarator(A). { X = A; } -class_member(X) ::= enum_def(A). { X = A; } -class_member(X) ::= struct_def(A). { X = A; } -class_member(X) ::= states_def(A). { X = A; } -class_member(X) ::= default_def(A). { X = A; } -class_member(X) ::= const_def(A). { X = A; } +class_member(X) ::= declarator(A). { X = A; /*X-overwrites-A*/ } +class_member(X) ::= enum_def(A). { X = A; /*X-overwrites-A*/ } +class_member(X) ::= struct_def(A). { X = A; /*X-overwrites-A*/ } +class_member(X) ::= states_def(A). { X = A; /*X-overwrites-A*/ } +class_member(X) ::= default_def(A). { X = A; /*X-overwrites-A*/ } +class_member(X) ::= const_def(A). { X = A; /*X-overwrites-A*/ } /*----- Struct Definition -----*/ /* Structs can define variables and enums. */ @@ -255,11 +255,11 @@ opt_struct_body(X) ::= struct_body(X). struct_body(X) ::= error. { X = NULL; } struct_body(X) ::= struct_member(X). -struct_body(X) ::= struct_member(A) struct_body(B). { X = A; A->AppendSibling(B); } +struct_body(X) ::= struct_member(A) struct_body(B). { X = A; /*X-overwrites-A*/ X->AppendSibling(B); } -struct_member(X) ::= declarator_no_fun(A). { X = A; } -struct_member(X) ::= enum_def(A). { X = A; } -struct_member(X) ::= const_def(A). { X = A; } +struct_member(X) ::= declarator_no_fun(A). { X = A; /*X-overwrites-A*/ } +struct_member(X) ::= enum_def(A). { X = A; /*X-overwrites-A*/ } +struct_member(X) ::= const_def(A). { X = A; /*X-overwrites-A*/ } /*----- Constant Definition ------*/ /* Like UnrealScript, a constant's type is implied by its value's type. */ @@ -340,11 +340,11 @@ enum_def(X) ::= ENUM(T) IDENTIFIER(A) enum_type(B) LBRACE opt_enum_list(C) RBRAC } enum_type(X) ::= . { X.Int = ZCC_IntAuto; X.SourceLoc = stat->sc.GetMessageLine(); } -enum_type(X) ::= COLON int_type(A). { X = A; } +enum_type(X) ::= COLON int_type(A). { X = A; /*X-overwrites-A*/ } enum_list(X) ::= error. { X = NULL; } enum_list(X) ::= enumerator(X). -enum_list(X) ::= enum_list(A) COMMA enumerator(B). { X = A; A->AppendSibling(B); } +enum_list(X) ::= enum_list(A) COMMA enumerator(B). { X = A; /*X-overwrites-A*/ X->AppendSibling(B); } opt_enum_list(X) ::= . { X = NULL; } opt_enum_list(X) ::= enum_list(X) opt_comma. @@ -402,9 +402,9 @@ scanner_mode ::= . { stat->sc.SetStateMode(true); } states_body(X) ::= . { X = NULL; } states_body(X) ::= error. { X = NULL; } -states_body(X) ::= states_body(A) state_line(B). { SAFE_APPEND(A,B); X = A; } -states_body(X) ::= states_body(A) state_label(B). { SAFE_APPEND(A,B); X = A; } -states_body(X) ::= states_body(A) state_flow(B). { SAFE_APPEND(A,B); X = A; } +states_body(X) ::= states_body(A) state_line(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } +states_body(X) ::= states_body(A) state_label(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } +states_body(X) ::= states_body(A) state_flow(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } state_label(X) ::= NWS(A) COLON. { @@ -428,7 +428,7 @@ state_flow_type(X) ::= GOTO(T) dottable_id(A) state_goto_offset(B). } state_goto_offset(X) ::= . { X = NULL; } -state_goto_offset(X) ::= PLUS expr(A). { X = A; } /* Must evaluate to a non-negative integer constant. */ +state_goto_offset(X) ::= PLUS expr(A). { X = A; /*X-overwrites-A*/ } /* Must evaluate to a non-negative integer constant. */ state_line(X) ::= NWS(A) NWS(B) expr state_opts(C) state_action(D). { @@ -454,21 +454,21 @@ state_line(X) ::= NWS(A) NWS(B) expr state_opts(C) state_action(D). } state_opts(X) ::= . { StateOpts opts; opts.Zero(); X = opts; } -state_opts(X) ::= state_opts(A) BRIGHT. { A.Bright = true; X = A; } -state_opts(X) ::= state_opts(A) FAST. { A.Fast = true; X = A; } -state_opts(X) ::= state_opts(A) SLOW. { A.Slow = true; X = A; } -state_opts(X) ::= state_opts(A) NODELAY. { A.NoDelay = true; X = A; } -state_opts(X) ::= state_opts(A) CANRAISE. { A.CanRaise = true; X = A; } -state_opts(X) ::= state_opts(A) OFFSET LPAREN expr(B) COMMA expr(C) RPAREN. { A.Offset = B; B->AppendSibling(C); X = A; } -state_opts(X) ::= state_opts(A) LIGHT LPAREN light_list RPAREN. { X = A; } ///FIXME: GZDoom would want to know this +state_opts(X) ::= state_opts(A) BRIGHT. { A.Bright = true; X = A; /*X-overwrites-A*/ } +state_opts(X) ::= state_opts(A) FAST. { A.Fast = true; X = A; /*X-overwrites-A*/ } +state_opts(X) ::= state_opts(A) SLOW. { A.Slow = true; X = A; /*X-overwrites-A*/ } +state_opts(X) ::= state_opts(A) NODELAY. { A.NoDelay = true; X = A; /*X-overwrites-A*/ } +state_opts(X) ::= state_opts(A) CANRAISE. { A.CanRaise = true; X = A; /*X-overwrites-A*/ } +state_opts(X) ::= state_opts(A) OFFSET LPAREN expr(B) COMMA expr(C) RPAREN. { A.Offset = B; B->AppendSibling(C); X = A; /*X-overwrites-A*/ } +state_opts(X) ::= state_opts(A) LIGHT LPAREN light_list RPAREN. { X = A; /*X-overwrites-A*/ } ///FIXME: GZDoom would want to know this light_list ::= STRCONST. light_list ::= light_list COMMA STRCONST. /* A state action can be either a compound statement or a single action function call. */ -state_action(X) ::= LBRACE statement_list(A) scanner_mode RBRACE. { X = A; } +state_action(X) ::= LBRACE statement_list(A) scanner_mode RBRACE. { X = A; /*X-overwrites-A*/ } state_action(X) ::= LBRACE error scanner_mode RBRACE. { X = NULL; } -state_action(X) ::= state_call(A) scanner_mode SEMICOLON. { X = A; } +state_action(X) ::= state_call(A) scanner_mode SEMICOLON. { X = A; /*X-overwrites-A*/ } state_call(X) ::= . { X = NULL; } state_call(X) ::= IDENTIFIER(A) state_call_params(B). @@ -485,11 +485,11 @@ state_call(X) ::= IDENTIFIER(A) state_call_params(B). } state_call_params(X) ::= . { X = NULL; } -state_call_params(X) ::= LPAREN func_expr_list(A) RPAREN. { X = A; } +state_call_params(X) ::= LPAREN func_expr_list(A) RPAREN. { X = A; /*X-overwrites-A*/ } /* Definition of a default class instance. */ %type default_def {ZCC_CompoundStmt *} -default_def(X) ::= DEFAULT compound_statement(A). { X = A; } +default_def(X) ::= DEFAULT compound_statement(A). { X = A; /*X-overwrites-A*/ } /* Type names */ %type type_name {ZCC_BasicType *} @@ -590,16 +590,16 @@ aggregate_type(X) ::= CLASS(T) class_restrictor(A). /* class */ X = cls; } class_restrictor(X) ::= . { X = NULL; } -class_restrictor(X) ::= LT dottable_id(A) GT. { X = A; } +class_restrictor(X) ::= LT dottable_id(A) GT. { X = A; /*X-overwrites-A*/ } -type(X) ::= type_name(A). { X = A; A->ArraySize = NULL; } -type(X) ::= aggregate_type(A). { X = A; A->ArraySize = NULL; } +type(X) ::= type_name(A). { X = A; /*X-overwrites-A*/ X->ArraySize = NULL; } +type(X) ::= aggregate_type(A). { X = A; /*X-overwrites-A*/ X->ArraySize = NULL; } type_or_array(X) ::= type(X). -type_or_array(X) ::= type(A) array_size(B). { X = A; A->ArraySize = B; } +type_or_array(X) ::= type(A) array_size(B). { X = A; /*X-overwrites-A*/ X->ArraySize = B; } type_list(X) ::= type_or_array(X). /* A comma-separated list of types */ -type_list(X) ::= type_list(A) COMMA type_or_array(B). { X = A; A->AppendSibling(B); } +type_list(X) ::= type_list(A) COMMA type_or_array(B). { X = A; /*X-overwrites-A*/ X->AppendSibling(B); } type_list_or_void(X) ::= VOID. { X = NULL; } type_list_or_void(X) ::= type_list(X). @@ -622,7 +622,7 @@ array_size(X) ::= array_size_expr(X). array_size(X) ::= array_size(A) array_size_expr(B). { A->AppendSibling(B); - X = A; + X = A; /*X-overwrites-A*/ } %type variables_or_function {VarOrFun} @@ -735,7 +735,7 @@ variable_list(X) ::= variable_name(X). variable_list(X) ::= variable_list(A) COMMA variable_name(B). { A->AppendSibling(B); - X = A; + X = A; /*X-overwrites-A*/ } decl_flags(X) ::= . { X.Int = 0; X.SourceLoc = 0; } @@ -765,7 +765,7 @@ func_params(X) ::= VOID. { X = NULL; } func_params(X) ::= func_param_list(X). func_param_list(X) ::= func_param(X). -func_param_list(X) ::= func_param_list(A) COMMA func_param(B). { X = A; A->AppendSibling(B); } +func_param_list(X) ::= func_param_list(A) COMMA func_param(B). { X = A; /*X-overwrites-A*/ X->AppendSibling(B); } func_param(X) ::= func_param_flags(A) type(B) IDENTIFIER(C). { @@ -809,7 +809,7 @@ primary(X) ::= SUPER(T). expr->Type = NULL; X = expr; } -primary(X) ::= constant(A). { X = A; } +primary(X) ::= constant(A). { X = A; /*X-overwrites-A*/ } primary(X) ::= SELF(T). { NEW_AST_NODE(Expression, expr, T); @@ -819,7 +819,7 @@ primary(X) ::= SELF(T). } primary(X) ::= LPAREN expr(A) RPAREN. { - X = A; + X = A; /*X-overwrites-A*/ } primary ::= LPAREN error RPAREN. primary(X) ::= primary(A) LPAREN func_expr_list(B) RPAREN. [DOT] // Function call @@ -1102,8 +1102,8 @@ expr(X) ::= expr(A) QUESTION expr(B) COLON expr(C). expr_list(X) ::= expr(X). expr_list(X) ::= expr_list(A) COMMA expr(B). { - X = A; - A->AppendSibling(B); + X = A; /*X-overwrites-A*/ + X->AppendSibling(B); } /*----- Function argument lists -----*/ @@ -1133,8 +1133,8 @@ func_expr_list(X) ::= func_expr_list(A) COMMA(T) func_expr_item(B). nil_b->Label = NAME_None; B = nil_b; } - X = A; - A->AppendSibling(B); + X = A; /*X-overwrites-A*/ + X->AppendSibling(B); } func_expr_item(X) ::= . @@ -1224,14 +1224,14 @@ function_body(X) ::= compound_statement(X). %type statement{ZCC_Statement *} statement(X) ::= SEMICOLON. { X = NULL; } -statement(X) ::= labeled_statement(A). { X = A; } -statement(X) ::= compound_statement(A). { X = A; } -statement(X) ::= expression_statement(A) SEMICOLON. { X = A; } +statement(X) ::= labeled_statement(A). { X = A; /*X-overwrites-A*/ } +statement(X) ::= compound_statement(A). { X = A; /*X-overwrites-A*/ } +statement(X) ::= expression_statement(A) SEMICOLON. { X = A; /*X-overwrites-A*/ } statement(X) ::= selection_statement(X). statement(X) ::= iteration_statement(X). statement(X) ::= jump_statement(X). -statement(X) ::= assign_statement(A) SEMICOLON. { X = A; } -statement(X) ::= local_var(A) SEMICOLON. { X = A; } +statement(X) ::= assign_statement(A) SEMICOLON. { X = A; /*X-overwrites-A*/ } +statement(X) ::= local_var(A) SEMICOLON. { X = A; /*X-overwrites-A*/ } statement(X) ::= error SEMICOLON. { X = NULL; } /*----- Jump Statements -----*/ @@ -1287,12 +1287,12 @@ compound_statement(X) ::= LBRACE(T) error RBRACE. statement_list(X) ::= statement(A). { - X = A; + X = A; /*X-overwrites-A*/ } statement_list(X) ::= statement_list(A) statement(B). { SAFE_APPEND(A,B); - X = A; + X = A; /*X-overwrites-A*/ } /*----- Expression Statements -----*/ @@ -1376,13 +1376,13 @@ while_or_until(X) ::= UNTIL(T). } %type for_init{ZCC_Statement *} -for_init(X) ::= local_var(A). { X = A; } -for_init(X) ::= for_bump(A). { X = A; } +for_init(X) ::= local_var(A). { X = A /*X-overwrites-A*/; } +for_init(X) ::= for_bump(A). { X = A /*X-overwrites-A*/; } %type for_bump{ZCC_Statement *} for_bump(X) ::= . { X = NULL; } -for_bump(X) ::= expression_statement(A). { X = A; } -for_bump(X) ::= assign_statement(A). { X = A; } +for_bump(X) ::= expression_statement(A). { X = A; /*X-overwrites-A*/ } +for_bump(X) ::= assign_statement(A). { X = A; /*X-overwrites-A*/ } /*----- If Statements -----*/ @@ -1397,12 +1397,12 @@ for_bump(X) ::= assign_statement(A). { X = A; } selection_statement(X) ::= if_front(A). [IF] { - X = A; + X = A; /*X-overwrites-A*/ } selection_statement(X) ::= if_front(A) ELSE statement(B). [ELSE] { A->FalsePath = B; - X = A; + X = A; /*X-overwrites-A*/ } if_front(X) ::= IF(T) LPAREN expr(A) RPAREN statement(B). @@ -1481,4 +1481,4 @@ local_var(X) ::= type(A) variable_list(B) var_init(C). %type var_init{ZCC_Expression *} var_init(X) ::= . { X = NULL; } -var_init(X) ::= EQ expr_list(A). { X = A; } +var_init(X) ::= EQ expr_list(A). { X = A; /*X-overwrites-A*/ } From f38dbc5055ccd7c95b58a9fb101eff41e8480d15 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 20 Mar 2016 15:53:04 -0500 Subject: [PATCH 47/47] Use LHS/RHS matching for some simple SAFE_APPENDs --- src/zscript/zcc-parse.lemon | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/zscript/zcc-parse.lemon b/src/zscript/zcc-parse.lemon index 37f69842f..ee8296ff6 100644 --- a/src/zscript/zcc-parse.lemon +++ b/src/zscript/zcc-parse.lemon @@ -128,7 +128,7 @@ main ::= translation_unit(A). { stat->TopNode = A; stat->sc.ScriptMessage("Parse %type translation_unit {ZCC_TreeNode *} translation_unit(X) ::= . { X = NULL; } -translation_unit(X) ::= translation_unit(A) external_declaration(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } +translation_unit(X) ::= translation_unit(X) external_declaration(B). { SAFE_APPEND(X,B); } translation_unit(X) ::= translation_unit(X) EOF. translation_unit(X) ::= error. { X = NULL; } @@ -221,7 +221,7 @@ class_body(X) ::= SEMICOLON class_innards(A) EOF. { X = A; /*X-overwrites-A*/ } class_body(X) ::= LBRACE class_innards(A) RBRACE. { X = A; /*X-overwrites-A*/ } class_innards(X) ::= . { X = NULL; } -class_innards(X) ::= class_innards(A) class_member(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } +class_innards(X) ::= class_innards(X) class_member(B). { SAFE_APPEND(X,B); } %type struct_def{ZCC_Struct *} %type enum_def {ZCC_Enum *} @@ -402,9 +402,9 @@ scanner_mode ::= . { stat->sc.SetStateMode(true); } states_body(X) ::= . { X = NULL; } states_body(X) ::= error. { X = NULL; } -states_body(X) ::= states_body(A) state_line(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } -states_body(X) ::= states_body(A) state_label(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } -states_body(X) ::= states_body(A) state_flow(B). { SAFE_APPEND(A,B); X = A; /*X-overwrites-A*/ } +states_body(X) ::= states_body(X) state_line(B). { SAFE_APPEND(X,B); } +states_body(X) ::= states_body(X) state_label(B). { SAFE_APPEND(X,B); } +states_body(X) ::= states_body(X) state_flow(B). { SAFE_APPEND(X,B); } state_label(X) ::= NWS(A) COLON. { @@ -1289,10 +1289,9 @@ statement_list(X) ::= statement(A). { X = A; /*X-overwrites-A*/ } -statement_list(X) ::= statement_list(A) statement(B). +statement_list(X) ::= statement_list(X) statement(B). { - SAFE_APPEND(A,B); - X = A; /*X-overwrites-A*/ + SAFE_APPEND(X,B); } /*----- Expression Statements -----*/