From de56be6c01bdb2747b93a5072591f2af29363f2d Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 21:41:54 -0500 Subject: [PATCH 1/8] Lemon update 2016-04-29 11:28:35 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Lemon bug fix: Do not merge action routines unless their destructors are also identical. Problem and suggested fix reported on the mailing list by Kelvin Sherlock. (user: drh) --- tools/lemon/lemon.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 546cf515e..c796f4f12 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -4404,7 +4404,8 @@ void ReportTable( writeRuleText(out, rp); fprintf(out," */\n"); lineno++; for(rp2=rp->next; rp2; rp2=rp2->next){ - if( rp2->code==rp->code ){ + if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix + && rp2->codeSuffix == rp->codeSuffix ){ fprintf(out," case %d: /*",rp2->iRule); writeRuleText(out, rp2); fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; @@ -4413,7 +4414,7 @@ void ReportTable( } emit_code(out,rp,lemp,&lineno); fprintf(out," break;\n"); lineno++; - rp->code = 0; + rp->code = 0; } /* Finally, output the default: rule. We choose as the default: all ** empty actions. */ From 45d441f103160e3a9f326b97d3b2ff23392aa7b2 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 21:48:29 -0500 Subject: [PATCH 2/8] Lemon update 2016-04-29 11:28:35 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Lemon bug fix: Do not merge action routines unless their destructors are also identical. Problem and suggested fix reported on the mailing list by Kelvin Sherlock. (user: drh) --- tools/lemon/lemon.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index c796f4f12..56364f897 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -3559,10 +3559,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ rp->line = rp->ruleline; } - if( rp->lhsalias==0 ){ - /* There is no LHS value symbol. */ - lhsdirect = 1; - }else if( rp->nrhs==0 ){ + if( rp->nrhs==0 ){ /* If there are no RHS symbols, then writing directly to the LHS is ok */ lhsdirect = 1; }else if( rp->rhsalias[0]==0 ){ @@ -3575,6 +3572,9 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ rp->rhs[0]->index,1-rp->nrhs,0); rp->codePrefix = Strsafe(append_str(0,0,0,0,0)); } + }else if( rp->lhsalias==0 ){ + /* There is no LHS value symbol. */ + lhsdirect = 1; }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){ /* The LHS symbol and the left-most RHS symbol are the same, so ** direct writing is allowed */ @@ -3717,7 +3717,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ /* Suffix code generation complete */ cp = append_str(0,0,0,0,0); - if( cp ) rp->codeSuffix = Strsafe(cp); + if( cp && cp[0] ) rp->codeSuffix = Strsafe(cp); return rc; } @@ -4399,7 +4399,14 @@ void ReportTable( for(rp=lemp->rule; rp; rp=rp->next){ struct rule *rp2; /* Other rules with the same action */ if( rp->code==0 ) continue; - if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */ + if( rp->code[0]=='\n' + && rp->code[1]==0 + && rp->codePrefix==0 + && rp->codeSuffix==0 + ){ + /* No actions, so this will be part of the "default:" rule */ + continue; + } fprintf(out," case %d: /* ",rp->iRule); writeRuleText(out, rp); fprintf(out," */\n"); lineno++; @@ -4422,6 +4429,8 @@ void ReportTable( for(rp=lemp->rule; rp; rp=rp->next){ if( rp->code==0 ) continue; assert( rp->code[0]=='\n' && rp->code[1]==0 ); + assert( rp->codePrefix==0 ); + assert( rp->codeSuffix==0 ); fprintf(out," /* (%d) ", rp->iRule); writeRuleText(out, rp); fprintf(out," */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; From 0d67d107abb6d7de0a4305c18f506609cd925bb8 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 22:07:01 -0500 Subject: [PATCH 3/8] Lemon update 2016-05-23 14:24:31 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Fix comment typos and improve clarity of presention in Lemon. The output should be identical. (user: drh) --- tools/lemon/lemon.c | 51 ++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 56364f897..27487b160 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -192,6 +192,8 @@ struct rule { const char *code; /* The code executed when this rule is reduced */ const char *codePrefix; /* Setup code before code[] above */ const char *codeSuffix; /* Breakdown code after code[] above */ + int noCode; /* True if this rule has no associated C code */ + int codeEmitted; /* True if the code has been emitted already */ struct symbol *precsym; /* Precedence symbol for this rule */ int index; /* An index number for this rule */ int iRule; /* Rule number as used in the generated tables */ @@ -253,7 +255,7 @@ struct state { struct config *bp; /* The basis configurations for this state */ struct config *cfp; /* All configurations in this set */ int statenum; /* Sequential number for this state */ - struct action *ap; /* Array of actions for this state */ + struct action *ap; /* List of actions for this state */ int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ int iDfltReduce; /* Default action is to REDUCE by this rule */ @@ -1483,7 +1485,7 @@ static void handle_C_option(char *z){ } } -/* Merge together to lists of rules order by rule.iRule */ +/* Merge together to lists of rules ordered by rule.iRule */ static struct rule *Rule_merge(struct rule *pA, struct rule *pB){ struct rule *pFirst = 0; struct rule **ppPrev = &pFirst; @@ -1627,7 +1629,10 @@ int main(int argc, char **argv) for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++); lem.nterminal = i; - /* Assign sequential rule numbers */ + /* Assign sequential rule numbers. Start with 0. Put rules that have no + ** reduce action C-code associated with them last, so that the switch() + ** statement that selects reduction actions will have a smaller jump table. + */ for(i=0, rp=lem.rule; rp; rp=rp->next){ rp->iRule = rp->code ? i++ : -1; } @@ -2179,7 +2184,7 @@ static void parseonetoken(struct pstate *psp) "There is no prior rule upon which to attach the code \ fragment which begins on this line."); psp->errorcnt++; - }else if( psp->prevrule->code!=0 ){ + }else if( psp->prevrule->code!=0 ){ ErrorMsg(psp->filename,psp->tokenlineno, "Code fragment beginning on this line is not the first \ to follow the previous rule."); @@ -2187,7 +2192,8 @@ to follow the previous rule."); }else{ psp->prevrule->line = psp->tokenlineno; psp->prevrule->code = &x[1]; - } + psp->prevrule->noCode = 0; + } }else if( x[0]=='[' ){ psp->state = PRECEDENCE_MARK_1; }else{ @@ -2293,6 +2299,7 @@ to follow the previous rule."); rp->lhsalias = psp->lhsalias; rp->nrhs = psp->nrhs; rp->code = 0; + rp->noCode = 1; rp->precsym = 0; rp->index = psp->gp->nrule++; rp->nextlhs = rp->lhs->rule; @@ -3531,9 +3538,8 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2, int bNoSubst) } /* -** zCode is a string that is the action associated with a rule. Expand -** the symbols in this string so that the refer to elements of the parser -** stack. +** Write and transform the rp->code string so that symbols are expanded. +** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate. ** ** Return 1 if the expanded code requires that "yylhsminor" local variable ** to be defined. @@ -3557,6 +3563,9 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ static char newlinestr[2] = { '\n', '\0' }; rp->code = newlinestr; rp->line = rp->ruleline; + rp->noCode = 1; + }else{ + rp->noCode = 0; } if( rp->nrhs==0 ){ @@ -3571,6 +3580,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, rp->rhs[0]->index,1-rp->nrhs,0); rp->codePrefix = Strsafe(append_str(0,0,0,0,0)); + rp->noCode = 0; } }else if( rp->lhsalias==0 ){ /* There is no LHS value symbol. */ @@ -3717,7 +3727,10 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ /* Suffix code generation complete */ cp = append_str(0,0,0,0,0); - if( cp && cp[0] ) rp->codeSuffix = Strsafe(cp); + if( cp && cp[0] ){ + rp->codeSuffix = Strsafe(cp); + rp->noCode = 0; + } return rc; } @@ -4398,13 +4411,9 @@ void ReportTable( /* First output rules other than the default: rule */ for(rp=lemp->rule; rp; rp=rp->next){ struct rule *rp2; /* Other rules with the same action */ - if( rp->code==0 ) continue; - if( rp->code[0]=='\n' - && rp->code[1]==0 - && rp->codePrefix==0 - && rp->codeSuffix==0 - ){ - /* No actions, so this will be part of the "default:" rule */ + if( rp->codeEmitted ) continue; + if( rp->noCode ){ + /* No C code actions, so this will be part of the "default:" rule */ continue; } fprintf(out," case %d: /* ",rp->iRule); @@ -4416,21 +4425,19 @@ void ReportTable( fprintf(out," case %d: /*",rp2->iRule); writeRuleText(out, rp2); fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; - rp2->code = 0; + rp2->codeEmitted = 1; } } emit_code(out,rp,lemp,&lineno); fprintf(out," break;\n"); lineno++; - rp->code = 0; + rp->codeEmitted = 1; } /* Finally, output the default: rule. We choose as the default: all ** empty actions. */ fprintf(out," default:\n"); lineno++; for(rp=lemp->rule; rp; rp=rp->next){ - if( rp->code==0 ) continue; - assert( rp->code[0]=='\n' && rp->code[1]==0 ); - assert( rp->codePrefix==0 ); - assert( rp->codeSuffix==0 ); + if( rp->codeEmitted ) continue; + assert( rp->noCode ); fprintf(out," /* (%d) ", rp->iRule); writeRuleText(out, rp); fprintf(out," */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; From 64bd551d203c4f302ecb933a3773a282b01f4716 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 22:09:13 -0500 Subject: [PATCH 4/8] Lemon update 2016-05-23 14:24:31 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Fix comment typos and improve clarity of presention in Lemon. The output should be identical. (user: drh) --- tools/lemon/lemon.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 27487b160..9994b940f 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -245,6 +245,7 @@ struct action { struct state *stp; /* The new state, if a shift */ struct rule *rp; /* The rule, if a reduce */ } x; + struct symbol *spOpt; /* SHIFTREDUCE optimization to this symbol */ struct action *next; /* Next action for this state */ struct action *collide; /* Next action with the same hash */ }; @@ -435,6 +436,7 @@ void Action_add( *app = newaction; newaction->type = type; newaction->sp = sp; + newaction->spOpt = 0; if( type==SHIFT ){ newaction->x.stp = (struct state *)arg; }else{ @@ -3160,6 +3162,9 @@ int PrintAction( result = 0; break; } + if( result && ap->spOpt ){ + fprintf(fp," /* because %s==%s */", ap->sp->name, ap->spOpt->name); + } return result; } @@ -4513,7 +4518,7 @@ void ReportHeader(struct lemon *lemp) void CompressTables(struct lemon *lemp) { struct state *stp; - struct action *ap, *ap2; + struct action *ap, *ap2, *nextap; struct rule *rp, *rp2, *rbest; int nbest, n; int i; @@ -4590,6 +4595,36 @@ void CompressTables(struct lemon *lemp) } } } + + /* If a SHIFTREDUCE action specifies a rule that has a single RHS term + ** (meaning that the SHIFTREDUCE will land back in the state where it + ** started) and if there is no C-code associated with the reduce action, + ** then we can go ahead and convert the action to be the same as the + ** action for the RHS of the rule. + */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=nextap){ + nextap = ap->next; + if( ap->type!=SHIFTREDUCE ) continue; + rp = ap->x.rp; + if( rp->noCode==0 ) continue; + if( rp->nrhs!=1 ) continue; +#if 1 + /* Only apply this optimization to non-terminals. It would be OK to + ** apply it to terminal symbols too, but that makes the parser tables + ** larger. */ + if( ap->sp->indexnterminal ) continue; +#endif + /* If we reach this point, it means the optimization can be applied */ + nextap = ap; + for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){} + assert( ap2!=0 ); + ap->spOpt = ap2->sp; + ap->type = ap2->type; + ap->x = ap2->x; + } + } } From 96afce241d6bc08cfdb0018ea6e5f0c23e8ba5c8 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 22:18:20 -0500 Subject: [PATCH 5/8] Lemon update 2016-05-24 18:55:08 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Enhance Lemon and the parser template so that it can once again build parsers that have no unreachable branches. (user: drh) --- tools/lemon/lemon.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 9994b940f..a1453716a 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -198,6 +198,7 @@ struct rule { int index; /* An index number for this rule */ int iRule; /* Rule number as used in the generated tables */ Boolean canReduce; /* True if this rule is ever reduced */ + Boolean doesReduce; /* Reduce actions occur after optimization */ struct rule *nextlhs; /* Next rule with the same LHS */ struct rule *next; /* Next rule in the global list */ }; @@ -4154,6 +4155,19 @@ void ReportTable( } free(ax); + /* Mark rules that are actually used for reduce actions after all + ** optimizations have been applied + */ + for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE; + for(i=0; inxstate; i++){ + struct action *ap; + for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){ + if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){ + ap->x.rp->doesReduce = i; + } + } + } + /* Finish rendering the constants now that the action table has ** been computed */ fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; @@ -4445,7 +4459,12 @@ void ReportTable( assert( rp->noCode ); fprintf(out," /* (%d) ", rp->iRule); writeRuleText(out, rp); - fprintf(out," */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; + if( rp->doesReduce ){ + fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; + }else{ + fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n", + rp->iRule); lineno++; + } } fprintf(out," break;\n"); lineno++; tplt_xfer(lemp->name,in,out,&lineno); From 3b1a04888506adf9b97424d9e31c185c51873798 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 22:22:04 -0500 Subject: [PATCH 6/8] Lemon update 2016-05-24 18:55:08 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Enhance Lemon and the parser template so that it can once again build parsers that have no unreachable branches. (user: drh) --- tools/lemon/lemon.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index a1453716a..1e673be8a 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -4233,20 +4233,21 @@ void ReportTable( fprintf(out, "};\n"); lineno++; /* Output the yy_shift_ofst[] table */ - fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++; n = lemp->nxstate; while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; - fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; - fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; - fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; + fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++; + fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; + fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; fprintf(out, "static const %s yy_shift_ofst[] = {\n", - minimum_size_type(mnTknOfst-1, mxTknOfst, &sz)); lineno++; + minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz)); + lineno++; lemp->tablesize += n*sz; for(i=j=0; isorted[i]; ofst = stp->iTknOfst; - if( ofst==NO_OFFSET ) ofst = mnTknOfst - 1; + if( ofst==NO_OFFSET ) ofst = lemp->nactiontab; if( j==0 ) fprintf(out," /* %5d */ ", i); fprintf(out, " %4d,", ofst); if( j==9 || i==n-1 ){ From 696beca40f92e33d08ac976f1bf5a7753aa01089 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 22:23:28 -0500 Subject: [PATCH 7/8] Lemon update 2016-08-16 16:46:40 on branch trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — Fix a bug in destructorprocessing of Lemon. That has no impact on the SQLite grammar. The bug was introduced by prior work to optimize the Lemon-generated parser used by SQLite. (user: drh) --- tools/lemon/lemon.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index 1e673be8a..f2da9c6f8 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -167,7 +167,8 @@ struct symbol { int useCnt; /* Number of times used */ char *destructor; /* Code which executes whenever this symbol is ** popped from the stack during error processing */ - int destLineno; /* Line number for start of destructor */ + int destLineno; /* Line number for start of destructor. Set to + ** -1 for duplicate destructors. */ char *datatype; /* The data type of information held by this ** object. Only used if type==NONTERMINAL */ int dtnum; /* The data type number. In the parser, the value @@ -4387,6 +4388,7 @@ void ReportTable( for(i=0; insymbol; i++){ struct symbol *sp = lemp->symbols[i]; if( sp==0 || sp->type==TERMINAL || sp->destructor==0 ) continue; + if( sp->destLineno<0 ) continue; /* Already emitted */ fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; /* Combine duplicate destructors into a single case */ @@ -4397,7 +4399,7 @@ void ReportTable( && strcmp(sp->destructor,sp2->destructor)==0 ){ fprintf(out," case %d: /* %s */\n", sp2->index, sp2->name); lineno++; - sp2->destructor = 0; + sp2->destLineno = -1; /* Avoid emitting this destructor again */ } } From 97107b6b6d7704660e356bbb016f132e2bfebe91 Mon Sep 17 00:00:00 2001 From: Marisa Heit Date: Thu, 13 Oct 2016 22:30:12 -0500 Subject: [PATCH 8/8] Update lempar.c to 2016-10-04 version - Every update rolled into one, because I'm pretty sure I missed some while updating lemon.c (not counting today's commits), since it wasn't always updated at the same time as lemon.c. - In particular, I think this check-in from 2016-06-06 was very important to us after commit 3d5867d29e8932d238084e5c66dead1b4502f82f (For the Lemon-generated parser, add a new action type SHIFTREDUCE and use it to further compress the parser tables and improve parser performance.): * Fix lempar.c so that the shift-reduce optimization works for error processing. --- src/zscript/zcc-parse.lemon | 2 +- tools/lemon/lempar.c | 263 +++++++++++++++++++----------------- 2 files changed, 141 insertions(+), 124 deletions(-) diff --git a/src/zscript/zcc-parse.lemon b/src/zscript/zcc-parse.lemon index ee8296ff6..12cbb6625 100644 --- a/src/zscript/zcc-parse.lemon +++ b/src/zscript/zcc-parse.lemon @@ -77,7 +77,7 @@ static void SetNodeLine(ZCC_TreeNode *name, int line) FString unexpected, expecting; int i; - int stateno = yypParser->yystack[yypParser->yyidx].stateno; + int stateno = yypParser->yytos->stateno; unexpected << "Unexpected " << ZCCTokenName(yymajor); diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index 0549a9dad..0e360bab9 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -41,6 +41,7 @@ ***************** Begin makeheaders token definitions *************************/ %% /**************** End makeheaders token definitions ***************************/ + /* The next section is a series of control #defines. ** various aspects of the generated parser. ** YYCODETYPE is the data type used to store the integer codes @@ -50,8 +51,6 @@ ** YYNOCODE is a number of type YYCODETYPE that is not used for ** any terminal or nonterminal symbol. ** YYFALLBACK If defined, this indicates that one or more tokens -** have fall-back values which should be used if the -** original value of the token will not parse. ** (also known as: "terminal symbols") have fall-back ** values which should be used if the original symbol ** would not parse. This permits keywords to sometimes @@ -126,7 +125,7 @@ ** ** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE ** and YY_MAX_REDUCE - +** ** N == YY_ERROR_ACTION A syntax error has occurred. ** ** N == YY_ACCEPT_ACTION The parser accepts its input. @@ -135,16 +134,20 @@ ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as +** Given state S and lookahead X, the action is computed as either: ** -** yy_action[ yy_shift_ofst[S] + X ] +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] ** -** If the index value yy_shift_ofst[S]+X is out of range or if the value -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] -** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table -** and that yy_default[S] should be used instead. +** The (A) formula is preferred. The B formula is used instead if: +** (1) The yy_shift_ofst[S]+X value is out of range, or +** (2) yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or +** (3) yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT. +** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that +** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X. +** Hence only tests (1) and (2) need to be evaluated.) ** -** The formula above is for computing the action when the lookahead is +** The formulas above are for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the yy_reduce_ofst[] array is used in place of ** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of @@ -213,9 +216,9 @@ typedef struct yyStackEntry yyStackEntry; /* The state of the parser is completely contained in an instance of ** the following structure */ struct yyParser { - int yyidx; /* Index of top element in stack */ + yyStackEntry *yytos; /* Pointer to top element of the stack */ #ifdef YYTRACKMAXSTACKDEPTH - int yyidxMax; /* Maximum value of yyidx */ + int yyhwm; /* High-water mark of the stack */ #endif #ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ @@ -224,6 +227,7 @@ struct yyParser { #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ #else yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ #endif @@ -278,26 +282,37 @@ static const char *const yyRuleName[] = { }; #endif /* NDEBUG */ + #if YYSTACKDEPTH<=0 /* -** Try to increase the size of the parser stack. +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. */ -static void yyGrowStack(yyParser *p){ +static int yyGrowStack(yyParser *p){ int newSize; + int idx; yyStackEntry *pNew; newSize = p->yystksz*2 + 100; - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } if( pNew ){ p->yystack = pNew; - p->yystksz = newSize; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", - yyTracePrompt, p->yystksz); + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); } #endif + p->yystksz = newSize; } + return pNew==0; } #endif @@ -326,15 +341,24 @@ void *ParseAlloc(void *(CDECL *mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( pParser ){ - pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH - pParser->yyidxMax = 0; + pParser->yyhwm = 0; #endif #if YYSTACKDEPTH<=0 + pParser->yytos = NULL; pParser->yystack = NULL; pParser->yystksz = 0; - yyGrowStack(pParser); + if( yyGrowStack(pParser) ){ + pParser->yystack = &pParser->yystk0; + pParser->yystksz = 1; + } #endif +#ifndef YYNOERRORRECOVERY + pParser->yyerrcnt = -1; +#endif + pParser->yytos = pParser->yystack; + pParser->yystack[0].stateno = 0; + pParser->yystack[0].major = 0; } return pParser; } @@ -378,8 +402,9 @@ static void yy_destructor( */ static void yy_pop_parser_stack(yyParser *pParser){ yyStackEntry *yytos; - assert( pParser->yyidx>=0 ); - yytos = &pParser->yystack[pParser->yyidx--]; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sPopping %s\n", @@ -390,10 +415,10 @@ static void yy_pop_parser_stack(yyParser *pParser){ yy_destructor(pParser, yytos->major, &yytos->minor); } -/* +/* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. -* +** ** If the YYPARSEFREENEVERNULL macro exists (for example because it ** is defined in a %include section of the input grammar) then it is ** assumed that the input pointer is never NULL. @@ -406,9 +431,9 @@ void ParseFree( #ifndef YYPARSEFREENEVERNULL if( pParser==0 ) return; #endif - while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 - free(pParser->yystack); + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); #endif (*freeProc)((void*)pParser); } @@ -419,7 +444,7 @@ void ParseFree( #ifdef YYTRACKMAXSTACKDEPTH int ParseStackPeak(void *p){ yyParser *pParser = (yyParser*)p; - return pParser->yyidxMax; + return pParser->yyhwm; } #endif @@ -432,56 +457,53 @@ static unsigned int yy_find_shift_action( YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; - int stateno = pParser->yystack[pParser->yyidx].stateno; + int stateno = pParser->yytos->stateno; if( stateno>=YY_MIN_REDUCE ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); do{ i = yy_shift_ofst[stateno]; - if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ #ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } #endif #ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( + { + int j = i - iLookAhead + YYWILDCARD; + if( #if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && + j>=0 && #endif #if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 + ){ #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); - } -#endif /* NDEBUG */ - return yy_action[j]; + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); } +#endif /* NDEBUG */ + return yy_action[j]; } -#endif /* YYWILDCARD */ } +#endif /* YYWILDCARD */ return yy_default[stateno]; }else{ return yy_action[i]; @@ -500,7 +522,7 @@ static int yy_find_reduce_action( int i; #ifdef YYERRORSYMBOL if( stateno>YY_REDUCE_COUNT ){ - return yy_default[stateno]; + return yy_default[stateno]; } #else assert( stateno<=YY_REDUCE_COUNT ); @@ -525,15 +547,15 @@ static int yy_find_reduce_action( */ static void yyStackOverflow(yyParser *yypParser){ ParseARG_FETCH; - yypParser->yyidx--; + yypParser->yytos--; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser - ** stack ever overflows */ + ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ %% /******** End %stack_overflow code ********************************************/ @@ -548,11 +570,11 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){ if( yyTraceFILE ){ if( yyNewStateyystack[yypParser->yyidx].major], + yyTracePrompt,yyTokenName[yypParser->yytos->major], yyNewState); }else{ fprintf(yyTraceFILE,"%sShift '%s'\n", - yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]); + yyTracePrompt,yyTokenName[yypParser->yytos->major]); } } } @@ -570,27 +592,30 @@ static void yy_shift( ParseTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; - yypParser->yyidx++; + yypParser->yytos++; #ifdef YYTRACKMAXSTACKDEPTH - if( yypParser->yyidx>yypParser->yyidxMax ){ - yypParser->yyidxMax = yypParser->yyidx; + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH ){ +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){ yyStackOverflow(yypParser); return; } #else - if( yypParser->yyidx>=yypParser->yystksz ){ - yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz ){ + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); return; } } #endif - yytos = &yypParser->yystack[yypParser->yyidx]; + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor.yy0 = yyMinor; @@ -622,7 +647,7 @@ static void yy_reduce( yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ ParseARG_FETCH; - yymsp = &yypParser->yystack[yypParser->yyidx]; + yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ yysize = yyRuleInfo[yyruleno].nrhs; @@ -636,22 +661,23 @@ static void yy_reduce( ** enough on the stack to push the LHS value */ if( yyRuleInfo[yyruleno].nrhs==0 ){ #ifdef YYTRACKMAXSTACKDEPTH - if( yypParser->yyidx>yypParser->yyidxMax ){ - yypParser->yyidxMax = yypParser->yyidx; + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH-1 ){ +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){ yyStackOverflow(yypParser); return; } #else - if( yypParser->yyidx>=yypParser->yystksz-1 ){ - yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz-1 ){ + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); return; } + yymsp = yypParser->yytos; } #endif } @@ -674,15 +700,17 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - yypParser->yyidx -= yysize - 1; + if( yyact>YY_MAX_SHIFT ){ + yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } yymsp -= yysize-1; + yypParser->yytos = yymsp; yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yyTraceShift(yypParser, yyact); }else{ assert( yyact == YY_ACCEPT_ACTION ); - yypParser->yyidx -= yysize; + yypParser->yytos -= yysize; yy_accept(yypParser); } } @@ -700,7 +728,7 @@ static void yy_parse_failed( fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ /************ Begin %parse_failure code ***************************************/ @@ -738,7 +766,10 @@ static void yy_accept( fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); /* Here code is inserted which will be executed whenever the ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ @@ -782,28 +813,8 @@ void Parse( #endif yyParser *yypParser; /* The parser */ - /* (re)initialize the parser, if necessary */ yypParser = (yyParser*)yyp; - if( yypParser->yyidx<0 ){ -#if YYSTACKDEPTH<=0 - if( yypParser->yystksz <=0 ){ - yyStackOverflow(yypParser); - return; - } -#endif - yypParser->yyidx = 0; -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n", - yyTracePrompt); - } -#endif - } + assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif @@ -818,7 +829,6 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; yy_shift(yypParser,yyact,yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; @@ -827,11 +837,11 @@ void Parse( }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; #ifdef YYERRORSYMBOL int yymx; #endif - assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); @@ -840,7 +850,7 @@ void Parse( #ifdef YYERRORSYMBOL /* A syntax error has occurred. ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". + ** grammar defines an error token "ERROR". ** ** This is what we do if the grammar does define ERROR: ** @@ -860,7 +870,7 @@ void Parse( if( yypParser->yyerrcnt<0 ){ yy_syntax_error(yypParser,yymajor,yyminor); } - yymx = yypParser->yystack[yypParser->yyidx].major; + yymx = yypParser->yytos->major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ #ifndef NDEBUG if( yyTraceFILE ){ @@ -871,18 +881,20 @@ void Parse( yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); yymajor = YYNOCODE; }else{ - while( - yypParser->yyidx >= 0 && - yymx != YYERRORSYMBOL && - (yyact = yy_find_reduce_action( - yypParser->yystack[yypParser->yyidx].stateno, + while( yypParser->yytos >= yypParser->yystack + && yymx != YYERRORSYMBOL + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } - if( yypParser->yyidx < 0 || yymajor==0 ){ + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); @@ -901,7 +913,7 @@ void Parse( yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; - + #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -919,18 +931,23 @@ void Parse( yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif } yymajor = YYNOCODE; #endif } - }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); + }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); #ifndef NDEBUG if( yyTraceFILE ){ - int i; + yyStackEntry *i; + char cDiv = '['; fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', - yyTokenName[yypParser->yystack[i].major]); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } fprintf(yyTraceFILE,"]\n"); } #endif