zdoom/qzdoom
0
0
Fork 0
mirror of https://github.com/ZDoom/qzdoom.git synced 2025-05-31 00:51:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

Directory restructuring to make it easier to version projects that don't build zdoom.exe.

Browse source 
SVN r4 (trunk)
This commit is contained in:
Randy Heit 2006-02-24 04:48:15 +00:00
commit cf11cbdb30
821 changed files with 361202 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

27
tools/dehsupp/Makefile.mgw Normal file
View file

@ -0,0 +1,27 @@
EXE = dehsupp.exe
CCDV = @../../ccdv
CC = gcc
CFLAGS = $(LOC) -Os -Wall -fomit-frame-pointer
BISON = bison
BISONFLAGS =
OBJS = parse.tab.o
all: $(EXE)
$(EXE): $(OBJS)
$(CCDV) $(CC) -o $(EXE) $(OBJS)
.PHONY: clean
clean:
-del /q /f $(EXE) 2>nul
-del /q /f *.o 2>nul
parse.tab.o: parse.tab.c
$(CCDV) $(CC) $(CFLAGS) -c -o $@ $<
parse.tab.c: parse.y
$(CCDV) $(BISON) $(BISONFLAGS) -o $@ $<

130
tools/dehsupp/dehsupp.vcproj Normal file
View file

@ -0,0 +1,130 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="7.10"
Name="dehsupp"
ProjectGUID="{AC64EE8F-F019-4A3E-BCAF-BD1FD072B9C8}"
Keyword="Win32Proj">
<Platforms>
<Platform
Name="Win32"/>
</Platforms>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="Debug"
IntermediateDirectory="Debug"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="5"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="4"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/dehsupp.exe"
LinkIncremental="2"
GenerateDebugInformation="TRUE"
ProgramDatabaseFile="$(OutDir)/dehsupp.pdb"
SubSystem="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="copy &quot;$(TargetPath)&quot; &quot;$(ProjectDir)\$(TargetFileName)&quot;"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="Release"
IntermediateDirectory="Release"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="4"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="3"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/dehsupp.exe"
LinkIncremental="1"
GenerateDebugInformation="TRUE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="copy &quot;$(TargetPath)&quot; &quot;$(ProjectDir)\$(TargetFileName)&quot;"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<File
RelativePath=".\parse.tab.c">
</File>
<File
RelativePath=".\parse.y">
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCustomBuildTool"
CommandLine="bison parse.y
"
Outputs="parse.tab.c"/>
</FileConfiguration>
</File>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

2182
tools/dehsupp/parse.tab.c Normal file
View file

File diff suppressed because it is too large Load diff

1029
tools/dehsupp/parse.y Normal file
View file

File diff suppressed because it is too large Load diff

23
tools/lemon/Makefile.mgw Normal file
View file

@ -0,0 +1,23 @@
EXE = lemon.exe
CCDV = @../../ccdv
CC = gcc
CFLAGS = $(LOC) -D_WIN32 -Os -Wall -Wno-implicit -fomit-frame-pointer
OBJS = lemon.o
all: $(EXE)
.c.o:
$(CCDV) $(CC) $(CFLAGS) -c -o $@ $<
$(EXE): $(OBJS)
$(CCDV) $(CC) -o $(EXE) $(OBJS)
.PHONY: clean
clean:
-del /q /f $(EXE) 2>nul
-del /q /f *.o 2>nul
lemon.o: lemon.c lempar.c

4587
tools/lemon/lemon.c Normal file
View file

File diff suppressed because it is too large Load diff

861
tools/lemon/lemon.html Normal file
View file

@ -0,0 +1,861 @@
<html>
<head>
<title>The Lemon Parser Generator</title>
</head>
<body bgcolor=white>
<h1 align=center>The Lemon Parser Generator</h1>
<p>Lemon is an LALR(1) parser generator for C or C++.
It does the same job as ``bison'' and ``yacc''.
But lemon is not another bison or yacc clone. It
uses a different grammar syntax which is designed to
reduce the number of coding errors. Lemon also uses a more
sophisticated parsing engine that is faster than yacc and
bison and which is both reentrant and thread-safe.
Furthermore, Lemon implements features that can be used
to eliminate resource leaks, making is suitable for use
in long-running programs such as graphical user interfaces
or embedded controllers.</p>
<p>This document is an introduction to the Lemon
parser generator.</p>
<h2>Theory of Operation</h2>
<p>The main goal of Lemon is to translate a context free grammar (CFG)
for a particular language into C code that implements a parser for
that language.
The program has two inputs:
<ul>
<li>The grammar specification.
<li>A parser template file.
</ul>
Typically, only the grammar specification is supplied by the programmer.
Lemon comes with a default parser template which works fine for most
applications. But the user is free to substitute a different parser
template if desired.</p>
<p>Depending on command-line options, Lemon will generate between
one and three files of outputs.
<ul>
<li>C code to implement the parser.
<li>A header file defining an integer ID for each terminal symbol.
<li>An information file that describes the states of the generated parser
automaton.
</ul>
By default, all three of these output files are generated.
The header file is suppressed if the ``-m'' command-line option is
used and the report file is omitted when ``-q'' is selected.</p>
<p>The grammar specification file uses a ``.y'' suffix, by convention.
In the examples used in this document, we'll assume the name of the
grammar file is ``gram.y''. A typical use of Lemon would be the
following command:
<pre>
lemon gram.y
</pre>
This command will generate three output files named ``gram.c'',
``gram.h'' and ``gram.out''.
The first is C code to implement the parser. The second
is the header file that defines numerical values for all
terminal symbols, and the last is the report that explains
the states used by the parser automaton.</p>
<h3>Command Line Options</h3>
<p>The behavior of Lemon can be modified using command-line options.
You can obtain a list of the available command-line options together
with a brief explanation of what each does by typing
<pre>
lemon -?
</pre>
As of this writing, the following command-line options are supported:
<ul>
<li><tt>-b</tt>
<li><tt>-c</tt>
<li><tt>-g</tt>
<li><tt>-m</tt>
<li><tt>-q</tt>
<li><tt>-s</tt>
<li><tt>-x</tt>
</ul>
The ``-b'' option reduces the amount of text in the report file by
printing only the basis of each parser state, rather than the full
configuration.
The ``-c'' option suppresses action table compression. Using -c
will make the parser a little larger and slower but it will detect
syntax errors sooner.
The ``-g'' option causes no output files to be generated at all.
Instead, the input grammar file is printed on standard output but
with all comments, actions and other extraneous text deleted. This
is a useful way to get a quick summary of a grammar.
The ``-m'' option causes the output C source file to be compatible
with the ``makeheaders'' program.
Makeheaders is a program that automatically generates header files
from C source code. When the ``-m'' option is used, the header
file is not output since the makeheaders program will take care
of generated all header files automatically.
The ``-q'' option suppresses the report file.
Using ``-s'' causes a brief summary of parser statistics to be
printed. Like this:
<pre>
Parser statistics: 74 terminals, 70 nonterminals, 179 rules
340 states, 2026 parser table entries, 0 conflicts
</pre>
Finally, the ``-x'' option causes Lemon to print its version number
and copyright information
and then stop without attempting to read the grammar or generate a parser.</p>
<h3>The Parser Interface</h3>
<p>Lemon doesn't generate a complete, working program. It only generates
a few subroutines that implement a parser. This section describes
the interface to those subroutines. It is up to the programmer to
call these subroutines in an appropriate way in order to produce a
complete system.</p>
<p>Before a program begins using a Lemon-generated parser, the program
must first create the parser.
A new parser is created as follows:
<pre>
void *pParser = ParseAlloc( malloc );
</pre>
The ParseAlloc() routine allocates and initializes a new parser and
returns a pointer to it.
The actual data structure used to represent a parser is opaque --
its internal structure is not visible or usable by the calling routine.
For this reason, the ParseAlloc() routine returns a pointer to void
rather than a pointer to some particular structure.
The sole argument to the ParseAlloc() routine is a pointer to the
subroutine used to allocate memory. Typically this means ``malloc()''.</p>
<p>After a program is finished using a parser, it can reclaim all
memory allocated by that parser by calling
<pre>
ParseFree(pParser, free);
</pre>
The first argument is the same pointer returned by ParseAlloc(). The
second argument is a pointer to the function used to release bulk
memory back to the system.</p>
<p>After a parser has been allocated using ParseAlloc(), the programmer
must supply the parser with a sequence of tokens (terminal symbols) to
be parsed. This is accomplished by calling the following function
once for each token:
<pre>
Parse(pParser, hTokenID, sTokenData, pArg);
</pre>
The first argument to the Parse() routine is the pointer returned by
ParseAlloc().
The second argument is a small positive integer that tells the parse the
type of the next token in the data stream.
There is one token type for each terminal symbol in the grammar.
The gram.h file generated by Lemon contains #define statements that
map symbolic terminal symbol names into appropriate integer values.
(A value of 0 for the second argument is a special flag to the
parser to indicate that the end of input has been reached.)
The third argument is the value of the given token. By default,
the type of the third argument is integer, but the grammar will
usually redefine this type to be some kind of structure.
Typically the second argument will be a broad category of tokens
such as ``identifier'' or ``number'' and the third argument will
be the name of the identifier or the value of the number.</p>
<p>The Parse() function may have either three or four arguments,
depending on the grammar. If the grammar specification file request
it, the Parse() function will have a fourth parameter that can be
of any type chosen by the programmer. The parser doesn't do anything
with this argument except to pass it through to action routines.
This is a convenient mechanism for passing state information down
to the action routines without having to use global variables.</p>
<p>A typical use of a Lemon parser might look something like the
following:
<pre>
01 ParseTree *ParseFile(const char *zFilename){
02 Tokenizer *pTokenizer;
03 void *pParser;
04 Token sToken;
05 int hTokenId;
06 ParserState sState;
07
08 pTokenizer = TokenizerCreate(zFilename);
09 pParser = ParseAlloc( malloc );
10 InitParserState(&sState);
11 while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
12 Parse(pParser, hTokenId, sToken, &sState);
13 }
14 Parse(pParser, 0, sToken, &sState);
15 ParseFree(pParser, free );
16 TokenizerFree(pTokenizer);
17 return sState.treeRoot;
18 }
</pre>
This example shows a user-written routine that parses a file of
text and returns a pointer to the parse tree.
(We've omitted all error-handling from this example to keep it
simple.)
We assume the existence of some kind of tokenizer which is created
using TokenizerCreate() on line 8 and deleted by TokenizerFree()
on line 16. The GetNextToken() function on line 11 retrieves the
next token from the input file and puts its type in the
integer variable hTokenId. The sToken variable is assumed to be
some kind of structure that contains details about each token,
such as its complete text, what line it occurs on, etc. </p>
<p>This example also assumes the existence of structure of type
ParserState that holds state information about a particular parse.
An instance of such a structure is created on line 6 and initialized
on line 10. A pointer to this structure is passed into the Parse()
routine as the optional 4th argument.
The action routine specified by the grammar for the parser can use
the ParserState structure to hold whatever information is useful and
appropriate. In the example, we note that the treeRoot field of
the ParserState structure is left pointing to the root of the parse
tree.</p>
<p>The core of this example as it relates to Lemon is as follows:
<pre>
ParseFile(){
pParser = ParseAlloc( malloc );
while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
Parse(pParser, hTokenId, sToken);
}
Parse(pParser, 0, sToken);
ParseFree(pParser, free );
}
</pre>
Basically, what a program has to do to use a Lemon-generated parser
is first create the parser, then send it lots of tokens obtained by
tokenizing an input source. When the end of input is reached, the
Parse() routine should be called one last time with a token type
of 0. This step is necessary to inform the parser that the end of
input has been reached. Finally, we reclaim memory used by the
parser by calling ParseFree().</p>
<p>There is one other interface routine that should be mentioned
before we move on.
The ParseTrace() function can be used to generate debugging output
from the parser. A prototype for this routine is as follows:
<pre>
ParseTrace(FILE *stream, char *zPrefix);
</pre>
After this routine is called, a short (one-line) message is written
to the designated output stream every time the parser changes states
or calls an action routine. Each such message is prefaced using
the text given by zPrefix. This debugging output can be turned off
by calling ParseTrace() again with a first argument of NULL (0).</p>
<h3>Differences With YACC and BISON</h3>
<p>Programmers who have previously used the yacc or bison parser
generator will notice several important differences between yacc and/or
bison and Lemon.
<ul>
<li>In yacc and bison, the parser calls the tokenizer. In Lemon,
the tokenizer calls the parser.
<li>Lemon uses no global variables. Yacc and bison use global variables
to pass information between the tokenizer and parser.
<li>Lemon allows multiple parsers to be running simultaneously. Yacc
and bison do not.
</ul>
These differences may cause some initial confusion for programmers
with prior yacc and bison experience.
But after years of experience using Lemon, I firmly
believe that the Lemon way of doing things is better.</p>
<h2>Input File Syntax</h2>
<p>The main purpose of the grammar specification file for Lemon is
to define the grammar for the parser. But the input file also
specifies additional information Lemon requires to do its job.
Most of the work in using Lemon is in writing an appropriate
grammar file.</p>
<p>The grammar file for lemon is, for the most part, free format.
It does not have sections or divisions like yacc or bison. Any
declaration can occur at any point in the file.
Lemon ignores whitespace (except where it is needed to separate
tokens) and it honors the same commenting conventions as C and C++.</p>
<h3>Terminals and Nonterminals</h3>
<p>A terminal symbol (token) is any string of alphanumeric
and underscore characters
that begins with an upper case letter.
A terminal can contain lower class letters after the first character,
but the usual convention is to make terminals all upper case.
A nonterminal, on the other hand, is any string of alphanumeric
and underscore characters than begins with a lower case letter.
Again, the usual convention is to make nonterminals use all lower
case letters.</p>
<p>In Lemon, terminal and nonterminal symbols do not need to
be declared or identified in a separate section of the grammar file.
Lemon is able to generate a list of all terminals and nonterminals
by examining the grammar rules, and it can always distinguish a
terminal from a nonterminal by checking the case of the first
character of the name.</p>
<p>Yacc and bison allow terminal symbols to have either alphanumeric
names or to be individual characters included in single quotes, like
this: ')' or '$'. Lemon does not allow this alternative form for
terminal symbols. With Lemon, all symbols, terminals and nonterminals,
must have alphanumeric names.</p>
<h3>Grammar Rules</h3>
<p>The main component of a Lemon grammar file is a sequence of grammar
rules.
Each grammar rule consists of a nonterminal symbol followed by
the special symbol ``::='' and then a list of terminals and/or nonterminals.
The rule is terminated by a period.
The list of terminals and nonterminals on the right-hand side of the
rule can be empty.
Rules can occur in any order, except that the left-hand side of the
first rule is assumed to be the start symbol for the grammar (unless
specified otherwise using the <tt>%start</tt> directive described below.)
A typical sequence of grammar rules might look something like this:
<pre>
expr ::= expr PLUS expr.
expr ::= expr TIMES expr.
expr ::= LPAREN expr RPAREN.
expr ::= VALUE.
</pre>
</p>
<p>There is one non-terminal in this example, ``expr'', and five
terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
``RPAREN'' and ``VALUE''.</p>
<p>Like yacc and bison, Lemon allows the grammar to specify a block
of C code that will be executed whenever a grammar rule is reduced
by the parser.
In Lemon, this action is specified by putting the C code (contained
within curly braces <tt>{...}</tt>) immediately after the
period that closes the rule.
For example:
<pre>
expr ::= expr PLUS expr. { printf("Doing an addition...\n"); }
</pre>
</p>
<p>In order to be useful, grammar actions must normally be linked to
their associated grammar rules.
In yacc and bison, this is accomplished by embedding a ``$$'' in the
action to stand for the value of the left-hand side of the rule and
symbols ``$1'', ``$2'', and so forth to stand for the value of
the terminal or nonterminal at position 1, 2 and so forth on the
right-hand side of the rule.
This idea is very powerful, but it is also very error-prone. The
single most common source of errors in a yacc or bison grammar is
to miscount the number of symbols on the right-hand side of a grammar
rule and say ``$7'' when you really mean ``$8''.</p>
<p>Lemon avoids the need to count grammar symbols by assigning symbolic
names to each symbol in a grammar rule and then using those symbolic
names in the action.
In yacc or bison, one would write this:
<pre>
expr -> expr PLUS expr { $$ = $1 + $3; };
</pre>
But in Lemon, the same rule becomes the following:
<pre>
expr(A) ::= expr(B) PLUS expr(C). { A = B+C; }
</pre>
In the Lemon rule, any symbol in parentheses after a grammar rule
symbol becomes a place holder for that symbol in the grammar rule.
This place holder can then be used in the associated C action to
stand for the value of that symbol.<p>
<p>The Lemon notation for linking a grammar rule with its reduce
action is superior to yacc/bison on several counts.
First, as mentioned above, the Lemon method avoids the need to
count grammar symbols.
Secondly, if a terminal or nonterminal in a Lemon grammar rule
includes a linking symbol in parentheses but that linking symbol
is not actually used in the reduce action, then an error message
is generated.
For example, the rule
<pre>
expr(A) ::= expr(B) PLUS expr(C). { A = B; }
</pre>
will generate an error because the linking symbol ``C'' is used
in the grammar rule but not in the reduce action.</p>
<p>The Lemon notation for linking grammar rules to reduce actions
also facilitates the use of destructors for reclaiming memory
allocated by the values of terminals and nonterminals on the
right-hand side of a rule.</p>
<h3>Precedence Rules</h3>
<p>Lemon resolves parsing ambiguities in exactly the same way as
yacc and bison. A shift-reduce conflict is resolved in favor
of the shift, and a reduce-reduce conflict is resolved by reducing
whichever rule comes first in the grammar file.</p>
<p>Just like in
yacc and bison, Lemon allows a measure of control
over the resolution of paring conflicts using precedence rules.
A precedence value can be assigned to any terminal symbol
using the %left, %right or %nonassoc directives. Terminal symbols
mentioned in earlier directives have a lower precedence that
terminal symbols mentioned in later directives. For example:</p>
<p><pre>
%left AND.
%left OR.
%nonassoc EQ NE GT GE LT LE.
%left PLUS MINUS.
%left TIMES DIVIDE MOD.
%right EXP NOT.
</pre></p>
<p>In the preceding sequence of directives, the AND operator is
defined to have the lowest precedence. The OR operator is one
precedence level higher. And so forth. Hence, the grammar would
attempt to group the ambiguous expression
<pre>
a AND b OR c
</pre>
like this
<pre>
a AND (b OR c).
</pre>
The associativity (left, right or nonassoc) is used to determine
the grouping when the precedence is the same. AND is left-associative
in our example, so
<pre>
a AND b AND c
</pre>
is parsed like this
<pre>
(a AND b) AND c.
</pre>
The EXP operator is right-associative, though, so
<pre>
a EXP b EXP c
</pre>
is parsed like this
<pre>
a EXP (b EXP c).
</pre>
The nonassoc precedence is used for non-associative operators.
So
<pre>
a EQ b EQ c
</pre>
is an error.</p>
<p>The precedence of non-terminals is transferred to rules as follows:
The precedence of a grammar rule is equal to the precedence of the
left-most terminal symbol in the rule for which a precedence is
defined. This is normally what you want, but in those cases where
you want to precedence of a grammar rule to be something different,
you can specify an alternative precedence symbol by putting the
symbol in square braces after the period at the end of the rule and
before any C-code. For example:</p>
<p><pre>
expr = MINUS expr. [NOT]
</pre></p>
<p>This rule has a precedence equal to that of the NOT symbol, not the
MINUS symbol as would have been the case by default.</p>
<p>With the knowledge of how precedence is assigned to terminal
symbols and individual
grammar rules, we can now explain precisely how parsing conflicts
are resolved in Lemon. Shift-reduce conflicts are resolved
as follows:
<ul>
<li> If either the token to be shifted or the rule to be reduced
lacks precedence information, then resolve in favor of the
shift, but report a parsing conflict.
<li> If the precedence of the token to be shifted is greater than
the precedence of the rule to reduce, then resolve in favor
of the shift. No parsing conflict is reported.
<li> If the precedence of the token it be shifted is less than the
precedence of the rule to reduce, then resolve in favor of the
reduce action. No parsing conflict is reported.
<li> If the precedences are the same and the shift token is
right-associative, then resolve in favor of the shift.
No parsing conflict is reported.
<li> If the precedences are the same the the shift token is
left-associative, then resolve in favor of the reduce.
No parsing conflict is reported.
<li> Otherwise, resolve the conflict by doing the shift and
report the parsing conflict.
</ul>
Reduce-reduce conflicts are resolved this way:
<ul>
<li> If either reduce rule
lacks precedence information, then resolve in favor of the
rule that appears first in the grammar and report a parsing
conflict.
<li> If both rules have precedence and the precedence is different
then resolve the dispute in favor of the rule with the highest
precedence and do not report a conflict.
<li> Otherwise, resolve the conflict by reducing by the rule that
appears first in the grammar and report a parsing conflict.
</ul>
<h3>Special Directives</h3>
<p>The input grammar to Lemon consists of grammar rules and special
directives. We've described all the grammar rules, so now we'll
talk about the special directives.</p>
<p>Directives in lemon can occur in any order. You can put them before
the grammar rules, or after the grammar rules, or in the mist of the
grammar rules. It doesn't matter. The relative order of
directives used to assign precedence to terminals is important, but
other than that, the order of directives in Lemon is arbitrary.</p>
<p>Lemon supports the following special directives:
<ul>
<li><tt>%destructor</tt>
<li><tt>%extra_argument</tt>
<li><tt>%include</tt>
<li><tt>%left</tt>
<li><tt>%name</tt>
<li><tt>%nonassoc</tt>
<li><tt>%parse_accept</tt>
<li><tt>%parse_failure </tt>
<li><tt>%right</tt>
<li><tt>%stack_overflow</tt>
<li><tt>%stack_size</tt>
<li><tt>%start_symbol</tt>
<li><tt>%syntax_error</tt>
<li><tt>%token_destructor</tt>
<li><tt>%token_prefix</tt>
<li><tt>%token_type</tt>
<li><tt>%type</tt>
</ul>
Each of these directives will be described separately in the
following sections:</p>
<h4>The <tt>%destructor</tt> directive</h4>
<p>The %destructor directive is used to specify a destructor for
a non-terminal symbol.
(See also the %token_destructor directive which is used to
specify a destructor for terminal symbols.)</p>
<p>A non-terminal's destructor is called to dispose of the
non-terminal's value whenever the non-terminal is popped from
the stack. This includes all of the following circumstances:
<ul>
<li> When a rule reduces and the value of a non-terminal on
the right-hand side is not linked to C code.
<li> When the stack is popped during error processing.
<li> When the ParseFree() function runs.
</ul>
The destructor can do whatever it wants with the value of
the non-terminal, but its design is to deallocate memory
or other resources held by that non-terminal.</p>
<p>Consider an example:
<pre>
%type nt {void*}
%destructor nt { free($$); }
nt(A) ::= ID NUM. { A = malloc( 100 ); }
</pre>
This example is a bit contrived but it serves to illustrate how
destructors work. The example shows a non-terminal named
``nt'' that holds values of type ``void*''. When the rule for
an ``nt'' reduces, it sets the value of the non-terminal to
space obtained from malloc(). Later, when the nt non-terminal
is popped from the stack, the destructor will fire and call
free() on this malloced space, thus avoiding a memory leak.
(Note that the symbol ``$$'' in the destructor code is replaced
by the value of the non-terminal.)</p>
<p>It is important to note that the value of a non-terminal is passed
to the destructor whenever the non-terminal is removed from the
stack, unless the non-terminal is used in a C-code action. If
the non-terminal is used by C-code, then it is assumed that the
C-code will take care of destroying it if it should really
be destroyed. More commonly, the value is used to build some
larger structure and we don't want to destroy it, which is why
the destructor is not called in this circumstance.</p>
<p>By appropriate use of destructors, it is possible to
build a parser using Lemon that can be used within a long-running
program, such as a GUI, that will not leak memory or other resources.
To do the same using yacc or bison is much more difficult.</p>
<h4>The <tt>%extra_argument</tt> directive</h4>
The %extra_argument directive instructs Lemon to add a 4th parameter
to the parameter list of the Parse() function it generates. Lemon
doesn't do anything itself with this extra argument, but it does
make the argument available to C-code action routines, destructors,
and so forth. For example, if the grammar file contains:</p>
<p><pre>
%extra_argument { MyStruct *pAbc }
</pre></p>
<p>Then the Parse() function generated will have an 4th parameter
of type ``MyStruct*'' and all action routines will have access to
a variable named ``pAbc'' that is the value of the 4th parameter
in the most recent call to Parse().</p>
<h4>The <tt>%include</tt> directive</h4>
<p>The %include directive specifies C code that is included at the
top of the generated parser. You can include any text you want --
the Lemon parser generator copies to blindly. If you have multiple
%include directives in your grammar file, their values are concatenated
before being put at the beginning of the generated parser.</p>
<p>The %include directive is very handy for getting some extra #include
preprocessor statements at the beginning of the generated parser.
For example:</p>
<p><pre>
%include {#include &lt;unistd.h&gt;}
</pre></p>
<p>This might be needed, for example, if some of the C actions in the
grammar call functions that are prototyed in unistd.h.</p>
<h4>The <tt>%left</tt> directive</h4>
The %left directive is used (along with the %right and
%nonassoc directives) to declare precedences of terminal
symbols. Every terminal symbol whose name appears after
a %left directive but before the next period (``.'') is
given the same left-associative precedence value. Subsequent
%left directives have higher precedence. For example:</p>
<p><pre>
%left AND.
%left OR.
%nonassoc EQ NE GT GE LT LE.
%left PLUS MINUS.
%left TIMES DIVIDE MOD.
%right EXP NOT.
</pre></p>
<p>Note the period that terminates each %left, %right or %nonassoc
directive.</p>
<p>LALR(1) grammars can get into a situation where they require
a large amount of stack space if you make heavy use or right-associative
operators. For this reason, it is recommended that you use %left
rather than %right whenever possible.</p>
<h4>The <tt>%name</tt> directive</h4>
<p>By default, the functions generated by Lemon all begin with the
five-character string ``Parse''. You can change this string to something
different using the %name directive. For instance:</p>
<p><pre>
%name Abcde
</pre></p>
<p>Putting this directive in the grammar file will cause Lemon to generate
functions named
<ul>
<li> AbcdeAlloc(),
<li> AbcdeFree(),
<li> AbcdeTrace(), and
<li> Abcde().
</ul>
The %name directive allows you to generator two or more different
parsers and link them all into the same executable.
</p>
<h4>The <tt>%nonassoc</tt> directive</h4>
<p>This directive is used to assign non-associative precedence to
one or more terminal symbols. See the section on precedence rules
or on the %left directive for additional information.</p>
<h4>The <tt>%parse_accept</tt> directive</h4>
<p>The %parse_accept directive specifies a block of C code that is
executed whenever the parser accepts its input string. To ``accept''
an input string means that the parser was able to process all tokens
without error.</p>
<p>For example:</p>
<p><pre>
%parse_accept {
printf("parsing complete!\n");
}
</pre></p>
<h4>The <tt>%parse_failure</tt> directive</h4>
<p>The %parse_failure directive specifies a block of C code that
is executed whenever the parser fails complete. This code is not
executed until the parser has tried and failed to resolve an input
error using is usual error recovery strategy. The routine is
only invoked when parsing is unable to continue.</p>
<p><pre>
%parse_failure {
fprintf(stderr,"Giving up. Parser is hopelessly lost...\n");
}
</pre></p>
<h4>The <tt>%right</tt> directive</h4>
<p>This directive is used to assign right-associative precedence to
one or more terminal symbols. See the section on precedence rules
or on the %left directive for additional information.</p>
<h4>The <tt>%stack_overflow</tt> directive</h4>
<p>The %stack_overflow directive specifies a block of C code that
is executed if the parser's internal stack ever overflows. Typically
this just prints an error message. After a stack overflow, the parser
will be unable to continue and must be reset.</p>
<p><pre>
%stack_overflow {
fprintf(stderr,"Giving up. Parser stack overflow\n");
}
</pre></p>
<p>You can help prevent parser stack overflows by avoiding the use
of right recursion and right-precedence operators in your grammar.
Use left recursion and and left-precedence operators instead, to
encourage rules to reduce sooner and keep the stack size down.
For example, do rules like this:
<pre>
list ::= list element. // left-recursion. Good!
list ::= .
</pre>
Not like this:
<pre>
list ::= element list. // right-recursion. Bad!
list ::= .
</pre>
<h4>The <tt>%stack_size</tt> directive</h4>
<p>If stack overflow is a problem and you can't resolve the trouble
by using left-recursion, then you might want to increase the size
of the parser's stack using this directive. Put an positive integer
after the %stack_size directive and Lemon will generate a parse
with a stack of the requested size. The default value is 100.</p>
<p><pre>
%stack_size 2000
</pre></p>
<h4>The <tt>%start_symbol</tt> directive</h4>
<p>By default, the start-symbol for the grammar that Lemon generates
is the first non-terminal that appears in the grammar file. But you
can choose a different start-symbol using the %start_symbol directive.</p>
<p><pre>
%start_symbol prog
</pre></p>
<h4>The <tt>%token_destructor</tt> directive</h4>
<p>The %destructor directive assigns a destructor to a non-terminal
symbol. (See the description of the %destructor directive above.)
This directive does the same thing for all terminal symbols.</p>
<p>Unlike non-terminal symbols which may each have a different data type
for their values, terminals all use the same data type (defined by
the %token_type directive) and so they use a common destructor. Other
than that, the token destructor works just like the non-terminal
destructors.</p>
<h4>The <tt>%token_prefix</tt> directive</h4>
<p>Lemon generates #defines that assign small integer constants
to each terminal symbol in the grammar. If desired, Lemon will
add a prefix specified by this directive
to each of the #defines it generates.
So if the default output of Lemon looked like this:
<pre>
#define AND 1
#define MINUS 2
#define OR 3
#define PLUS 4
</pre>
You can insert a statement into the grammar like this:
<pre>
%token_prefix TOKEN_
</pre>
to cause Lemon to produce these symbols instead:
<pre>
#define TOKEN_AND 1
#define TOKEN_MINUS 2
#define TOKEN_OR 3
#define TOKEN_PLUS 4
</pre>
<h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
<p>These directives are used to specify the data types for values
on the parser's stack associated with terminal and non-terminal
symbols. The values of all terminal symbols must be of the same
type. This turns out to be the same data type as the 3rd parameter
to the Parse() function generated by Lemon. Typically, you will
make the value of a terminal symbol by a pointer to some kind of
token structure. Like this:</p>
<p><pre>
%token_type {Token*}
</pre></p>
<p>If the data type of terminals is not specified, the default value
is ``int''.</p>
<p>Non-terminal symbols can each have their own data types. Typically
the data type of a non-terminal is a pointer to the root of a parse-tree
structure that contains all information about that non-terminal.
For example:</p>
<p><pre>
%type expr {Expr*}
</pre></p>
<p>Each entry on the parser's stack is actually a union containing
instances of all data types for every non-terminal and terminal symbol.
Lemon will automatically use the correct element of this union depending
on what the corresponding non-terminal or terminal symbol is. But
the grammar designer should keep in mind that the size of the union
will be the size of its largest element. So if you have a single
non-terminal whose data type requires 1K of storage, then your 100
entry parser stack will require 100K of heap space. If you are willing
and able to pay that price, fine. You just need to know.</p>
<h3>Error Processing</h3>
<p>After extensive experimentation over several years, it has been
discovered that the error recovery strategy used by yacc is about
as good as it gets. And so that is what Lemon uses.</p>
<p>When a Lemon-generated parser encounters a syntax error, it
first invokes the code specified by the %syntax_error directive, if
any. It then enters its error recovery strategy. The error recovery
strategy is to begin popping the parsers stack until it enters a
state where it is permitted to shift a special non-terminal symbol
named ``error''. It then shifts this non-terminal and continues
parsing. But the %syntax_error routine will not be called again
until at least three new tokens have been successfully shifted.</p>
<p>If the parser pops its stack until the stack is empty, and it still
is unable to shift the error symbol, then the %parse_failed routine
is invoked and the parser resets itself to its start state, ready
to begin parsing a new file. This is what will happen at the very
first syntax error, of course, if there are no instances of the
``error'' non-terminal in your grammar.</p>
</body>
</html>

168
tools/lemon/lemon.vcproj Normal file
View file

@ -0,0 +1,168 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="7.10"
Name="lemon"
ProjectGUID="{0F80ACBF-460E-44F0-B28E-B3272D1774A7}"
Keyword="Win32Proj">
<Platforms>
<Platform
Name="Win32"/>
</Platforms>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="."
IntermediateDirectory="Build"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="1"
OptimizeForProcessor="3"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
StringPooling="TRUE"
ExceptionHandling="FALSE"
RuntimeLibrary="2"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="0"
CallingConvention="0"
CompileAs="1"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/lemon.exe"
LinkIncremental="1"
GenerateDebugInformation="FALSE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
OptimizeForWindows98="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="."
IntermediateDirectory="Build"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="1"
OptimizeForProcessor="3"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
StringPooling="TRUE"
ExceptionHandling="FALSE"
RuntimeLibrary="2"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="0"
CallingConvention="0"
CompileAs="1"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/lemon.exe"
LinkIncremental="1"
GenerateDebugInformation="FALSE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
OptimizeForWindows98="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
<File
RelativePath=".\lemon.c">
<FileConfiguration
Name="Release|Win32">
<Tool
Name="VCCLCompilerTool"
Detect64BitPortabilityProblems="FALSE"/>
</FileConfiguration>
</File>
<File
RelativePath=".\lempar.c">
<FileConfiguration
Name="Debug|Win32"
ExcludedFromBuild="TRUE">
<Tool
Name="VCCLCompilerTool"/>
</FileConfiguration>
<FileConfiguration
Name="Release|Win32"
ExcludedFromBuild="TRUE">
<Tool
Name="VCCLCompilerTool"/>
</FileConfiguration>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
</Filter>
<Filter
Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
</Filter>
<File
RelativePath=".\lemon.html">
</File>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

687
tools/lemon/lempar.c Normal file
View file

@ -0,0 +1,687 @@
/* Driver template for the LEMON parser generator.
** The author disclaims copyright to this source code.
*/
/* First off, code is include which follows the "include" declaration
** in the input file. */
#include <stdio.h>
%%
/* Next is all token values, in a form suitable for use by makeheaders.
** This section will be null unless lemon is run with the -m switch.
*/
/*
** These constants (all generated automatically by the parser generator)
** specify the various kinds of tokens (terminals) that the parser
** understands.
**
** Each symbol here is a terminal symbol in the grammar.
*/
%%
/* Make sure the INTERFACE macro is defined.
*/
#ifndef INTERFACE
# define INTERFACE 1
#endif
/* The next thing included is series of defines which control
** various aspects of the generated parser.
** YYCODETYPE is the data type used for storing terminal
** and nonterminal numbers. "unsigned char" is
** used if there are fewer than 250 terminals
** and nonterminals. "int" is used otherwise.
** YYNOCODE is a number of type YYCODETYPE which corresponds
** to no legal terminal or nonterminal number. This
** number is used to fill in empty slots of the hash
** table.
** YYFALLBACK If defined, this indicates that one or more tokens
** have fall-back values which should be used if the
** original value of the token will not parse.
** YYACTIONTYPE is the data type used for storing terminal
** and nonterminal numbers. "unsigned char" is
** used if there are fewer than 250 rules and
** states combined. "int" is used otherwise.
** ParseTOKENTYPE is the data type used for minor tokens given
** directly to the parser from the tokenizer.
** YYMINORTYPE is the data type used for all minor tokens.
** This is typically a union of many types, one of
** which is ParseTOKENTYPE. The entry in the union
** for base tokens is called "yy0".
** YYSTACKDEPTH is the maximum depth of the parser's stack.
** ParseARG_SDECL A static variable declaration for the %extra_argument
** ParseARG_PDECL A parameter declaration for the %extra_argument
** ParseARG_STORE Code to store %extra_argument into yypParser
** ParseARG_FETCH Code to extract %extra_argument from yypParser
** YYNSTATE the combined number of states.
** YYNRULE the number of rules in the grammar
** YYERRORSYMBOL is the code number of the error symbol. If not
** defined, then do no error processing.
*/
%%
#define YY_NO_ACTION (YYNSTATE+YYNRULE+2)
#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1)
#define YY_ERROR_ACTION (YYNSTATE+YYNRULE)
/* Next are that tables used to determine what action to take based on the
** current state and lookahead token. These tables are used to implement
** functions that take a state number and lookahead value and return an
** action integer.
**
** Suppose the action integer is N. Then the action is determined as
** follows
**
** 0 <= N < YYNSTATE Shift N. That is, push the lookahead
** token onto the stack and goto state N.
**
** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE.
**
** N == YYNSTATE+YYNRULE A syntax error has occurred.
**
** N == YYNSTATE+YYNRULE+1 The parser accepts its input.
**
** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused
** slots in the yy_action[] table.
**
** The action table is constructed as a single large table named yy_action[].
** Given state S and lookahead X, the action is computed as
**
** yy_action[ yy_shift_ofst[S] + X ]
**
** If the index value yy_shift_ofst[S]+X is out of range or if the value
** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S]
** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table
** and that yy_default[S] should be used instead.
**
** The formula above is for computing the action when the lookahead is
** a terminal symbol. If the lookahead is a non-terminal (as occurs after
** a reduce action) then the yy_reduce_ofst[] array is used in place of
** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of
** YY_SHIFT_USE_DFLT.
**
** The following are the tables generated in this section:
**
** yy_action[] A single table containing all actions.
** yy_lookahead[] A table containing the lookahead for each entry in
** yy_action. Used to detect hash collisions.
** yy_shift_ofst[] For each state, the offset into yy_action for
** shifting terminals.
** yy_reduce_ofst[] For each state, the offset into yy_action for
** shifting non-terminals after a reduce.
** yy_default[] Default action for each state.
*/
%%
#define YY_SZ_ACTTAB (sizeof(yy_action)/sizeof(yy_action[0]))
/* The next table maps tokens into fallback tokens. If a construct
** like the following:
**
** %fallback ID X Y Z.
**
** appears in the grammer, then ID becomes a fallback token for X, Y,
** and Z. Whenever one of the tokens X, Y, or Z is input to the parser
** but it does not parse, the type of the token is changed to ID and
** the parse is retried before an error is thrown.
*/
#ifdef YYFALLBACK
static const YYCODETYPE yyFallback[] = {
%%
};
#endif /* YYFALLBACK */
/* The following structure represents a single element of the
** parser's stack. Information stored includes:
**
** + The state number for the parser at this level of the stack.
**
** + The value of the token stored at this level of the stack.
** (In other words, the "major" token.)
**
** + The semantic value stored at this level of the stack. This is
** the information used by the action routines in the grammar.
** It is sometimes called the "minor" token.
*/
struct yyStackEntry {
int stateno; /* The state-number */
int major; /* The major token value. This is the code
** number for the token at this stack level */
YYMINORTYPE minor; /* The user-supplied minor token value. This
** is the value of the token */
};
typedef struct yyStackEntry yyStackEntry;
/* The state of the parser is completely contained in an instance of
** the following structure */
struct yyParser {
int yyidx; /* Index of top element in stack */
int yyerrcnt; /* Shifts left before out of the error */
ParseARG_SDECL /* A place to hold %extra_argument */
yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */
};
typedef struct yyParser yyParser;
#ifndef NDEBUG
#include <stdio.h>
static FILE *yyTraceFILE = 0;
static char *yyTracePrompt = 0;
#endif /* NDEBUG */
#ifndef NDEBUG
/*
** Turn parser tracing on by giving a stream to which to write the trace
** and a prompt to preface each trace message. Tracing is turned off
** by making either argument NULL
**
** Inputs:
** <ul>
** <li> A FILE* to which trace output should be written.
** If NULL, then tracing is turned off.
** <li> A prefix string written at the beginning of every
** line of trace output. If NULL, then tracing is
** turned off.
** </ul>
**
** Outputs:
** None.
*/
void ParseTrace(FILE *TraceFILE, char *zTracePrompt){
yyTraceFILE = TraceFILE;
yyTracePrompt = zTracePrompt;
if( yyTraceFILE==0 ) yyTracePrompt = 0;
else if( yyTracePrompt==0 ) yyTraceFILE = 0;
}
#endif /* NDEBUG */
#ifndef NDEBUG
/* For tracing shifts, the names of all terminals and nonterminals
** are required. The following table supplies these names */
static const char *yyTokenName[] = {
%%
};
#endif /* NDEBUG */
#ifndef NDEBUG
/* For tracing reduce actions, the names of all rules are required.
*/
static const char *yyRuleName[] = {
%%
};
#endif /* NDEBUG */
/*
** This function returns the symbolic name associated with a token
** value.
*/
const char *ParseTokenName(int tokenType){
#ifndef NDEBUG
if( tokenType>0 && tokenType<(sizeof(yyTokenName)/sizeof(yyTokenName[0])) ){
return yyTokenName[tokenType];
}else{
return "Unknown";
}
#else
return "";
#endif
}
/*
** This function allocates a new parser.
** The only argument is a pointer to a function which works like
** malloc.
**
** Inputs:
** A pointer to the function used to allocate memory.
**
** Outputs:
** A pointer to a parser. This pointer is used in subsequent calls
** to Parse and ParseFree.
*/
void *ParseAlloc(void *(*mallocProc)(size_t)){
yyParser *pParser;
pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) );
if( pParser ){
pParser->yyidx = -1;
}
return pParser;
}
/* The following function deletes the value associated with a
** symbol. The symbol can be either a terminal or nonterminal.
** "yymajor" is the symbol code, and "yypminor" is a pointer to
** the value.
*/
static void yy_destructor(YYCODETYPE yymajor, YYMINORTYPE *yypminor){
switch( yymajor ){
/* Here is inserted the actions which take place when a
** terminal or non-terminal is destroyed. This can happen
** when the symbol is popped from the stack during a
** reduce or during error processing or when a parser is
** being destroyed before it is finished parsing.
**
** Note: during a reduce, the only symbols destroyed are those
** which appear on the RHS of the rule, but which are not used
** inside the C code.
*/
%%
default: break; /* If no destructor action specified: do nothing */
}
}
/*
** Pop the parser's stack once.
**
** If there is a destructor routine associated with the token which
** is popped from the stack, then call it.
**
** Return the major token number for the symbol popped.
*/
static int yy_pop_parser_stack(yyParser *pParser){
YYCODETYPE yymajor;
yyStackEntry *yytos = &pParser->yystack[pParser->yyidx];
if( pParser->yyidx<0 ) return 0;
#ifndef NDEBUG
if( yyTraceFILE && pParser->yyidx>=0 ){
fprintf(yyTraceFILE,"%sPopping %s\n",
yyTracePrompt,
yyTokenName[yytos->major]);
}
#endif
yymajor = yytos->major;
yy_destructor( yymajor, &yytos->minor);
pParser->yyidx--;
return yymajor;
}
/*
** Deallocate and destroy a parser. Destructors are all called for
** all stack elements before shutting the parser down.
**
** Inputs:
** <ul>
** <li> A pointer to the parser. This should be a pointer
** obtained from ParseAlloc.
** <li> A pointer to a function used to reclaim memory obtained
** from malloc.
** </ul>
*/
void ParseFree(
void *p, /* The parser to be deleted */
void (*freeProc)(void*) /* Function used to reclaim memory */
){
yyParser *pParser = (yyParser*)p;
if( pParser==0 ) return;
while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
(*freeProc)((void*)pParser);
}
/*
** Find the appropriate action for a parser given the terminal
** look-ahead token iLookAhead.
**
** If the look-ahead token is YYNOCODE, then check to see if the action is
** independent of the look-ahead. If it is, return the action, otherwise
** return YY_NO_ACTION.
*/
static int yy_find_shift_action(
yyParser *pParser, /* The parser */
int iLookAhead /* The look-ahead token */
){
int i;
int stateno = pParser->yystack[pParser->yyidx].stateno;
/* if( pParser->yyidx<0 ) return YY_NO_ACTION; */
i = yy_shift_ofst[stateno];
if( i==YY_SHIFT_USE_DFLT ){
return yy_default[stateno];
}
if( iLookAhead==YYNOCODE ){
return YY_NO_ACTION;
}
i += iLookAhead;
if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){
#ifdef YYFALLBACK
int iFallback; /* Fallback token */
if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
&& (iFallback = yyFallback[iLookAhead])!=0 ){
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
}
#endif
return yy_find_shift_action(pParser, iFallback);
}
#endif
return yy_default[stateno];
}else{
return yy_action[i];
}
}
/*
** Find the appropriate action for a parser given the non-terminal
** look-ahead token iLookAhead.
**
** If the look-ahead token is YYNOCODE, then check to see if the action is
** independent of the look-ahead. If it is, return the action, otherwise
** return YY_NO_ACTION.
*/
static int yy_find_reduce_action(
yyParser *pParser, /* The parser */
int iLookAhead /* The look-ahead token */
){
int i;
int stateno = pParser->yystack[pParser->yyidx].stateno;
i = yy_reduce_ofst[stateno];
if( i==YY_REDUCE_USE_DFLT ){
return yy_default[stateno];
}
if( iLookAhead==YYNOCODE ){
return YY_NO_ACTION;
}
i += iLookAhead;
if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){
return yy_default[stateno];
}else{
return yy_action[i];
}
}
/*
** Perform a shift action.
*/
static void yy_shift(
yyParser *yypParser, /* The parser to be shifted */
int yyNewState, /* The new state to shift in */
int yyMajor, /* The major token to shift in */
YYMINORTYPE *yypMinor /* Pointer ot the minor token to shift in */
){
yyStackEntry *yytos;
yypParser->yyidx++;
if( yypParser->yyidx>=YYSTACKDEPTH ){
ParseARG_FETCH;
yypParser->yyidx--;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
}
#endif
while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
/* Here code is inserted which will execute if the parser
** stack every overflows */
%%
ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
return;
}
yytos = &yypParser->yystack[yypParser->yyidx];
yytos->stateno = yyNewState;
yytos->major = yyMajor;
yytos->minor = *yypMinor;
#ifndef NDEBUG
if( yyTraceFILE && yypParser->yyidx>0 ){
int i;
fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
for(i=1; i<=yypParser->yyidx; i++)
fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
fprintf(yyTraceFILE,"\n");
}
#endif
}
/* The following table contains information about every rule that
** is used during the reduce.
*/
static struct {
YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */
unsigned char nrhs; /* Number of right-hand side symbols in the rule */
} yyRuleInfo[] = {
%%
};
static void yy_accept(yyParser*); /* Forward Declaration */
/*
** Perform a reduce action and the shift that must immediately
** follow the reduce.
*/
static void yy_reduce(
yyParser *yypParser, /* The parser */
int yyruleno /* Number of the rule by which to reduce */
){
int yygoto; /* The next state */
int yyact; /* The next action */
YYMINORTYPE yygotominor; /* The LHS of the rule reduced */
yyStackEntry *yymsp; /* The top of the parser's stack */
int yysize; /* Amount to pop the stack */
ParseARG_FETCH;
yymsp = &yypParser->yystack[yypParser->yyidx];
#ifndef NDEBUG
if( yyTraceFILE && yyruleno>=0
&& yyruleno<sizeof(yyRuleName)/sizeof(yyRuleName[0]) ){
fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt,
yyRuleName[yyruleno]);
}
#endif /* NDEBUG */
switch( yyruleno ){
/* Beginning here are the reduction cases. A typical example
** follows:
** case 0:
** #line <lineno> <grammarfile>
** { ... } // User supplied code
** #line <lineno> <thisfile>
** break;
*/
%%
};
yygoto = yyRuleInfo[yyruleno].lhs;
yysize = yyRuleInfo[yyruleno].nrhs;
yypParser->yyidx -= yysize;
yyact = yy_find_reduce_action(yypParser,yygoto);
if( yyact < YYNSTATE ){
yy_shift(yypParser,yyact,yygoto,&yygotominor);
}else if( yyact == YYNSTATE + YYNRULE + 1 ){
yy_accept(yypParser);
}
}
/*
** The following code executes when the parse fails
*/
static void yy_parse_failed(
yyParser *yypParser /* The parser */
){
ParseARG_FETCH;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
}
#endif
while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
/* Here code is inserted which will be executed whenever the
** parser fails */
%%
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
}
/*
** The following code executes when a syntax error first occurs.
*/
static void yy_syntax_error(
yyParser *yypParser, /* The parser */
int yymajor, /* The major type of the error token */
YYMINORTYPE yyminor /* The minor type of the error token */
){
ParseARG_FETCH;
#define TOKEN (yyminor.yy0)
%%
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
}
/*
** The following is executed when the parser accepts
*/
static void yy_accept(
yyParser *yypParser /* The parser */
){
ParseARG_FETCH;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
}
#endif
while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
/* Here code is inserted which will be executed whenever the
** parser accepts */
%%
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
}
/* The main parser program.
** The first argument is a pointer to a structure obtained from
** "ParseAlloc" which describes the current state of the parser.
** The second argument is the major token number. The third is
** the minor token. The fourth optional argument is whatever the
** user wants (and specified in the grammar) and is available for
** use by the action routines.
**
** Inputs:
** <ul>
** <li> A pointer to the parser (an opaque structure.)
** <li> The major token number.
** <li> The minor token number.
** <li> An option argument of a grammar-specified type.
** </ul>
**
** Outputs:
** None.
*/
void Parse(
void *yyp, /* The parser */
int yymajor, /* The major token code number */
ParseTOKENTYPE yyminor /* The value for the token */
ParseARG_PDECL /* Optional %extra_argument parameter */
){
YYMINORTYPE yyminorunion;
int yyact; /* The parser action. */
int yyendofinput; /* True if we are at the end of input */
int yyerrorhit = 0; /* True if yymajor has invoked an error */
yyParser *yypParser; /* The parser */
/* (re)initialize the parser, if necessary */
yypParser = (yyParser*)yyp;
if( yypParser->yyidx<0 ){
if( yymajor==0 ) return;
yypParser->yyidx = 0;
yypParser->yyerrcnt = -1;
yypParser->yystack[0].stateno = 0;
yypParser->yystack[0].major = 0;
}
yyminorunion.yy0 = yyminor;
yyendofinput = (yymajor==0);
ParseARG_STORE;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]);
}
#endif
do{
yyact = yy_find_shift_action(yypParser,yymajor);
if( yyact<YYNSTATE ){
yy_shift(yypParser,yyact,yymajor,&yyminorunion);
yypParser->yyerrcnt--;
if( yyendofinput && yypParser->yyidx>=0 ){
yymajor = 0;
}else{
yymajor = YYNOCODE;
}
}else if( yyact < YYNSTATE + YYNRULE ){
yy_reduce(yypParser,yyact-YYNSTATE);
}else if( yyact == YY_ERROR_ACTION ){
int yymx;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt);
}
#endif
#ifdef YYERRORSYMBOL
/* A syntax error has occurred.
** The response to an error depends upon whether or not the
** grammar defines an error token "ERROR".
**
** This is what we do if the grammar does define ERROR:
**
** * Call the %syntax_error function.
**
** * Begin popping the stack until we enter a state where
** it is legal to shift the error symbol, then shift
** the error symbol.
**
** * Set the error count to three.
**
** * Begin accepting and shifting new tokens. No new error
** processing will occur until three tokens have been
** shifted successfully.
**
*/
if( yypParser->yyerrcnt<0 ){
yy_syntax_error(yypParser,yymajor,yyminorunion);
}
yymx = yypParser->yystack[yypParser->yyidx].major;
if( yymx==YYERRORSYMBOL || yyerrorhit ){
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sDiscard input token %s\n",
yyTracePrompt,yyTokenName[yymajor]);
}
#endif
yy_destructor(yymajor,&yyminorunion);
yymajor = YYNOCODE;
}else{
while(
yypParser->yyidx >= 0 &&
yymx != YYERRORSYMBOL &&
(yyact = yy_find_shift_action(yypParser,YYERRORSYMBOL)) >= YYNSTATE
){
yy_pop_parser_stack(yypParser);
}
if( yypParser->yyidx < 0 || yymajor==0 ){
yy_destructor(yymajor,&yyminorunion);
yy_parse_failed(yypParser);
yymajor = YYNOCODE;
}else if( yymx!=YYERRORSYMBOL ){
YYMINORTYPE u2;
u2.YYERRSYMDT = 0;
yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2);
}
}
yypParser->yyerrcnt = 3;
yyerrorhit = 1;
#else /* YYERRORSYMBOL is not defined */
/* This is what we do if the grammar does not define ERROR:
**
** * Report an error message, and throw away the input token.
**
** * If the input token is $, then fail the parse.
**
** As before, subsequent error messages are suppressed until
** three input tokens have been successfully shifted.
*/
if( yypParser->yyerrcnt<=0 ){
yy_syntax_error(yypParser,yymajor,yyminorunion);
}
yypParser->yyerrcnt = 3;
yy_destructor(yymajor,&yyminorunion);
if( yyendofinput ){
yy_parse_failed(yypParser);
}
yymajor = YYNOCODE;
#endif
}else{
yy_accept(yypParser);
yymajor = YYNOCODE;
}
}while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
return;
}

23
tools/makewad/Makefile.mgw Normal file
View file

@ -0,0 +1,23 @@
EXE = makewad.exe
CCDV = @../../ccdv
CC = gcc
CFLAGS = $(LOC) -Os -Wall -fomit-frame-pointer
OBJS = makewad.o
all: $(EXE)
.c.o:
$(CCDV) $(CC) $(CFLAGS) -c -o $@ $<
$(EXE): $(OBJS)
$(CCDV) $(CC) -o $(EXE) $(OBJS)
.PHONY: clean
clean:
-del /q /f $(EXE) 2>nul
-del /q /f *.o 2>nul
makewad.o: makewad.c

274
tools/makewad/makewad.c Normal file
View file

@ -0,0 +1,274 @@
// makewad.c
//
// Reads a wad specification from stdin.
// Produces a wad file -or- writes a Makefile to stdout.
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#define MAX_LUMPS 4096
typedef struct
{
char magic[4];
unsigned int numlumps;
int infotableofs;
} wadinfo_t;
typedef struct
{
int filepos;
unsigned int size;
char name[8];
} filelump_t;
int appendlump (FILE *wadfile, char *filename)
{
char readbuf[64*1024];
FILE *lumpfile;
size_t readlen;
int ret = 0;
lumpfile = fopen (filename, "rb");
if (lumpfile == NULL)
{
fprintf (stderr, "Could not open %s: %s\n", filename, strerror(errno));
return 1;
}
while (lumpfile != NULL)
{
readlen = fread (readbuf, 1, sizeof(readbuf), lumpfile);
if (readlen < sizeof(readbuf))
{
if (ferror (lumpfile))
{
fprintf (stderr, "Error reading %s: %s\n", filename, strerror(errno));
ret |= 1;
}
fclose (lumpfile);
lumpfile = NULL;
}
if (fwrite (readbuf, 1, readlen, wadfile) < readlen)
{
fprintf (stderr, "Error writing to wad: %s\n", strerror(errno));
fclose (lumpfile);
lumpfile = NULL;
ret |= 1;
}
}
return ret;
}
int buildwad (FILE *listfile, char *listfilename, char *makecmd, char *makefile)
{
wadinfo_t header;
filelump_t directory[MAX_LUMPS];
char str[256];
FILE *wadfile = NULL;
char *pt;
char *lumpname, *filename;
int lineno = 0;
int ret = 0;
int i;
strncpy (header.magic, "PWAD", 4);
header.infotableofs = 0;
header.numlumps = 0;
memset (directory, 0, sizeof(directory));
while (fgets (str, sizeof(str), listfile))
{
lineno++;
// Strip comments
pt = strchr (str, '#');
if (pt) *pt = 0;
// Strip trailing whitespace
pt = str + strlen (str) - 1;
while (pt >= str && isspace(*pt)) pt--;
if (pt < str) continue;
pt[1] = 0;
// Skip leading whitespace
pt = str;
while (isspace(*pt)) pt++;
if (*pt == '@')
{ // Rest of line is wadfile to create
if (wadfile != NULL)
{
fprintf (stderr, "Line %d: Tried to reopen wadfile as %s.\n", lineno, pt + 1);
fclose (wadfile);
return 1;
}
filename = makefile ? makefile : pt+1;
wadfile = fopen (filename, makefile ? "w" : "wb");
if (wadfile == NULL)
{
fprintf (stderr, "Line %d: Could not open %s: %s\n", lineno, filename, strerror(errno));
return 1;
}
if (makefile)
{ // Write out the only rule the makefile has
fprintf (wadfile, "%s: %s", pt+1, listfilename);
}
else
{
// The correct header will be written once the wad is complete
fwrite (&header, sizeof(header), 1, wadfile);
}
continue;
}
// Everything up to the next whitespace is the lump name
lumpname = pt;
filename = NULL;
while (*pt && !isspace(*pt)) pt++;
// If there is more, skip whitespace and use the remainder of the line
// as the file to insert into the lump. If no filename is given, then
// a 0-length marker lump is inserted.
if (*pt)
{
*pt = 0;
filename = pt + 1;
while (*filename && isspace(*filename)) filename++;
if (*filename == 0) filename = NULL;
}
if (wadfile == NULL)
{
fprintf (stderr, "Line %d: No wad specified before lumps.\n", lineno);
return 1;
}
if (makefile)
{
if (filename != NULL)
{
if (strchr (filename, ' '))
{
fprintf (wadfile, " \\\n\t\"%s\"", filename);
}
else
{
fprintf (wadfile, " \\\n\t%s", filename);
}
}
}
else
{
for (i = 0; lumpname[i]; ++i)
{
lumpname[i] = toupper(lumpname[i]);
}
strncpy (directory[header.numlumps].name, lumpname, 8);
directory[header.numlumps].filepos = ftell (wadfile);
if (filename != NULL)
{
ret |= appendlump (wadfile, filename);
directory[header.numlumps].size = ftell (wadfile) - directory[header.numlumps].filepos;
}
header.numlumps++;
}
}
if (wadfile != NULL)
{
if (makefile)
{
fprintf (wadfile, "\n\t%s %s\n", makecmd, listfilename);
}
else
{
header.infotableofs = ftell (wadfile);
#ifdef WORDS_BIGENDIAN
#define SWAP(x) ((((x)>>24)|(((x)>>8) & 0xff00)|(((x)<<8) & 0xff0000)|((x)<<24)))
for (i = 0; i < header.numlumps; ++i)
{
directory[i].filepos = SWAP(directory[i].filepos);
directory[i].size = SWAP(directory[i].size);
}
#endif
if (fwrite (directory, sizeof(directory[0]), header.numlumps, wadfile) != header.numlumps)
{
fprintf (stderr, "Error writing to wad: %s\n", strerror(errno));
ret |= 1;
}
else
{
#ifdef WORDS_BIGENDIAN
SWAP(header.infotableofs);
SWAP(header.numlumps);
#endif
fseek (wadfile, 0, SEEK_SET);
if (fwrite (&header, sizeof(header), 1, wadfile) != 1)
{
fprintf (stderr, "Error writing to wad: %s\n", strerror(errno));
ret |= 1;
}
}
}
fclose (wadfile);
}
return ret;
}
int main (int argc, char **argv)
{
FILE *listfile = NULL;
char *listfilename = NULL;
char *makefile = NULL;
int ret;
int i;
for (i = 1; i < argc; ++i)
{
if (strcmp (argv[i], "-make") == 0)
{
if (i >= argc-1)
{
goto baduse;
}
makefile = argv[++i];
}
else if (listfile == NULL)
{
listfilename = argv[i];
listfile = fopen (listfilename, "r");
if (listfile == NULL)
{
fprintf (stderr, "Can't open %s: %s\n", listfilename, strerror(errno));
return 1;
}
}
else
{
fclose (listfile);
goto baduse;
}
}
if (makefile != NULL && listfile == NULL)
{
fprintf (stderr, "You must specify a listfile if you want a makefile\n");
return 1;
}
ret = buildwad (listfile ? listfile : stdin, listfilename, argv[0], makefile);
if (listfile != NULL)
{
fclose (listfile);
}
return ret;
baduse:
fprintf (stderr, "Usage: makewad [-make makecommand makefile] [listfile]\n");
return 1;
}

120
tools/makewad/makewad.vcproj Normal file
View file

@ -0,0 +1,120 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="7.10"
Name="makewad"
ProjectGUID="{24A19C02-F041-4AB0-A1A1-02E1E88EDBD3}"
RootNamespace="makewad"
Keyword="Win32Proj">
<Platforms>
<Platform
Name="Win32"/>
</Platforms>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="Debug"
IntermediateDirectory="Debug"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="5"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="4"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/makewad.exe"
LinkIncremental="2"
GenerateDebugInformation="TRUE"
ProgramDatabaseFile="$(OutDir)/makewad.pdb"
SubSystem="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="copy &quot;$(TargetPath)&quot; &quot;$(ProjectDir)\$(TargetFileName)&quot;"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="Release"
IntermediateDirectory="Release"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="4"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="3"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/makewad.exe"
LinkIncremental="1"
GenerateDebugInformation="TRUE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="copy &quot;$(TargetPath)&quot; &quot;$(ProjectDir)\$(TargetFileName)&quot;"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<File
RelativePath=".\makewad.c">
</File>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

47
tools/re2c/Makefile.am Normal file
View file

@ -0,0 +1,47 @@
# $Id: Makefile.am,v 1.15 2004/05/26 23:59:15 nuffer Exp $
bin_PROGRAMS = re2c
re2c_SOURCES = code.cc dfa.cc main.cc parser.cc actions.cc scanner.re substr.cc\
translate.cc scanner.cc mbo_getopt.cc \
basics.h dfa.h globals.h ins.h parser.h re.h scanner.h \
substr.h token.h mbo_getopt.h
BUILT_SOURCES = parser.cc scanner.cc
man_MANS = re2c.1
#CXXFLAGS = -O2 -Wall -I. -Wno-unused -Wno-parentheses -Wno-deprecated
YFLAGS = -d
RE2C = re2c
RE2CFLAGS = -s
CLEANFILES = parser.cc y.tab.c y.tab.h parser.cc re2c.1 .version
DISTCLEANFILES = makerpm re2c.spec README scanner.cc re2c$(EXEEXT)
EXTRA_DIST = $(man_MANS) README parser.y scanner.re makerpm.in re2c.spec.in \
README.in y.tab.h \
examples/basemmap.c examples/c.re examples/cmmap.re \
examples/cnokw.re examples/cunroll.re examples/modula.re \
examples/sample.re examples/simple.re examples/rexx/README \
examples/rexx/rexx.l examples/rexx/scanio.c doc/loplas.ps \
doc/sample.bib
rpm-files: $(bin_PROGRAMS) $(EXTRA_DIST)
parser.cc: $(top_srcdir)/parser.y
$(YACC) $(YFLAGS) $<
mv -f y.tab.c $(top_srcdir)/parser.cc
if cmp -s y.tab.h $(top_srcdir)/y.tab.h; then :; else mv -f y.tab.h $(top_srcdir)/y.tab.h; fi
scanner.cc: $(top_srcdir)/scanner.re
@if test -x ./re2c$(EXEEXT); then \
echo "re2c $(RE2CFLAGS) -o $@ $<"; \
./re2c $(RE2CFLAGS) -o $@ $< && cp $@ $(top_srcdir)/bootstrap/; \
else \
echo "cp -f $(top_srcdir)/bootstrap/$@ $@"; \
cp -f $(top_srcdir)/bootstrap/$@ $@; \
fi
.version:
echo $(PACKAGE_VERSION) > .version

472
tools/re2c/Makefile.in Normal file
View file

@ -0,0 +1,472 @@
# Makefile.in generated automatically by automake 1.4-p6 from Makefile.am
# Copyright (C) 1994, 1995-8, 1999, 2001 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
# $Id: Makefile.am,v 1.15 2004/05/26 23:59:15 nuffer Exp $
SHELL = @SHELL@
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
prefix = @prefix@
exec_prefix = @exec_prefix@
bindir = @bindir@
sbindir = @sbindir@
libexecdir = @libexecdir@
datadir = @datadir@
sysconfdir = @sysconfdir@
sharedstatedir = @sharedstatedir@
localstatedir = @localstatedir@
libdir = @libdir@
infodir = @infodir@
mandir = @mandir@
includedir = @includedir@
oldincludedir = /usr/include
DESTDIR =
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = .
ACLOCAL = @ACLOCAL@
AUTOCONF = @AUTOCONF@
AUTOMAKE = @AUTOMAKE@
AUTOHEADER = @AUTOHEADER@
INSTALL = @INSTALL@
INSTALL_PROGRAM = @INSTALL_PROGRAM@ $(AM_INSTALL_PROGRAM_FLAGS)
INSTALL_DATA = @INSTALL_DATA@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
transform = @program_transform_name@
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
CC = @CC@
CXX = @CXX@
MAKEINFO = @MAKEINFO@
PACKAGE = @PACKAGE@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
VERSION = @VERSION@
YACC = @YACC@
bin_PROGRAMS = re2c
re2c_SOURCES = code.cc dfa.cc main.cc parser.cc actions.cc scanner.re substr.cc translate.cc scanner.cc mbo_getopt.cc basics.h dfa.h globals.h ins.h parser.h re.h scanner.h substr.h token.h mbo_getopt.h
BUILT_SOURCES = parser.cc scanner.cc
man_MANS = re2c.1
#CXXFLAGS = -O2 -Wall -I. -Wno-unused -Wno-parentheses -Wno-deprecated
YFLAGS = -d
RE2C = re2c
RE2CFLAGS = -s
CLEANFILES = parser.cc y.tab.c y.tab.h parser.cc re2c.1 .version
DISTCLEANFILES = makerpm re2c.spec README scanner.cc re2c$(EXEEXT)
EXTRA_DIST = $(man_MANS) README parser.y scanner.re makerpm.in re2c.spec.in README.in y.tab.h examples/basemmap.c examples/c.re examples/cmmap.re examples/cnokw.re examples/cunroll.re examples/modula.re examples/sample.re examples/simple.re examples/rexx/README examples/rexx/rexx.l examples/rexx/scanio.c doc/loplas.ps doc/sample.bib
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
CONFIG_HEADER = config.h
CONFIG_CLEAN_FILES = README makerpm re2c.1 re2c.spec
PROGRAMS = $(bin_PROGRAMS)
DEFS = @DEFS@ -I. -I$(srcdir) -I.
CPPFLAGS = @CPPFLAGS@
LDFLAGS = @LDFLAGS@
LIBS = @LIBS@
re2c_OBJECTS = code.o dfa.o main.o parser.o actions.o substr.o \
translate.o scanner.o mbo_getopt.o
re2c_LDADD = $(LDADD)
re2c_DEPENDENCIES =
re2c_LDFLAGS =
CXXFLAGS = @CXXFLAGS@
CXXCOMPILE = $(CXX) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
CXXLD = $(CXX)
CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(LDFLAGS) -o $@
CFLAGS = @CFLAGS@
COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
CCLD = $(CC)
LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@
man1dir = $(mandir)/man1
MANS = $(man_MANS)
NROFF = nroff
DIST_COMMON = README ./stamp-h.in Makefile.am Makefile.in README.in \
aclocal.m4 config.h.in configure configure.in install-sh makerpm.in \
missing mkinstalldirs re2c.1.in re2c.spec.in
DISTFILES = $(DIST_COMMON) $(SOURCES) $(HEADERS) $(TEXINFOS) $(EXTRA_DIST)
TAR = gtar
GZIP_ENV = --best
SOURCES = $(re2c_SOURCES)
OBJECTS = $(re2c_OBJECTS)
all: all-redirect
.SUFFIXES:
.SUFFIXES: .S .c .cc .o .s
$(srcdir)/Makefile.in: Makefile.am $(top_srcdir)/configure.in $(ACLOCAL_M4)
cd $(top_srcdir) && $(AUTOMAKE) --foreign --include-deps Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
$(ACLOCAL_M4): configure.in
cd $(srcdir) && $(ACLOCAL)
config.status: $(srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
$(SHELL) ./config.status --recheck
$(srcdir)/configure: $(srcdir)/configure.in $(ACLOCAL_M4) $(CONFIGURE_DEPENDENCIES)
cd $(srcdir) && $(AUTOCONF)
config.h: stamp-h
@if test ! -f $@; then \
rm -f stamp-h; \
$(MAKE) stamp-h; \
else :; fi
stamp-h: $(srcdir)/config.h.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES= CONFIG_HEADERS=config.h \
$(SHELL) ./config.status
@echo timestamp > stamp-h 2> /dev/null
$(srcdir)/config.h.in: $(srcdir)/stamp-h.in
@if test ! -f $@; then \
rm -f $(srcdir)/stamp-h.in; \
$(MAKE) $(srcdir)/stamp-h.in; \
else :; fi
$(srcdir)/stamp-h.in: $(top_srcdir)/configure.in $(ACLOCAL_M4)
cd $(top_srcdir) && $(AUTOHEADER)
@echo timestamp > $(srcdir)/stamp-h.in 2> /dev/null
mostlyclean-hdr:
clean-hdr:
distclean-hdr:
-rm -f config.h
maintainer-clean-hdr:
README: $(top_builddir)/config.status README.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
makerpm: $(top_builddir)/config.status makerpm.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
re2c.1: $(top_builddir)/config.status re2c.1.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
re2c.spec: $(top_builddir)/config.status re2c.spec.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
mostlyclean-binPROGRAMS:
clean-binPROGRAMS:
-test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS)
distclean-binPROGRAMS:
maintainer-clean-binPROGRAMS:
install-binPROGRAMS: $(bin_PROGRAMS)
@$(NORMAL_INSTALL)
$(mkinstalldirs) $(DESTDIR)$(bindir)
@list='$(bin_PROGRAMS)'; for p in $$list; do \
if test -f $$p; then \
echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/`echo $$p|sed 's/$(EXEEXT)$$//'|sed '$(transform)'|sed 's/$$/$(EXEEXT)/'`"; \
$(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/`echo $$p|sed 's/$(EXEEXT)$$//'|sed '$(transform)'|sed 's/$$/$(EXEEXT)/'`; \
else :; fi; \
done
uninstall-binPROGRAMS:
@$(NORMAL_UNINSTALL)
list='$(bin_PROGRAMS)'; for p in $$list; do \
rm -f $(DESTDIR)$(bindir)/`echo $$p|sed 's/$(EXEEXT)$$//'|sed '$(transform)'|sed 's/$$/$(EXEEXT)/'`; \
done
.c.o:
$(COMPILE) -c $<
.s.o:
$(COMPILE) -c $<
.S.o:
$(COMPILE) -c $<
mostlyclean-compile:
-rm -f *.o core *.core
clean-compile:
distclean-compile:
-rm -f *.tab.c
maintainer-clean-compile:
re2c: $(re2c_OBJECTS) $(re2c_DEPENDENCIES)
@rm -f re2c
$(CXXLINK) $(re2c_LDFLAGS) $(re2c_OBJECTS) $(re2c_LDADD) $(LIBS)
.cc.o:
$(CXXCOMPILE) -c $<
install-man1:
$(mkinstalldirs) $(DESTDIR)$(man1dir)
@list='$(man1_MANS)'; \
l2='$(man_MANS)'; for i in $$l2; do \
case "$$i" in \
*.1*) list="$$list $$i" ;; \
esac; \
done; \
for i in $$list; do \
if test -f $(srcdir)/$$i; then file=$(srcdir)/$$i; \
else file=$$i; fi; \
ext=`echo $$i | sed -e 's/^.*\\.//'`; \
inst=`echo $$i | sed -e 's/\\.[0-9a-z]*$$//'`; \
inst=`echo $$inst | sed '$(transform)'`.$$ext; \
echo " $(INSTALL_DATA) $$file $(DESTDIR)$(man1dir)/$$inst"; \
$(INSTALL_DATA) $$file $(DESTDIR)$(man1dir)/$$inst; \
done
uninstall-man1:
@list='$(man1_MANS)'; \
l2='$(man_MANS)'; for i in $$l2; do \
case "$$i" in \
*.1*) list="$$list $$i" ;; \
esac; \
done; \
for i in $$list; do \
ext=`echo $$i | sed -e 's/^.*\\.//'`; \
inst=`echo $$i | sed -e 's/\\.[0-9a-z]*$$//'`; \
inst=`echo $$inst | sed '$(transform)'`.$$ext; \
echo " rm -f $(DESTDIR)$(man1dir)/$$inst"; \
rm -f $(DESTDIR)$(man1dir)/$$inst; \
done
install-man: $(MANS)
@$(NORMAL_INSTALL)
$(MAKE) $(AM_MAKEFLAGS) install-man1
uninstall-man:
@$(NORMAL_UNINSTALL)
$(MAKE) $(AM_MAKEFLAGS) uninstall-man1
tags: TAGS
ID: $(HEADERS) $(SOURCES) $(LISP)
list='$(SOURCES) $(HEADERS)'; \
unique=`for i in $$list; do echo $$i; done | \
awk ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
here=`pwd` && cd $(srcdir) \
&& mkid -f$$here/ID $$unique $(LISP)
TAGS: $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS)'; \
unique=`for i in $$list; do echo $$i; done | \
awk ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(ETAGS_ARGS)config.h.in$$unique$(LISP)$$tags" \
|| (cd $(srcdir) && etags $(ETAGS_ARGS) $$tags config.h.in $$unique $(LISP) -o $$here/TAGS)
mostlyclean-tags:
clean-tags:
distclean-tags:
-rm -f TAGS ID
maintainer-clean-tags:
distdir = $(PACKAGE)-$(VERSION)
top_distdir = $(distdir)
# This target untars the dist file and tries a VPATH configuration. Then
# it guarantees that the distribution is self-contained by making another
# tarfile.
distcheck: dist
-rm -rf $(distdir)
GZIP=$(GZIP_ENV) $(TAR) zxf $(distdir).tar.gz
mkdir $(distdir)/=build
mkdir $(distdir)/=inst
dc_install_base=`cd $(distdir)/=inst && pwd`; \
cd $(distdir)/=build \
&& ../configure --srcdir=.. --prefix=$$dc_install_base \
&& $(MAKE) $(AM_MAKEFLAGS) \
&& $(MAKE) $(AM_MAKEFLAGS) dvi \
&& $(MAKE) $(AM_MAKEFLAGS) check \
&& $(MAKE) $(AM_MAKEFLAGS) install \
&& $(MAKE) $(AM_MAKEFLAGS) installcheck \
&& $(MAKE) $(AM_MAKEFLAGS) dist
-rm -rf $(distdir)
@banner="$(distdir).tar.gz is ready for distribution"; \
dashes=`echo "$$banner" | sed s/./=/g`; \
echo "$$dashes"; \
echo "$$banner"; \
echo "$$dashes"
dist: distdir
-chmod -R a+r $(distdir)
GZIP=$(GZIP_ENV) $(TAR) chozf $(distdir).tar.gz $(distdir)
-rm -rf $(distdir)
dist-all: distdir
-chmod -R a+r $(distdir)
GZIP=$(GZIP_ENV) $(TAR) chozf $(distdir).tar.gz $(distdir)
-rm -rf $(distdir)
distdir: $(DISTFILES)
-rm -rf $(distdir)
mkdir $(distdir)
-chmod 777 $(distdir)
$(mkinstalldirs) $(distdir)/doc $(distdir)/examples \
$(distdir)/examples/rexx
@for file in $(DISTFILES); do \
d=$(srcdir); \
if test -d $$d/$$file; then \
cp -pr $$d/$$file $(distdir)/$$file; \
else \
test -f $(distdir)/$$file \
|| ln $$d/$$file $(distdir)/$$file 2> /dev/null \
|| cp -p $$d/$$file $(distdir)/$$file || :; \
fi; \
done
actions.o: actions.cc globals.h basics.h config.h parser.h scanner.h \
token.h substr.h re.h ins.h dfa.h
code.o: code.cc substr.h basics.h config.h globals.h dfa.h re.h token.h \
ins.h parser.h scanner.h
dfa.o: dfa.cc globals.h basics.h config.h substr.h dfa.h re.h token.h \
ins.h
main.o: main.cc config.h globals.h basics.h parser.h scanner.h token.h \
substr.h re.h ins.h dfa.h mbo_getopt.h
mbo_getopt.o: mbo_getopt.cc mbo_getopt.h
parser.o: parser.cc config.h globals.h basics.h parser.h scanner.h \
token.h substr.h re.h ins.h
scanner.o: scanner.cc scanner.h token.h substr.h basics.h config.h \
parser.h re.h ins.h y.tab.h
substr.o: substr.cc substr.h basics.h config.h globals.h
translate.o: translate.cc globals.h basics.h config.h
info-am:
info: info-am
dvi-am:
dvi: dvi-am
check-am: all-am
check: check-am
installcheck-am:
installcheck: installcheck-am
all-recursive-am: config.h
$(MAKE) $(AM_MAKEFLAGS) all-recursive
install-exec-am: install-binPROGRAMS
install-exec: install-exec-am
install-data-am: install-man
install-data: install-data-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
install: install-am
uninstall-am: uninstall-binPROGRAMS uninstall-man
uninstall: uninstall-am
all-am: Makefile $(PROGRAMS) $(MANS) config.h
all-redirect: all-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) AM_INSTALL_PROGRAM_FLAGS=-s install
installdirs:
$(mkinstalldirs) $(DESTDIR)$(bindir) $(DESTDIR)$(mandir)/man1
mostlyclean-generic:
clean-generic:
-test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
distclean-generic:
-rm -f Makefile $(CONFIG_CLEAN_FILES)
-rm -f config.cache config.log stamp-h stamp-h[0-9]*
-test -z "$(DISTCLEANFILES)" || rm -f $(DISTCLEANFILES)
maintainer-clean-generic:
-test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES)
mostlyclean-am: mostlyclean-hdr mostlyclean-binPROGRAMS \
mostlyclean-compile mostlyclean-tags \
mostlyclean-generic
mostlyclean: mostlyclean-am
clean-am: clean-hdr clean-binPROGRAMS clean-compile clean-tags \
clean-generic mostlyclean-am
clean: clean-am
distclean-am: distclean-hdr distclean-binPROGRAMS distclean-compile \
distclean-tags distclean-generic clean-am
distclean: distclean-am
-rm -f config.status
maintainer-clean-am: maintainer-clean-hdr maintainer-clean-binPROGRAMS \
maintainer-clean-compile maintainer-clean-tags \
maintainer-clean-generic distclean-am
@echo "This command is intended for maintainers to use;"
@echo "it deletes files that may require special tools to rebuild."
maintainer-clean: maintainer-clean-am
-rm -f config.status
.PHONY: mostlyclean-hdr distclean-hdr clean-hdr maintainer-clean-hdr \
mostlyclean-binPROGRAMS distclean-binPROGRAMS clean-binPROGRAMS \
maintainer-clean-binPROGRAMS uninstall-binPROGRAMS install-binPROGRAMS \
mostlyclean-compile distclean-compile clean-compile \
maintainer-clean-compile install-man1 uninstall-man1 install-man \
uninstall-man tags mostlyclean-tags distclean-tags clean-tags \
maintainer-clean-tags distdir info-am info dvi-am dvi check check-am \
installcheck-am installcheck all-recursive-am install-exec-am \
install-exec install-data-am install-data install-am install \
uninstall-am uninstall all-redirect all-am all installdirs \
mostlyclean-generic distclean-generic clean-generic \
maintainer-clean-generic clean mostlyclean distclean maintainer-clean
rpm-files: $(bin_PROGRAMS) $(EXTRA_DIST)
parser.cc: $(top_srcdir)/parser.y
$(YACC) $(YFLAGS) $<
mv -f y.tab.c $(top_srcdir)/parser.cc
if cmp -s y.tab.h $(top_srcdir)/y.tab.h; then :; else mv -f y.tab.h $(top_srcdir)/y.tab.h; fi
scanner.cc: $(top_srcdir)/scanner.re
@if test -x ./re2c$(EXEEXT); then \
echo "re2c $(RE2CFLAGS) -o $@ $<"; \
./re2c $(RE2CFLAGS) -o $@ $< && cp $@ $(top_srcdir)/bootstrap/; \
else \
echo "cp -f $(top_srcdir)/bootstrap/$@ $@"; \
cp -f $(top_srcdir)/bootstrap/$@ $@; \
fi
.version:
echo $(PACKAGE_VERSION) > .version
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

31
tools/re2c/Makefile.mgw Normal file
View file

@ -0,0 +1,31 @@
EXE = re2c.exe
CCDV = @../../ccdv
CXX = g++
CXXFLAGS = $(LOC) -D_WIN32 -DNDEBUG -Os -Wall -Wno-unused
OBJS = actions.o code.o dfa.o main.o mbo_getopt.o parser.o scanner.o substr.o translate.o
all: $(EXE)
.cc.o:
$(CCDV) $(CXX) $(CXXFLAGS) -c -o $@ $<
$(EXE): $(OBJS)
$(CCDV) $(CXX) -s -o $(EXE) $(OBJS)
.PHONY: clean
clean:
-del /q /f $(EXE) 2>nul
-del /q /f *.o 2>nul
actions.o: actions.cc
code.o: code.cc
dfa.o: dfa.cc
main.o: main.cc
mbo_getopt.o: mbo_getopt.cc
parser.o: parser.cc
scanner.o: scanner.cc
substr.o: substr.cc
translate.o: translate.cc

165
tools/re2c/README Normal file
View file

@ -0,0 +1,165 @@
re2c Version 0.9.3
------------------
Originally written by Peter Bumbulis (peter@csg.uwaterloo.ca)
Currently maintained by:
Dan Nuffer <nuffer at users.sourceforge.net>
Marcus Boerger <helly at users.sourceforge.net>
Hartmut Kaiser <hkaiser at users.sourceforge.net>
The re2c distribution can be found at:
http://sourceforge.net/projects/re2c/
This distribution is a cleaned up version of the 0.5 release. Several
bugs were fixed as well as code cleanup for warning free compilation.
It has been developed and tested with egcs 1.0.2 and gcc 2.7.2.3, 2.96
and 3.3.1 on Linux x86. You can compile your own version with other gcc
version if you have yacc or any working bison version (tested up to bison
1.875).
You can build this software by simply typing the following commands:
./autogen.sh
./configure
make
The above version will be based on the pregenerated scanner.cc file.
If you want to build that file yourself (recommended when installing
re2c) you need the following steps:
./autogen.sh
./configure
make
rm -f scanner.cc
make install
Or you can create a rpm package and install it by the following commands:
./autogen.sh
./configure
./makerpm <release>
rpm -Uhv <packagedir>/re2c-0.9.3-<release>.rpm
Here <realease> should be a number like 1. And <packagedir> must equal
the directory where the makerpm step has written the generated rpm to.
re2c is a great tool for writing fast and flexible lexers. It has
served many people well for many years. re2c is on the order of 2-3
times faster than a flex based scanner, and its input model is much
more flexible.
Peter's original version 0.5 ANNOUNCE and README follows.
--
re2c is a tool for generating C-based recognizers from regular
expressions. re2c-based scanners are efficient: for programming
languages, given similar specifications, an re2c-based scanner is
typically almost twice as fast as a flex-based scanner with little or no
increase in size (possibly a decrease on cisc architectures). Indeed,
re2c-based scanners are quite competitive with hand-crafted ones.
Unlike flex, re2c does not generate complete scanners: the user must
supply some interface code. While this code is not bulky (about 50-100
lines for a flex-like scanner; see the man page and examples in the
distribution) careful coding is required for efficiency (and
correctness). One advantage of this arrangement is that the generated
code is not tied to any particular input model. For example, re2c
generated code can be used to scan data from a null-byte terminated
buffer as illustrated below.
Given the following source
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\000-\377] {return NULL;}
*/
}
re2c will generate
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
Note that most compilers will perform dead-code elimination to remove
all YYCURSOR, YYLIMIT comparisions.
re2c was developed for a particular project (constructing a fast REXX
scanner of all things!) and so while it has some rough edges, it should
be quite usable. More information about re2c can be found in the
(admittedly skimpy) man page; the algorithms and heuristics used are
described in an upcoming LOPLAS article (included in the distribution).
Probably the best way to find out more about re2c is to try the supplied
examples. re2c is written in C++, and is currently being developed
under Linux using gcc 2.5.8.
Peter
--
re2c is distributed with no warranty whatever. The code is certain to
contain errors. Neither the author nor any contributor takes
responsibility for any consequences of its use.
re2c is in the public domain. The data structures and algorithms used
in re2c are all either taken from documents available to the general
public or are inventions of the author. Programs generated by re2c may
be distributed freely. re2c itself may be distributed freely, in source
or binary, unchanged or modified. Distributors may charge whatever fees
they can obtain for re2c.
If you do make use of re2c, or incorporate it into a larger project an
acknowledgement somewhere (documentation, research report, etc.) would
be appreciated.
Please send bug reports and feedback (including suggestions for
improving the distribution) to
peter@csg.uwaterloo.ca
Include a small example and the banner from parser.y with bug reports.

165
tools/re2c/README.in Normal file
View file

@ -0,0 +1,165 @@
re2c Version @PACKAGE_VERSION@
------------------
Originally written by Peter Bumbulis (peter@csg.uwaterloo.ca)
Currently maintained by:
Dan Nuffer <nuffer at users.sourceforge.net>
Marcus Boerger <helly at users.sourceforge.net>
Hartmut Kaiser <hkaiser at users.sourceforge.net>
The re2c distribution can be found at:
http://sourceforge.net/projects/re2c/
This distribution is a cleaned up version of the 0.5 release. Several
bugs were fixed as well as code cleanup for warning free compilation.
It has been developed and tested with egcs 1.0.2 and gcc 2.7.2.3, 2.96
and 3.3.1 on Linux x86. You can compile your own version with other gcc
version if you have yacc or any working bison version (tested up to bison
1.875).
You can build this software by simply typing the following commands:
./autogen.sh
./configure
make
The above version will be based on the pregenerated scanner.cc file.
If you want to build that file yourself (recommended when installing
re2c) you need the following steps:
./autogen.sh
./configure
make
rm -f scanner.cc
make install
Or you can create a rpm package and install it by the following commands:
./autogen.sh
./configure
./makerpm <release>
rpm -Uhv <packagedir>/re2c-@PACKAGE_VERSION@-<release>.rpm
Here <realease> should be a number like 1. And <packagedir> must equal
the directory where the makerpm step has written the generated rpm to.
re2c is a great tool for writing fast and flexible lexers. It has
served many people well for many years. re2c is on the order of 2-3
times faster than a flex based scanner, and its input model is much
more flexible.
Peter's original version 0.5 ANNOUNCE and README follows.
--
re2c is a tool for generating C-based recognizers from regular
expressions. re2c-based scanners are efficient: for programming
languages, given similar specifications, an re2c-based scanner is
typically almost twice as fast as a flex-based scanner with little or no
increase in size (possibly a decrease on cisc architectures). Indeed,
re2c-based scanners are quite competitive with hand-crafted ones.
Unlike flex, re2c does not generate complete scanners: the user must
supply some interface code. While this code is not bulky (about 50-100
lines for a flex-like scanner; see the man page and examples in the
distribution) careful coding is required for efficiency (and
correctness). One advantage of this arrangement is that the generated
code is not tied to any particular input model. For example, re2c
generated code can be used to scan data from a null-byte terminated
buffer as illustrated below.
Given the following source
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\000-\377] {return NULL;}
*/
}
re2c will generate
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
Note that most compilers will perform dead-code elimination to remove
all YYCURSOR, YYLIMIT comparisions.
re2c was developed for a particular project (constructing a fast REXX
scanner of all things!) and so while it has some rough edges, it should
be quite usable. More information about re2c can be found in the
(admittedly skimpy) man page; the algorithms and heuristics used are
described in an upcoming LOPLAS article (included in the distribution).
Probably the best way to find out more about re2c is to try the supplied
examples. re2c is written in C++, and is currently being developed
under Linux using gcc 2.5.8.
Peter
--
re2c is distributed with no warranty whatever. The code is certain to
contain errors. Neither the author nor any contributor takes
responsibility for any consequences of its use.
re2c is in the public domain. The data structures and algorithms used
in re2c are all either taken from documents available to the general
public or are inventions of the author. Programs generated by re2c may
be distributed freely. re2c itself may be distributed freely, in source
or binary, unchanged or modified. Distributors may charge whatever fees
they can obtain for re2c.
If you do make use of re2c, or incorporate it into a larger project an
acknowledgement somewhere (documentation, research report, etc.) would
be appreciated.
Please send bug reports and feedback (including suggestions for
improving the distribution) to
peter@csg.uwaterloo.ca
Include a small example and the banner from parser.y with bug reports.

157
tools/re2c/aclocal.m4 vendored Normal file
View file

@ -0,0 +1,157 @@
dnl aclocal.m4 generated automatically by aclocal 1.4-p6
dnl Copyright (C) 1994, 1995-8, 1999, 2001 Free Software Foundation, Inc.
dnl This file is free software; the Free Software Foundation
dnl gives unlimited permission to copy and/or distribute it,
dnl with or without modifications, as long as this notice is preserved.
dnl This program is distributed in the hope that it will be useful,
dnl but WITHOUT ANY WARRANTY, to the extent permitted by law; without
dnl even the implied warranty of MERCHANTABILITY or FITNESS FOR A
dnl PARTICULAR PURPOSE.
# Do all the work for Automake. This macro actually does too much --
# some checks are only needed if your package does certain things.
# But this isn't really a big deal.
# serial 1
dnl Usage:
dnl AM_INIT_AUTOMAKE(package,version, [no-define])
AC_DEFUN([AM_INIT_AUTOMAKE],
[AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl
AC_REQUIRE([AC_PROG_INSTALL])
PACKAGE=[$1]
AC_SUBST(PACKAGE)
VERSION=[$2]
AC_SUBST(VERSION)
dnl test to see if srcdir already configured
if test "`cd $srcdir && pwd`" != "`pwd`" && test -f $srcdir/config.status; then
AC_MSG_ERROR([source directory already configured; run "make distclean" there first])
fi
ifelse([$3],,
AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE", [Name of package])
AC_DEFINE_UNQUOTED(VERSION, "$VERSION", [Version number of package]))
AC_REQUIRE([AM_SANITY_CHECK])
AC_REQUIRE([AC_ARG_PROGRAM])
dnl FIXME This is truly gross.
missing_dir=`cd $ac_aux_dir && pwd`
AM_MISSING_PROG(ACLOCAL, aclocal-${am__api_version}, $missing_dir)
AM_MISSING_PROG(AUTOCONF, autoconf, $missing_dir)
AM_MISSING_PROG(AUTOMAKE, automake-${am__api_version}, $missing_dir)
AM_MISSING_PROG(AUTOHEADER, autoheader, $missing_dir)
AM_MISSING_PROG(MAKEINFO, makeinfo, $missing_dir)
AC_REQUIRE([AC_PROG_MAKE_SET])])
# Copyright 2002 Free Software Foundation, Inc.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# AM_AUTOMAKE_VERSION(VERSION)
# ----------------------------
# Automake X.Y traces this macro to ensure aclocal.m4 has been
# generated from the m4 files accompanying Automake X.Y.
AC_DEFUN([AM_AUTOMAKE_VERSION],[am__api_version="1.4"])
# AM_SET_CURRENT_AUTOMAKE_VERSION
# -------------------------------
# Call AM_AUTOMAKE_VERSION so it can be traced.
# This function is AC_REQUIREd by AC_INIT_AUTOMAKE.
AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
[AM_AUTOMAKE_VERSION([1.4-p6])])
#
# Check to make sure that the build environment is sane.
#
AC_DEFUN([AM_SANITY_CHECK],
[AC_MSG_CHECKING([whether build environment is sane])
# Just in case
sleep 1
echo timestamp > conftestfile
# Do `set' in a subshell so we don't clobber the current shell's
# arguments. Must try -L first in case configure is actually a
# symlink; some systems play weird games with the mod time of symlinks
# (eg FreeBSD returns the mod time of the symlink's containing
# directory).
if (
set X `ls -Lt $srcdir/configure conftestfile 2> /dev/null`
if test "[$]*" = "X"; then
# -L didn't work.
set X `ls -t $srcdir/configure conftestfile`
fi
if test "[$]*" != "X $srcdir/configure conftestfile" \
&& test "[$]*" != "X conftestfile $srcdir/configure"; then
# If neither matched, then we have a broken ls. This can happen
# if, for instance, CONFIG_SHELL is bash and it inherits a
# broken ls alias from the environment. This has actually
# happened. Such a system could not be considered "sane".
AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken
alias in your environment])
fi
test "[$]2" = conftestfile
)
then
# Ok.
:
else
AC_MSG_ERROR([newly created file is older than distributed files!
Check your system clock])
fi
rm -f conftest*
AC_MSG_RESULT(yes)])
dnl AM_MISSING_PROG(NAME, PROGRAM, DIRECTORY)
dnl The program must properly implement --version.
AC_DEFUN([AM_MISSING_PROG],
[AC_MSG_CHECKING(for working $2)
# Run test in a subshell; some versions of sh will print an error if
# an executable is not found, even if stderr is redirected.
# Redirect stdin to placate older versions of autoconf. Sigh.
if ($2 --version) < /dev/null > /dev/null 2>&1; then
$1=$2
AC_MSG_RESULT(found)
else
$1="$3/missing $2"
AC_MSG_RESULT(missing)
fi
AC_SUBST($1)])
# Like AC_CONFIG_HEADER, but automatically create stamp file.
AC_DEFUN([AM_CONFIG_HEADER],
[AC_PREREQ([2.12])
AC_CONFIG_HEADER([$1])
dnl When config.status generates a header, we must update the stamp-h file.
dnl This file resides in the same directory as the config header
dnl that is generated. We must strip everything past the first ":",
dnl and everything past the last "/".
AC_OUTPUT_COMMANDS(changequote(<<,>>)dnl
ifelse(patsubst(<<$1>>, <<[^ ]>>, <<>>), <<>>,
<<test -z "<<$>>CONFIG_HEADERS" || echo timestamp > patsubst(<<$1>>, <<^\([^:]*/\)?.*>>, <<\1>>)stamp-h<<>>dnl>>,
<<am_indx=1
for am_file in <<$1>>; do
case " <<$>>CONFIG_HEADERS " in
*" <<$>>am_file "*<<)>>
echo timestamp > `echo <<$>>am_file | sed -e 's%:.*%%' -e 's%[^/]*$%%'`stamp-h$am_indx
;;
esac
am_indx=`expr "<<$>>am_indx" + 1`
done<<>>dnl>>)
changequote([,]))])

568
tools/re2c/actions.cc Normal file
View file

@ -0,0 +1,568 @@
/* $Id: actions.cc,v 1.5 2004/05/13 02:58:17 nuffer Exp $ */
#include <time.h>
#include <string.h>
#include <iostream>
#include <iomanip>
#include "globals.h"
#include "parser.h"
#include "dfa.h"
Symbol *Symbol::first = NULL;
Symbol::Symbol(const SubStr &str) : next(first), name(str), re(NULL) {
first = this;
}
Symbol *Symbol::find(const SubStr &str){
for(Symbol *sym = first; sym; sym = sym->next)
if(sym->name == str) return sym;
return new Symbol(str);
}
void showIns(std::ostream &o, const Ins &i, const Ins &base){
o.width(3);
o << &i - &base << ": ";
switch(i.i.tag){
case CHAR: {
o << "match ";
for(const Ins *j = &(&i)[1]; j < (Ins*) i.i.link; ++j)
prtCh(o, j->c.value);
break;
} case GOTO:
o << "goto " << ((Ins*) i.i.link - &base);
break;
case FORK:
o << "fork " << ((Ins*) i.i.link - &base);
break;
case CTXT:
o << "term " << ((RuleOp*) i.i.link)->accept;
break;
case TERM:
o << "term " << ((RuleOp*) i.i.link)->accept;
break;
}
o << "\n";
}
uint RegExp::fixedLength(){
return ~0;
}
char *NullOp::type = "NullOp";
void NullOp::calcSize(Char*){
size = 0;
}
uint NullOp::fixedLength(){
return 0;
}
void NullOp::compile(Char*, Ins*){
;
}
void NullOp::split(CharSet&){
;
}
std::ostream& operator<<(std::ostream &o, const Range &r){
if((r.ub - r.lb) == 1){
prtCh(o, r.lb);
} else {
prtCh(o, r.lb); o << "-"; prtCh(o, r.ub-1);
}
return o << r.next;
}
Range *doUnion(Range *r1, Range *r2){
Range *r, **rP = &r;
for(;;){
Range *s;
if(r1->lb <= r2->lb){
s = new Range(*r1);
} else {
s = new Range(*r2);
}
*rP = s;
rP = &s->next;
for(;;){
if(r1->lb <= r2->lb){
if(r1->lb > s->ub)
break;
if(r1->ub > s->ub)
s->ub = r1->ub;
if(!(r1 = r1->next)){
uint ub = 0;
for(; r2 && r2->lb <= s->ub; r2 = r2->next)
ub = r2->ub;
if(ub > s->ub)
s->ub = ub;
*rP = r2;
return r;
}
} else {
if(r2->lb > s->ub)
break;
if(r2->ub > s->ub)
s->ub = r2->ub;
if(!(r2 = r2->next)){
uint ub = 0;
for(; r1 && r1->lb <= s->ub; r1 = r1->next)
ub = r1->ub;
if(ub > s->ub)
s->ub = ub;
*rP = r1;
return r;
}
}
}
}
*rP = NULL;
return r;
}
Range *doDiff(Range *r1, Range *r2){
Range *r, *s, **rP = &r;
for(; r1; r1 = r1->next){
uint lb = r1->lb;
for(; r2 && r2->ub <= r1->lb; r2 = r2->next);
for(; r2 && r2->lb < r1->ub; r2 = r2->next){
if(lb < r2->lb){
*rP = s = new Range(lb, r2->lb);
rP = &s->next;
}
if((lb = r2->ub) >= r1->ub)
goto noMore;
}
*rP = s = new Range(lb, r1->ub);
rP = &s->next;
noMore:;
}
*rP = NULL;
return r;
}
MatchOp *merge(MatchOp *m1, MatchOp *m2){
if(!m1)
return m2;
if(!m2)
return m1;
return new MatchOp(doUnion(m1->match, m2->match));
}
char *MatchOp::type = "MatchOp";
void MatchOp::display(std::ostream &o) const{
o << match;
}
void MatchOp::calcSize(Char *rep){
size = 1;
for(Range *r = match; r; r = r->next)
for(uint c = r->lb; c < r->ub; ++c)
if(rep[c] == c)
++size;
}
uint MatchOp::fixedLength(){
return 1;
}
void MatchOp::compile(Char *rep, Ins *i){
i->i.tag = CHAR;
i->i.link = &i[size];
Ins *j = &i[1];
uint bump = size;
for(Range *r = match; r; r = r->next){
for(uint c = r->lb; c < r->ub; ++c){
if(rep[c] == c){
j->c.value = c;
j->c.bump = --bump;
j++;
}
}
}
}
void MatchOp::split(CharSet &s){
for(Range *r = match; r; r = r->next){
for(uint c = r->lb; c < r->ub; ++c){
CharPtn *x = s.rep[c], *a = x->nxt;
if(!a){
if(x->card == 1)
continue;
x->nxt = a = s.freeHead;
if(!(s.freeHead = s.freeHead->nxt))
s.freeTail = &s.freeHead;
a->nxt = NULL;
x->fix = s.fix;
s.fix = x;
}
if(--(x->card) == 0){
*s.freeTail = x;
*(s.freeTail = &x->nxt) = NULL;
}
s.rep[c] = a;
++(a->card);
}
}
for(; s.fix; s.fix = s.fix->fix)
if(s.fix->card)
s.fix->nxt = NULL;
}
RegExp *mkDiff(RegExp *e1, RegExp *e2){
MatchOp *m1, *m2;
if(!(m1 = (MatchOp*) e1->isA(MatchOp::type)))
return NULL;
if(!(m2 = (MatchOp*) e2->isA(MatchOp::type)))
return NULL;
Range *r = doDiff(m1->match, m2->match);
return r? (RegExp*) new MatchOp(r) : (RegExp*) new NullOp;
}
RegExp *doAlt(RegExp *e1, RegExp *e2){
if(!e1)
return e2;
if(!e2)
return e1;
return new AltOp(e1, e2);
}
RegExp *mkAlt(RegExp *e1, RegExp *e2){
AltOp *a;
MatchOp *m1, *m2;
if((a = (AltOp*) e1->isA(AltOp::type))){
if((m1 = (MatchOp*) a->exp1->isA(MatchOp::type)))
e1 = a->exp2;
} else if((m1 = (MatchOp*) e1->isA(MatchOp::type))){
e1 = NULL;
}
if((a = (AltOp*) e2->isA(AltOp::type))){
if((m2 = (MatchOp*) a->exp1->isA(MatchOp::type)))
e2 = a->exp2;
} else if((m2 = (MatchOp*) e2->isA(MatchOp::type))){
e2 = NULL;
}
return doAlt(merge(m1, m2), doAlt(e1, e2));
}
char *AltOp::type = "AltOp";
void AltOp::calcSize(Char *rep){
exp1->calcSize(rep);
exp2->calcSize(rep);
size = exp1->size + exp2->size + 2;
}
uint AltOp::fixedLength(){
uint l1 = exp1->fixedLength();
uint l2 = exp1->fixedLength();
if(l1 != l2 || l1 == ~0u)
return ~0;
return l1;
}
void AltOp::compile(Char *rep, Ins *i){
i->i.tag = FORK;
Ins *j = &i[exp1->size + 1];
i->i.link = &j[1];
exp1->compile(rep, &i[1]);
j->i.tag = GOTO;
j->i.link = &j[exp2->size + 1];
exp2->compile(rep, &j[1]);
}
void AltOp::split(CharSet &s){
exp1->split(s);
exp2->split(s);
}
char *CatOp::type = "CatOp";
void CatOp::calcSize(Char *rep){
exp1->calcSize(rep);
exp2->calcSize(rep);
size = exp1->size + exp2->size;
}
uint CatOp::fixedLength(){
uint l1, l2;
if((l1 = exp1->fixedLength()) != ~0u )
if((l2 = exp2->fixedLength()) != ~0u)
return l1+l2;
return ~0;
}
void CatOp::compile(Char *rep, Ins *i){
exp1->compile(rep, &i[0]);
exp2->compile(rep, &i[exp1->size]);
}
void CatOp::split(CharSet &s){
exp1->split(s);
exp2->split(s);
}
char *CloseOp::type = "CloseOp";
void CloseOp::calcSize(Char *rep){
exp->calcSize(rep);
size = exp->size + 1;
}
void CloseOp::compile(Char *rep, Ins *i){
exp->compile(rep, &i[0]);
i += exp->size;
i->i.tag = FORK;
i->i.link = i - exp->size;
}
void CloseOp::split(CharSet &s){
exp->split(s);
}
char *CloseVOp::type = "CloseVOp";
void CloseVOp::calcSize(Char *rep){
exp->calcSize(rep);
if(max >= 0)
size = (exp->size * min) + ((1 + exp->size) * (max - min));
else
size = (exp->size * min) + 1;
}
void CloseVOp::compile(Char *rep, Ins *i){
Ins *jumppoint;
int st = 0;
jumppoint = i + ((1 + exp->size) * (max - min));
for(st = min; st < max; st++) {
i->i.tag = FORK;
i->i.link = jumppoint;
i+=1;
exp->compile(rep, &i[0]);
i += exp->size;
}
for(st = 0; st < min; st++) {
exp->compile(rep, &i[0]);
i += exp->size;
if(max < 0 && st == 0) {
i->i.tag = FORK;
i->i.link = i - exp->size;
i++;
}
}
}
void CloseVOp::split(CharSet &s){
exp->split(s);
}
RegExp *expr(Scanner &);
uchar unescape(SubStr &s){
s.len--;
uchar c;
if((c = *s.str++) != '\\' || s.len == 0)
return xlat[c];
s.len--;
switch(c = *s.str++){
case 'n':
return xlat['\n'];
case 't':
return xlat['\t'];
case 'v':
return xlat['\v'];
case 'b':
return xlat['\b'];
case 'r':
return xlat['\r'];
case 'f':
return xlat['\f'];
case 'a':
return xlat['\a'];
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7': {
uchar v = c - '0';
for(; s.len != 0 && '0' <= (c = *s.str) && c <= '7'; s.len--, s.str++)
v = v*8 + (c - '0');
return v;
} default:
return xlat[c];
}
}
Range *getRange(SubStr &s){
uchar lb = unescape(s), ub;
if(s.len < 2 || *s.str != '-'){
ub = lb;
} else {
s.len--; s.str++;
ub = unescape(s);
if(ub < lb){
uchar tmp;
tmp = lb; lb = ub; ub = tmp;
}
}
return new Range(lb, ub+1);
}
RegExp *matchChar(uint c){
return new MatchOp(new Range(c, c+1));
}
RegExp *strToRE(SubStr s){
s.len -= 2; s.str += 1;
if(s.len == 0)
return new NullOp;
RegExp *re = matchChar(unescape(s));
while(s.len > 0)
re = new CatOp(re, matchChar(unescape(s)));
return re;
}
RegExp *strToCaseInsensitiveRE(SubStr s){
s.len -= 2; s.str += 1;
if(s.len == 0)
return new NullOp;
uchar c = unescape(s);
RegExp *re, *reL, *reU;
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
reL = matchChar(tolower(c));
reU = matchChar(toupper(c));
re = mkAlt(reL, reU);
} else {
re = matchChar(c);
}
while(s.len > 0) {
uchar c = unescape(s);
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
reL = matchChar(tolower(c));
reU = matchChar(toupper(c));
re = new CatOp(re, mkAlt(reL, reU));
} else {
re = new CatOp(re, matchChar(c));
}
}
return re;
}
RegExp *ranToRE(SubStr s){
s.len -= 2; s.str += 1;
if(s.len == 0)
return new NullOp;
Range *r = getRange(s);
while(s.len > 0)
r = doUnion(r, getRange(s));
return new MatchOp(r);
}
char *RuleOp::type = "RuleOp";
RuleOp::RuleOp(RegExp *e, RegExp *c, Token *t, uint a)
: exp(e), ctx(c), ins(NULL), accept(a), code(t) {
;
}
void RuleOp::calcSize(Char *rep){
exp->calcSize(rep);
ctx->calcSize(rep);
size = exp->size + ctx->size + 1;
}
void RuleOp::compile(Char *rep, Ins *i){
ins = i;
exp->compile(rep, &i[0]);
i += exp->size;
ctx->compile(rep, &i[0]);
i += ctx->size;
i->i.tag = TERM;
i->i.link = this;
}
void RuleOp::split(CharSet &s){
exp->split(s);
ctx->split(s);
}
extern void printSpan(std::ostream&, uint, uint);
void optimize(Ins *i){
while(!isMarked(i)){
mark(i);
if(i->i.tag == CHAR){
i = (Ins*) i->i.link;
} else if(i->i.tag == GOTO || i->i.tag == FORK){
Ins *target = (Ins*) i->i.link;
optimize(target);
if(target->i.tag == GOTO)
i->i.link = target->i.link == target? i : target;
if(i->i.tag == FORK){
Ins *follow = (Ins*) &i[1];
optimize(follow);
if(follow->i.tag == GOTO && follow->i.link == follow){
i->i.tag = GOTO;
} else if(i->i.link == i){
i->i.tag = GOTO;
i->i.link = follow;
}
}
return;
} else {
++i;
}
}
}
void genCode(std::ostream& o, RegExp *re){
CharSet cs;
uint j;
memset(&cs, 0, sizeof(cs));
for(j = 0; j < nChars; ++j){
cs.rep[j] = &cs.ptn[0];
cs.ptn[j].nxt = &cs.ptn[j+1];
}
cs.freeHead = &cs.ptn[1];
*(cs.freeTail = &cs.ptn[nChars-1].nxt) = NULL;
cs.ptn[0].card = nChars;
cs.ptn[0].nxt = NULL;
re->split(cs);
/*
for(uint k = 0; k < nChars;){
for(j = k; ++k < nChars && cs.rep[k] == cs.rep[j];);
printSpan(cerr, j, k);
cerr << "\t" << cs.rep[j] - &cs.ptn[0] << endl;
}
*/
Char rep[nChars];
for(j = 0; j < nChars; ++j){
if(!cs.rep[j]->nxt)
cs.rep[j]->nxt = &cs.ptn[j];
rep[j] = (Char) (cs.rep[j]->nxt - &cs.ptn[0]);
}
re->calcSize(rep);
Ins *ins = new Ins[re->size+1];
memset(ins, 0, (re->size+1)*sizeof(Ins));
re->compile(rep, ins);
Ins *eoi = &ins[re->size];
eoi->i.tag = GOTO;
eoi->i.link = eoi;
optimize(ins);
for(j = 0; j < re->size;){
unmark(&ins[j]);
if(ins[j].i.tag == CHAR){
j = (Ins*) ins[j].i.link - ins;
} else {
j++;
}
}
DFA *dfa = new DFA(ins, re->size, 0, 256, rep);
dfa->emit(o);
delete dfa;
delete [] ins;
}

63
tools/re2c/basics.h Normal file
View file

@ -0,0 +1,63 @@
/* $Id: basics.h,v 1.3 2004/03/13 13:40:37 helly Exp $ */
#ifndef _basics_h
#define _basics_h
#ifdef HAVE_CONFIG_H
#include "config.h"
#elif _WIN32
#include "configwin.h"
#endif
#if SIZEOF_CHAR == 1
typedef unsigned char byte;
#elif SIZEOF_SHORT == 1
typedef unsigned short byte;
#elif SIZEOF_INT == 1
typedef unsigned int byte;
#elif SIZEOF_LONG == 1
typedef unsigned long byte;
#else
typedef unsigned char byte;
#endif
#if SIZEOF_CHAR == 2
typedef unsigned char word;
#elif SIZEOF_SHORT == 2
typedef unsigned short word;
#elif SIZEOF_INT == 2
typedef unsigned int word;
#elif SIZEOF_LONG == 2
typedef unsigned long word;
#else
typedef unsigned short word;
#endif
#if SIZEOF_CHAR == 4
typedef unsigned char dword;
#elif SIZEOF_SHORT == 4
typedef unsigned short dword;
#elif SIZEOF_INT == 4
typedef unsigned int dword;
#elif SIZEOF_LONG == 4
typedef unsigned long dword;
#else
typedef unsigned long dword;
#endif
#ifndef HAVE_UINT
typedef unsigned int uint;
#endif
#ifndef HAVE_UCHAR
typedef unsigned char uchar;
#endif
#ifndef HAVE_USHORT
typedef unsigned short ushort;
#endif
#ifndef HAVE_ULONG
typedef unsigned long ulong;
#endif
#endif

964
tools/re2c/code.cc Normal file
View file

@ -0,0 +1,964 @@
/* $Id: code.cc,v 1.10 2004/05/27 00:02:01 nuffer Exp $ */
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <iomanip>
#include <iostream>
#include "substr.h"
#include "globals.h"
#include "dfa.h"
#include "parser.h"
// there must be at least one span in list; all spans must cover
// same range
void Go::compact()
{
// arrange so that adjacent spans have different targets
uint i = 0;
for(uint j = 1; j < nSpans; ++j)
{
if(span[j].to != span[i].to)
{
++i; span[i].to = span[j].to;
}
span[i].ub = span[j].ub;
}
nSpans = i + 1;
}
void Go::unmap(Go *base, State *x)
{
Span *s = span, *b = base->span, *e = &b[base->nSpans];
uint lb = 0;
s->ub = 0;
s->to = NULL;
for(; b != e; ++b)
{
if(b->to == x)
{
if((s->ub - lb) > 1)
{
s->ub = b->ub;
}
} else {
if(b->to != s->to)
{
if(s->ub)
{
lb = s->ub; ++s;
}
s->to = b->to;
}
s->ub = b->ub;
}
}
s->ub = e[-1].ub; ++s;
nSpans = s - span;
}
void doGen(Go *g, State *s, uchar *bm, uchar m)
{
Span *b = g->span, *e = &b[g->nSpans];
uint lb = 0;
for(; b < e; ++b)
{
if(b->to == s)
{
for(; lb < b->ub; ++lb)
{
bm[lb] |= m;
}
}
lb = b->ub;
}
}
void prt(std::ostream& o, Go *g, State *s)
{
Span *b = g->span, *e = &b[g->nSpans];
uint lb = 0;
for(; b < e; ++b)
{
if(b->to == s)
{
printSpan(o, lb, b->ub);
}
lb = b->ub;
}
}
bool matches(Go *g1, State *s1, Go *g2, State *s2)
{
Span *b1 = g1->span, *e1 = &b1[g1->nSpans];
uint lb1 = 0;
Span *b2 = g2->span, *e2 = &b2[g2->nSpans];
uint lb2 = 0;
for(;;)
{
for(; b1 < e1 && b1->to != s1; ++b1)
{
lb1 = b1->ub;
}
for(; b2 < e2 && b2->to != s2; ++b2)
{
lb2 = b2->ub;
}
if(b1 == e1)
{
return b2 == e2;
}
if(b2 == e2){
return false;
}
if(lb1 != lb2 || b1->ub != b2->ub)
{
return false;
}
++b1; ++b2;
}
}
class BitMap {
public:
static BitMap *first;
Go *go;
State *on;
BitMap *next;
uint i;
uchar m;
public:
static BitMap *find(Go*, State*);
static BitMap *find(State*);
static void gen(std::ostream&, uint, uint);
static void stats();
BitMap(Go*, State*);
};
BitMap *BitMap::first = NULL;
BitMap::BitMap(Go *g, State *x) : go(g), on(x), next(first)
{
first = this;
}
BitMap *BitMap::find(Go *g, State *x)
{
for(BitMap *b = first; b; b = b->next)
{
if(matches(b->go, b->on, g, x))
{
return b;
}
}
return new BitMap(g, x);
}
BitMap *BitMap::find(State *x)
{
for(BitMap *b = first; b; b = b->next)
{
if(b->on == x)
{
return b;
}
}
return NULL;
}
void BitMap::gen(std::ostream &o, uint lb, uint ub)
{
BitMap *b = first;
if(b)
{
o << "\tstatic unsigned char yybm[] = {";
uint n = ub - lb;
uchar *bm = new uchar[n];
memset(bm, 0, n);
for(uint i = 0; b; i += n)
{
for(uchar m = 0x80; b && m; b = b->next, m >>= 1)
{
b->i = i; b->m = m;
doGen(b->go, b->on, bm-lb, m);
}
for(uint j = 0; j < n; ++j)
{
if(j%8 == 0)
{
o << "\n\t"; ++oline;
}
o << std::setw(3) << (uint) bm[j] << ", ";
}
}
o << "\n\t};\n";
oline += 2;
}
}
void BitMap::stats()
{
uint n = 0;
for(BitMap *b = first; b; b = b->next)
{
prt(std::cerr, b->go, b->on); std::cerr << std::endl;
++n;
}
std::cerr << n << " bitmaps\n";
first = NULL;
}
void genGoTo(std::ostream &o, State *from, State *to, bool & readCh)
{
if (readCh && from->label + 1 != to->label)
{
o << "\tyych = *YYCURSOR;\n";
readCh = false;
}
o << "\tgoto yy" << to->label << ";\n";
++oline;
}
void genIf(std::ostream &o, char *cmp, uint v, bool &readCh)
{
if (readCh)
{
o << "\tif((yych = *YYCURSOR) ";
readCh = false;
}
else
{
o << "\tif(yych ";
}
o << cmp << " '";
prtCh(o, v);
o << "')";
}
void indent(std::ostream &o, uint i)
{
while(i-- > 0)
{
o << "\t";
}
}
static void need(std::ostream &o, uint n, bool & readCh)
{
if(n == 1)
{
o << "\tif(YYLIMIT == YYCURSOR) YYFILL(1);\n";
++oline;
}
else
{
o << "\tif((YYLIMIT - YYCURSOR) < " << n << ") YYFILL(" << n << ");\n";
++oline;
}
o << "\tyych = *YYCURSOR;\n";
readCh = false;
++oline;
}
void Match::emit(std::ostream &o, bool &readCh)
{
if (state->link)
{
o << "\t++YYCURSOR;\n";
}
else if (!readAhead())
{
/* do not read next char if match */
o << "\t++YYCURSOR;\n";
readCh = true;
}
else
{
o << "\tyych = *++YYCURSOR;\n";
readCh = false;
}
++oline;
if(state->link)
{
++oline;
need(o, state->depth, readCh);
}
}
void Enter::emit(std::ostream &o, bool &readCh)
{
if(state->link){
o << "\t++YYCURSOR;\n";
o << "yy" << label << ":\n";
oline += 2;
need(o, state->depth, readCh);
} else {
/* we shouldn't need 'rule-following' protection here */
o << "\tyych = *++YYCURSOR;\n";
o << "yy" << label << ":\n";
oline += 2;
readCh = false;
}
}
void Save::emit(std::ostream &o, bool &readCh)
{
o << "\tyyaccept = " << selector << ";\n";
++oline;
if(state->link){
o << "\tYYMARKER = ++YYCURSOR;\n";
++oline;
need(o, state->depth, readCh);
} else {
o << "\tyych = *(YYMARKER = ++YYCURSOR);\n";
++oline;
readCh = false;
}
}
Move::Move(State *s) : Action(s) {
;
}
void Move::emit(std::ostream &o, bool &readCh){
;
}
Accept::Accept(State *x, uint n, uint *s, State **r)
: Action(x), nRules(n), saves(s), rules(r){
;
}
void Accept::emit(std::ostream &o, bool &readCh)
{
bool first = true;
for(uint i = 0; i < nRules; ++i)
if(saves[i] != ~0u)
{
if(first)
{
first = false;
o << "\tYYCURSOR = YYMARKER;\n";
o << "\tswitch(yyaccept){\n";
oline += 2;
}
o << "\tcase " << saves[i] << ":";
genGoTo(o, state, rules[i], readCh);
}
if(!first)
{
o << "\t}\n";
++oline;
}
}
Rule::Rule(State *s, RuleOp *r) : Action(s), rule(r) {
;
}
void Rule::emit(std::ostream &o, bool &readCh)
{
uint back = rule->ctx->fixedLength();
if(back != ~0u && back > 0u) {
o << "\tYYCURSOR -= " << back << ";";
}
o << "\n";
++oline;
line_source(rule->code->line, o);
o << rule->code->text;
// not sure if we need this or not. oline += std::count(rule->code->text, rule->code->text + ::strlen(rule->code->text), '\n');
o << "\n";
++oline;
o << "#line " << ++oline << " \"" << outputFileName << "\"\n";
// o << "\n#line " << rule->code->line
// << "\n\t" << rule->code->text << "\n";
}
void doLinear(std::ostream &o, uint i, Span *s, uint n, State *from, State *next, bool &readCh)
{
for(;;)
{
State *bg = s[0].to;
while(n >= 3 && s[2].to == bg && (s[1].ub - s[0].ub) == 1)
{
if(s[1].to == next && n == 3)
{
indent(o, i); genIf(o, "!=", s[0].ub, readCh); genGoTo(o, from, bg, readCh);
indent(o, i); genGoTo(o, from, next, readCh);
return;
}
else
{
indent(o, i); genIf(o, "==", s[0].ub, readCh); genGoTo(o, from, s[1].to, readCh);
}
n -= 2; s += 2;
}
if(n == 1)
{
// if(bg != next){
indent(o, i); genGoTo(o, from, s[0].to, readCh);
// }
return;
}
else if(n == 2 && bg == next)
{
indent(o, i); genIf(o, ">=", s[0].ub, readCh); genGoTo(o, from, s[1].to, readCh);
indent(o, i); genGoTo(o, from, next, readCh);
return;
}
else
{
indent(o, i); genIf(o, "<=", s[0].ub - 1, readCh); genGoTo(o, from, bg, readCh);
n -= 1; s += 1;
}
}
indent(o, i); genGoTo(o, from, next, readCh);
}
void Go::genLinear(std::ostream &o, State *from, State *next, bool &readCh)
{
doLinear(o, 0, span, nSpans, from, next, readCh);
}
void genCases(std::ostream &o, uint lb, Span *s)
{
if(lb < s->ub)
{
for(;;)
{
o << "\tcase '"; prtCh(o, lb); o << "':";
if(++lb == s->ub)
{
break;
}
o << "\n";
++oline;
}
}
}
void Go::genSwitch(std::ostream &o, State *from, State *next, bool &readCh)
{
if(nSpans <= 2){
genLinear(o, from, next, readCh);
}
else
{
State *def = span[nSpans-1].to;
Span **sP = new Span*[nSpans-1], **r, **s, **t;
t = &sP[0];
for(uint i = 0; i < nSpans; ++i)
{
if(span[i].to != def)
{
*(t++) = &span[i];
}
}
if (readCh)
{
o << "\tswitch((yych = *YYCURSOR)) {\n";
readCh =false;
}
else
{
o << "\tswitch(yych){\n";
}
++oline;
while(t != &sP[0])
{
r = s = &sP[0];
if(*s == &span[0])
genCases(o, 0, *s);
else
genCases(o, (*s)[-1].ub, *s);
State *to = (*s)->to;
while(++s < t)
{
if((*s)->to == to)
{
genCases(o, (*s)[-1].ub, *s);
}
else
{
*(r++) = *s;
}
}
genGoTo(o, from, to, readCh);
t = r;
}
o << "\tdefault:";
genGoTo(o, from, def, readCh);
o << "\t}\n";
++oline;
delete [] sP;
}
}
void doBinary(std::ostream &o, uint i, Span *s, uint n, State *from, State *next, bool &readCh)
{
if(n <= 4)
{
doLinear(o, i, s, n, from, next, readCh);
}
else
{
uint h = n/2;
indent(o, i); genIf(o, "<=", s[h-1].ub - 1, readCh); o << "{\n";
++oline;
doBinary(o, i+1, &s[0], h, from, next, readCh);
indent(o, i); o << "\t} else {\n";
++oline;
doBinary(o, i+1, &s[h], n - h, from, next, readCh);
indent(o, i); o << "\t}\n";
++oline;
}
}
void Go::genBinary(std::ostream &o, State *from, State *next, bool &readCh)
{
doBinary(o, 0, span, nSpans, from, next, readCh);
}
void Go::genBase(std::ostream &o, State *from, State *next, bool &readCh)
{
if(nSpans == 0)
{
return;
}
if(!sFlag)
{
genSwitch(o, from, next, readCh);
return;
}
if(nSpans > 8)
{
Span *bot = &span[0], *top = &span[nSpans-1];
uint util;
if(bot[0].to == top[0].to)
{
util = (top[-1].ub - bot[0].ub)/(nSpans - 2);
}
else
{
if(bot[0].ub > (top[0].ub - top[-1].ub))
{
util = (top[0].ub - bot[0].ub)/(nSpans - 1);
}
else
{
util = top[-1].ub/(nSpans - 1);
}
}
if(util <= 2)
{
genSwitch(o, from, next, readCh);
return;
}
}
if(nSpans > 5)
{
genBinary(o, from, next, readCh);
}
else
{
genLinear(o, from, next, readCh);
}
}
void Go::genGoto(std::ostream &o, State *from, State *next, bool &readCh)
{
if(bFlag)
{
for(uint i = 0; i < nSpans; ++i)
{
State *to = span[i].to;
if(to && to->isBase)
{
BitMap *b = BitMap::find(to);
if(b && matches(b->go, b->on, this, to))
{
Go go;
go.span = new Span[nSpans];
go.unmap(this, to);
o << "\tif(yybm[" << b->i << "+";
if (readCh)
{
o << "(yych = *YYCURSOR)";
}
else
{
o << "yych";
}
o << "] & " << (uint) b->m << ")";
genGoTo(o, from, to, readCh);
go.genBase(o, from, next, readCh);
delete [] go.span;
return;
}
}
}
}
genBase(o, from, next, readCh);
}
void State::emit(std::ostream &o, bool &readCh){
o << "yy" << label << ":";
/* o << "\nfprintf(stderr, \"<" << label << ">\");\n";*/
action->emit(o, readCh);
}
uint merge(Span *x0, State *fg, State *bg)
{
Span *x = x0, *f = fg->go.span, *b = bg->go.span;
uint nf = fg->go.nSpans, nb = bg->go.nSpans;
State *prev = NULL, *to;
// NB: we assume both spans are for same range
for(;;)
{
if(f->ub == b->ub)
{
to = f->to == b->to? bg : f->to;
if(to == prev){
--x;
}
else
{
x->to = prev = to;
}
x->ub = f->ub;
++x; ++f; --nf; ++b; --nb;
if(nf == 0 && nb == 0)
{
return x - x0;
}
}
while(f->ub < b->ub)
{
to = f->to == b->to? bg : f->to;
if(to == prev)
{
--x;
}
else
{
x->to = prev = to;
}
x->ub = f->ub;
++x; ++f; --nf;
}
while(b->ub < f->ub)
{
to = b->to == f->to? bg : f->to;
if(to == prev)
{
--x;
}
else
{
x->to = prev = to;
}
x->ub = b->ub;
++x; ++b; --nb;
}
}
}
const uint cInfinity = ~0;
class SCC {
public:
State **top, **stk;
public:
SCC(uint);
~SCC();
void traverse(State*);
};
SCC::SCC(uint size){
top = stk = new State*[size];
}
SCC::~SCC(){
delete [] stk;
}
void SCC::traverse(State *x)
{
*top = x;
uint k = ++top - stk;
x->depth = k;
for(uint i = 0; i < x->go.nSpans; ++i)
{
State *y = x->go.span[i].to;
if(y)
{
if(y->depth == 0)
{
traverse(y);
}
if(y->depth < x->depth)
{
x->depth = y->depth;
}
}
}
if(x->depth == k)
{
do
{
(*--top)->depth = cInfinity;
(*top)->link = x;
} while(*top != x);
}
}
uint maxDist(State *s)
{
uint mm = 0;
for(uint i = 0; i < s->go.nSpans; ++i)
{
State *t = s->go.span[i].to;
if(t)
{
uint m = 1;
if(!t->link)
{
m += maxDist(t);
}
if(m > mm)
{
mm = m;
}
}
}
return mm;
}
void calcDepth(State *head)
{
State *t;
for(State *s = head; s; s = s->next)
{
if(s->link == s){
for(uint i = 0; i < s->go.nSpans; ++i)
{
t = s->go.span[i].to;
if(t && t->link == s)
{
goto inSCC;
}
}
s->link = NULL;
}else
{
inSCC:
s->depth = maxDist(s);
}
}
}
void DFA::findSCCs()
{
SCC scc(nStates);
State *s;
for(s = head; s; s = s->next)
{
s->depth = 0;
s->link = NULL;
}
for(s = head; s; s = s->next)
{
if(!s->depth)
{
scc.traverse(s);
}
}
calcDepth(head);
}
void DFA::split(State *s)
{
State *move = new State;
(void) new Move(move);
addState(&s->next, move);
move->link = s->link;
move->rule = s->rule;
move->go = s->go;
s->rule = NULL;
s->go.nSpans = 1;
s->go.span = new Span[1];
s->go.span[0].ub = ubChar;
s->go.span[0].to = move;
}
void DFA::emit(std::ostream &o)
{
static uint label = 0;
State *s;
uint i;
findSCCs();
head->link = head;
head->depth = maxDist(head);
uint nRules = 0;
for(s = head; s; s = s->next)
{
if(s->rule && s->rule->accept >= nRules)
{
nRules = s->rule->accept + 1;
}
}
uint nSaves = 0;
uint *saves = new uint[nRules];
memset(saves, ~0, (nRules)*sizeof(*saves));
// mark backtracking points
for(s = head; s; s = s->next)
{
RuleOp *ignore = NULL;
if(s->rule)
{
for(i = 0; i < s->go.nSpans; ++i)
{
if(s->go.span[i].to && !s->go.span[i].to->rule){
delete s->action;
if(saves[s->rule->accept] == ~0u)
{
saves[s->rule->accept] = nSaves++;
}
(void) new Save(s, saves[s->rule->accept]);
continue;
}
}
ignore = s->rule;
}
}
// insert actions
State **rules = new State*[nRules];
memset(rules, 0, (nRules)*sizeof(*rules));
State *accept = NULL;
for(s = head; s; s = s->next)
{
State *ow;
if(!s->rule)
{
ow = accept;
}
else
{
if(!rules[s->rule->accept])
{
State *n = new State;
(void) new Rule(n, s->rule);
rules[s->rule->accept] = n;
addState(&s->next, n);
}
ow = rules[s->rule->accept];
}
for(i = 0; i < s->go.nSpans; ++i)
if(!s->go.span[i].to)
{
if(!ow)
{
ow = accept = new State;
(void) new Accept(accept, nRules, saves, rules);
addState(&s->next, accept);
}
s->go.span[i].to = ow;
}
}
// split ``base'' states into two parts
for(s = head; s; s = s->next)
{
s->isBase = false;
if(s->link)
{
for(i = 0; i < s->go.nSpans; ++i)
{
if(s->go.span[i].to == s){
s->isBase = true;
split(s);
if(bFlag)
{
BitMap::find(&s->next->go, s);
}
s = s->next;
break;
}
}
}
}
// find ``base'' state, if possible
Span *span = new Span[ubChar - lbChar];
for(s = head; s; s = s->next)
{
if(!s->link)
{
for(i = 0; i < s->go.nSpans; ++i)
{
State *to = s->go.span[i].to;
if(to && to->isBase)
{
to = to->go.span[0].to;
uint nSpans = merge(span, s, to);
if(nSpans < s->go.nSpans)
{
delete [] s->go.span;
s->go.nSpans = nSpans;
s->go.span = new Span[nSpans];
memcpy(s->go.span, span, nSpans*sizeof(Span));
}
break;
}
}
}
}
delete [] span;
delete head->action;
++oline;
o << "\n#line " << ++oline << " \"" << outputFileName << "\"\n";
o << "{\n\tYYCTYPE yych;\n\tunsigned int yyaccept;\n";
oline += 3;
if(bFlag)
{
BitMap::gen(o, lbChar, ubChar);
}
o << "\tgoto yy" << label << ";\n";
++oline;
(void) new Enter(head, label++);
for(s = head; s; s = s->next)
{
s->label = label++;
}
for(s = head; s; s = s->next)
{
bool readCh = false;
s->emit(o, readCh);
s->go.genGoto(o, s, s->next, readCh);
}
o << "}\n";
++oline;
BitMap::first = NULL;
delete [] saves;
delete [] rules;
}

118
tools/re2c/config.h.in Normal file
View file

@ -0,0 +1,118 @@
/* config.h.in. Generated from configure.in by autoheader. */
/* Define to 1 if you have the `getpagesize' function. */
#undef HAVE_GETPAGESIZE
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
to 0 otherwise. */
#undef HAVE_MALLOC
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
/* Define to 1 if you have the `memset' function. */
#undef HAVE_MEMSET
/* Define to 1 if you have a working `mmap' system call. */
#undef HAVE_MMAP
/* Define to 1 if you have the `munmap' function. */
#undef HAVE_MUNMAP
/* Define to 1 if stdbool.h conforms to C99. */
#undef HAVE_STDBOOL_H
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdlib.h> header file. */
#undef HAVE_STDLIB_H
/* Define to 1 if you have the `strdup' function. */
#undef HAVE_STRDUP
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
/* Define to 1 if you have the <string.h> header file. */
#undef HAVE_STRING_H
/* Define to 1 if you have the <sys/stat.h> header file. */
#undef HAVE_SYS_STAT_H
/* Define to 1 if you have the <sys/types.h> header file. */
#undef HAVE_SYS_TYPES_H
/* Define to 1 if you have uchar type definition */
#undef HAVE_UCHAR
/* Define to 1 if you have uint type definition */
#undef HAVE_UINT
/* Define to 1 if you have ulong type definition */
#undef HAVE_ULONG
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to 1 if you have ushort type definition */
#undef HAVE_USHORT
/* Define to 1 if the system has the type `_Bool'. */
#undef HAVE__BOOL
/* Name of package */
#undef PACKAGE
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#undef PACKAGE_NAME
/* Define to the full name and version of this package. */
#undef PACKAGE_STRING
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the version of this package. */
#undef PACKAGE_VERSION
/* The size of a `char', as computed by sizeof. */
#undef SIZEOF_CHAR
/* The size of a `int', as computed by sizeof. */
#undef SIZEOF_INT
/* The size of a `long', as computed by sizeof. */
#undef SIZEOF_LONG
/* The size of a `short', as computed by sizeof. */
#undef SIZEOF_SHORT
/* Define to 1 if you have the ANSI C header files. */
#undef STDC_HEADERS
/* Version number of package */
#undef VERSION
/* Define to empty if `const' does not conform to ANSI C. */
#undef const
/* Define as `__inline' if that's what the C compiler calls it, or to nothing
if it is not supported. */
#undef inline
/* Define to rpl_malloc if the replacement function should be used. */
#undef malloc
/* Define to `unsigned' if <sys/types.h> does not define. */
#undef size_t
/* Define to empty if the keyword `volatile' does not work. Warning: valid
code using `volatile' can become incorrect without. Disable with care. */
#undef volatile

7415
tools/re2c/configure vendored Normal file
View file

File diff suppressed because it is too large Load diff

59
tools/re2c/configure.in Normal file
View file

@ -0,0 +1,59 @@
AC_PREREQ([2.57])
AC_INIT(Makefile.am, 0.9.3, re2c-general@lists.sourceforge.net)
AM_INIT_AUTOMAKE(re2c, $PACKAGE_VERSION)
AC_CONFIG_SRCDIR(actions.cc)
AM_CONFIG_HEADER(config.h)
AC_SUBST(PACKAGE_VERSION)
AC_SUBST(PACKAGE_NAME)
AC_SUBST(PACKAGE_TARNAME)
# Checks for programs.
AC_PROG_YACC
AC_PROG_CXX
AC_PROG_CC
AC_PROG_INSTALL
# Checks for libraries.
# Checks for header files.
AC_HEADER_STDC
# We only rely on standard C/C++ headers. If we ever have a pressing need
# for something non-standard, we should add a check for it here.
# AC_CHECK_HEADERS([stddef.h stdlib.h string.h])
# Checks for typedefs, structures, and compiler characteristics.
AC_HEADER_STDBOOL
AC_C_CONST
AC_C_INLINE
AC_TYPE_SIZE_T
AC_C_VOLATILE
# Checks for library functions.
AC_FUNC_MALLOC
AC_FUNC_MEMCMP
AC_FUNC_MMAP
AC_CHECK_FUNCS([memset munmap strdup])
AC_CHECK_SIZEOF([char])
AC_CHECK_SIZEOF([short])
AC_CHECK_SIZEOF([int])
AC_CHECK_SIZEOF([long])
AC_CHECK_TYPE([uchar], [
AC_DEFINE([HAVE_UCHAR], [1], [Define to 1 if you have uchar type definition])
])
AC_CHECK_TYPE([ushort], [
AC_DEFINE([HAVE_USHORT], [1], [Define to 1 if you have ushort type definition])
])
AC_CHECK_TYPE([uint], [
AC_DEFINE([HAVE_UINT], [1], [Define to 1 if you have uint type definition])
])
AC_CHECK_TYPE([ulong], [
AC_DEFINE([HAVE_ULONG], [1], [Define to 1 if you have ulong type definition])
])
AC_CONFIG_FILES([Makefile README makerpm re2c.1 re2c.spec])
AC_OUTPUT_COMMANDS([chmod +x makerpm])
AC_OUTPUT

118
tools/re2c/configwin.h Normal file
View file

@ -0,0 +1,118 @@
/* config.h.in. Generated from configure.in by autoheader. */
/* Define to 1 if you have the `getpagesize' function. */
#undef HAVE_GETPAGESIZE
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
to 0 otherwise. */
#undef HAVE_MALLOC
#include <sys/types.h>
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define to 1 if you have the `memset' function. */
#define HAVE_MEMSET 1
/* Define to 1 if you have a working `mmap' system call. */
#undef HAVE_MMAP
/* Define to 1 if you have the `munmap' function. */
#undef HAVE_MUNMAP
/* Define to 1 if stdbool.h conforms to C99. */
#undef HAVE_STDBOOL_H
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the `strdup' function. */
#define HAVE_STRDUP 1
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have uchar type definition */
#undef HAVE_UCHAR
/* Define to 1 if you have uint type definition */
#undef HAVE_UINT
/* Define to 1 if you have ulong type definition */
#undef HAVE_ULONG
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to 1 if you have ushort type definition */
#undef HAVE_USHORT
/* Define to 1 if the system has the type `_Bool'. */
#undef HAVE__BOOL
/* Name of package */
#undef PACKAGE
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#define PACKAGE_NAME "re2c"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "re2c 0.9.3"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "re2c"
/* Define to the version of this package. */
#define PACKAGE_VERSION "0.9.3"
/* The size of a `char', as computed by sizeof. */
#define SIZEOF_CHAR 1
/* The size of a `int', as computed by sizeof. */
#define SIZEOF_INT 4
/* The size of a `long', as computed by sizeof. */
#define SIZEOF_LONG 4
/* The size of a `short', as computed by sizeof. */
#define SIZEOF_SHORT 2
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Version number of package */
#define VERSION "0.9.3"
/* Define to empty if `const' does not conform to ANSI C. */
//#undef const
/* Define as `__inline' if that's what the C compiler calls it, or to nothing
if it is not supported. */
//#undef inline
/* Define to rpl_malloc if the replacement function should be used. */
//#undef malloc
/* Define to `unsigned' if <sys/types.h> does not define. */
//#undef size_t
/* Define to empty if the keyword `volatile' does not work. Warning: valid
code using `volatile' can become incorrect without. Disable with care. */
//#undef volatile

226
tools/re2c/dfa.cc Normal file
View file

@ -0,0 +1,226 @@
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include "globals.h"
#include "substr.h"
#include "dfa.h"
inline char octCh(uint c){
return '0' + c%8;
}
void prtCh(std::ostream &o, uchar c){
uchar oc = talx[c];
switch(oc){
case '\'': o << "\\'"; break;
case '\n': o << "\\n"; break;
case '\t': o << "\\t"; break;
case '\v': o << "\\v"; break;
case '\b': o << "\\b"; break;
case '\r': o << "\\r"; break;
case '\f': o << "\\f"; break;
case '\a': o << "\\a"; break;
case '\\': o << "\\\\"; break;
default:
if(isprint(oc))
o << (char) oc;
else
o << '\\' << octCh(c/64) << octCh(c/8) << octCh(c);
}
}
void printSpan(std::ostream &o, uint lb, uint ub){
if(lb > ub)
o << "*";
o << "[";
if((ub - lb) == 1){
prtCh(o, lb);
} else {
prtCh(o, lb);
o << "-";
prtCh(o, ub-1);
}
o << "]";
}
uint Span::show(std::ostream &o, uint lb){
if(to){
printSpan(o, lb, ub);
o << " " << to->label << "; ";
}
return ub;
}
std::ostream& operator<<(std::ostream &o, const State &s){
o << "state " << s.label;
if(s.rule)
o << " accepts " << s.rule->accept;
o << "\n";
++oline;
uint lb = 0;
for(uint i = 0; i < s.go.nSpans; ++i)
lb = s.go.span[i].show(o, lb);
return o;
}
std::ostream& operator<<(std::ostream &o, const DFA &dfa){
for(State *s = dfa.head; s; s = s->next)
{
o << s << "\n\n";
++oline;
}
return o;
}
State::State() : rule(NULL), link(NULL), kCount(0), kernel(NULL), action(NULL) {
go.nSpans = 0;
go.span = NULL;
}
State::~State(){
delete [] kernel;
delete [] go.span;
}
static Ins **closure(Ins **cP, Ins *i){
while(!isMarked(i)){
mark(i);
*(cP++) = i;
if(i->i.tag == FORK){
cP = closure(cP, i + 1);
i = (Ins*) i->i.link;
} else if(i->i.tag == GOTO){
i = (Ins*) i->i.link;
} else
break;
}
return cP;
}
struct GoTo {
Char ch;
void *to;
};
DFA::DFA(Ins *ins, uint ni, uint lb, uint ub, Char *rep)
: lbChar(lb), ubChar(ub) {
Ins **work = new Ins*[ni+1];
uint nc = ub - lb;
GoTo *goTo = new GoTo[nc];
Span *span = new Span[nc];
memset((char*) goTo, 0, nc*sizeof(GoTo));
tail = &head;
head = NULL;
nStates = 0;
toDo = NULL;
findState(work, closure(work, &ins[0]) - work);
while(toDo){
State *s = toDo;
toDo = s->link;
Ins **cP, **iP, *i;
uint nGoTos = 0;
uint j;
s->rule = NULL;
for(iP = s->kernel; (i = *iP); ++iP){
if(i->i.tag == CHAR){
for(Ins *j = i + 1; j < (Ins*) i->i.link; ++j){
if(!(j->c.link = goTo[j->c.value - lb].to))
goTo[nGoTos++].ch = j->c.value;
goTo[j->c.value - lb].to = j;
}
} else if(i->i.tag == TERM){
if(!s->rule || ((RuleOp*) i->i.link)->accept < s->rule->accept)
s->rule = (RuleOp*) i->i.link;
}
}
for(j = 0; j < nGoTos; ++j){
GoTo *go = &goTo[goTo[j].ch - lb];
i = (Ins*) go->to;
for(cP = work; i; i = (Ins*) i->c.link)
cP = closure(cP, i + i->c.bump);
go->to = findState(work, cP - work);
}
s->go.nSpans = 0;
for(j = 0; j < nc;){
State *to = (State*) goTo[rep[j]].to;
while(++j < nc && goTo[rep[j]].to == to);
span[s->go.nSpans].ub = lb + j;
span[s->go.nSpans].to = to;
s->go.nSpans++;
}
for(j = nGoTos; j-- > 0;)
goTo[goTo[j].ch - lb].to = NULL;
s->go.span = new Span[s->go.nSpans];
memcpy((char*) s->go.span, (char*) span, s->go.nSpans*sizeof(Span));
(void) new Match(s);
}
delete [] work;
delete [] goTo;
delete [] span;
}
DFA::~DFA(){
State *s;
while((s = head)){
head = s->next;
delete s;
}
}
void DFA::addState(State **a, State *s){
s->label = nStates++;
s->next = *a;
*a = s;
if(a == tail)
tail = &s->next;
}
State *DFA::findState(Ins **kernel, uint kCount){
Ins **cP, **iP, *i;
State *s;
kernel[kCount] = NULL;
cP = kernel;
for(iP = kernel; (i = *iP); ++iP){
if(i->i.tag == CHAR || i->i.tag == TERM){
*cP++ = i;
} else {
unmark(i);
}
}
kCount = cP - kernel;
kernel[kCount] = NULL;
for(s = head; s; s = s->next){
if(s->kCount == kCount){
for(iP = s->kernel; (i = *iP); ++iP)
if(!isMarked(i))
goto nextState;
goto unmarkAll;
}
nextState:;
}
s = new State;
addState(tail, s);
s->kCount = kCount;
s->kernel = new Ins*[kCount+1];
memcpy(s->kernel, kernel, (kCount+1)*sizeof(Ins*));
s->link = toDo;
toDo = s;
unmarkAll:
for(iP = kernel; (i = *iP); ++iP)
unmark(i);
return s;
}

174
tools/re2c/dfa.h Normal file
View file

@ -0,0 +1,174 @@
/* $Id: dfa.h,v 1.5 2004/05/13 02:58:17 nuffer Exp $ */
#ifndef _dfa_h
#define _dfa_h
#include <iosfwd>
#include "re.h"
extern void prtCh(std::ostream&, uchar);
extern void printSpan(std::ostream&, uint, uint);
class DFA;
class State;
class Action {
public:
State *state;
public:
Action(State*);
virtual void emit(std::ostream&, bool&) = 0;
virtual bool isRule() const;
virtual bool isMatch() const;
virtual bool readAhead() const;
};
class Match: public Action {
public:
Match(State*);
void emit(std::ostream&, bool&);
bool isMatch() const;
};
class Enter: public Action {
public:
uint label;
public:
Enter(State*, uint);
void emit(std::ostream&, bool&);
};
class Save: public Match {
public:
uint selector;
public:
Save(State*, uint);
void emit(std::ostream&, bool&);
bool isMatch() const;
};
class Move: public Action {
public:
Move(State*);
void emit(std::ostream&, bool&);
};
class Accept: public Action {
public:
uint nRules;
uint *saves;
State **rules;
public:
Accept(State*, uint, uint*, State**);
void emit(std::ostream&, bool&);
};
class Rule: public Action {
public:
RuleOp *rule;
public:
Rule(State*, RuleOp*);
void emit(std::ostream&, bool&);
bool isRule() const;
};
class Span {
public:
uint ub;
State *to;
public:
uint show(std::ostream&, uint);
};
class Go {
public:
uint nSpans;
Span *span;
public:
void genGoto(std::ostream&, State *from, State*, bool &readCh);
void genBase(std::ostream&, State *from, State*, bool &readCh);
void genLinear(std::ostream&, State *from, State*, bool &readCh);
void genBinary(std::ostream&, State *from, State*, bool &readCh);
void genSwitch(std::ostream&, State *from, State*, bool &readCh);
void compact();
void unmap(Go*, State*);
};
class State {
public:
uint label;
RuleOp *rule;
State *next;
State *link;
uint depth; // for finding SCCs
uint kCount;
Ins **kernel;
bool isBase:1;
Go go;
Action *action;
public:
State();
~State();
void emit(std::ostream&, bool&);
friend std::ostream& operator<<(std::ostream&, const State&);
friend std::ostream& operator<<(std::ostream&, const State*);
};
class DFA {
public:
uint lbChar;
uint ubChar;
uint nStates;
State *head, **tail;
State *toDo;
public:
DFA(Ins*, uint, uint, uint, Char*);
~DFA();
void addState(State**, State*);
State *findState(Ins**, uint);
void split(State*);
void findSCCs();
void emit(std::ostream&);
friend std::ostream& operator<<(std::ostream&, const DFA&);
friend std::ostream& operator<<(std::ostream&, const DFA*);
};
inline Action::Action(State *s) : state(s) {
s->action = this;
}
inline bool Action::isRule() const
{ return false; }
inline bool Action::isMatch() const
{ return false; }
inline bool Action::readAhead() const
{ return !isMatch() || (state && state->next && state->next->action && !state->next->action->isRule()); }
inline Match::Match(State *s) : Action(s)
{ }
inline bool Match::isMatch() const
{ return true; }
inline Enter::Enter(State *s, uint l) : Action(s), label(l)
{ }
inline Save::Save(State *s, uint i) : Match(s), selector(i)
{ }
inline bool Save::isMatch() const
{ return false; }
inline bool Rule::isRule() const
{ return true; }
inline std::ostream& operator<<(std::ostream &o, const State *s)
{ return o << *s; }
inline std::ostream& operator<<(std::ostream &o, const DFA *dfa)
{ return o << *dfa; }
#endif

5249
tools/re2c/doc/loplas.ps Normal file
View file

File diff suppressed because it is too large Load diff

48
tools/re2c/doc/sample.bib Normal file
View file

@ -0,0 +1,48 @@
@Article{Bumbulis94,
author = {Peter Bumbulis and Donald D. Cowan},
title = {RE2C -- A More Versatile Scanner Generator},
journal = "ACM Letters on Programming Languages and Systems",
volume = 2,
number = "1--4",
year = 1994,
abstract = {
It is usually claimed that lexical analysis routines are still coded by
hand, despite the widespread availability of scanner generators, for
efficiency reasons. While efficiency is a consideration, there exist
freely available scanner generators such as GLA \cite{Gray88} that can
generate scanners that are faster than most hand-coded ones. However,
most generated scanners are tailored for a particular environment, and
retargetting these scanners to other environments, if possible, is
usually complex enough to make a hand-coded scanner more appealing. In
this paper we describe RE2C, a scanner generator that not only generates
scanners which are faster (and usually smaller) than those produced by
any other scanner generator known to the authors, including GLA, but
also adapt easily to any environment.
}
}
@Article{Gray88,
author = {Robert W. Gray},
title = {{$\gamma$-GLA} - {A} Generator for Lexical Analyzers That
Programmers Can Use},
journal = {USENIX Conference Proceedings},
year = {1988},
month = {June},
pages = {147-160},
abstract = {Writing an efficient lexical analyzer for even a simple
language is not a trivial task, and should not be done by hand. We
describe GLA, a tool that generates very efficient scanners. These
scanners do not use the conventional transition matrix, but instead
use a few 128 element vectors. Scanning time is only slightly
greater than the absolute minimum --- the time it takes to look at
each character in a file. The GLA language allows simple, concise
specification of scanners. Augmenting regular expressions with
auxiliary scanners easily handles nasty problems such as C comments
and C literal constants. We formalize the connection between token
scanning and token processing by associating a processor with
appropriate patterns. A library of canned descriptions simplifies the
specification of commonly used language pieces --- such as,
C\_IDENTIFIERS, C\_STRINGS, PASCAL\_COMMENTS, etc. Finally, carefully
tuned lexical analysis support modules are provided for error
handling, input buffering, storing identifiers in hash tables and
manipulating denotations.}
}

26
tools/re2c/examples/basemmap.c Normal file
View file

@ -0,0 +1,26 @@
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <string.h>
#ifndef MAP_NORESERVE
#define MAP_NORESERVE 0
#endif
volatile char ch;
main(){
struct stat statbuf;
uchar *buf;
fstat(0, &statbuf);
buf = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED|MAP_NORESERVE,
0, 0);
if(buf != (uchar*)(-1)){
uchar *cur, *lim = &buf[statbuf.st_size];
for(cur = buf; buf != lim; ++cur){
ch = *cur;
}
munmap(buf, statbuf.st_size);
}
}

272
tools/re2c/examples/c.re Normal file
View file

@ -0,0 +1,272 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof) {
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
close(in.fd);
}

267
tools/re2c/examples/cmmap.re Normal file
View file

@ -0,0 +1,267 @@
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int unint;
typedef unsigned char uchar;
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
uchar *tok, *ptr, *cur, *pos, *lim, *eof;
unint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
unint cnt = s->lim - s->tok;
uchar *buf = malloc((cnt + 1)*sizeof(uchar));
memcpy(buf, s->tok, cnt);
cursor = &buf[cursor - s->tok];
s->pos = &buf[s->pos - s->tok];
s->ptr = &buf[s->ptr - s->tok];
s->lim = &buf[cnt];
s->eof = s->lim; *(s->eof)++ = '\n';
s->tok = buf;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
#ifndef MAP_NORESERVE
#define MAP_NORESERVE 0
#endif
main(){
Scanner in;
struct stat statbuf;
uchar *buf;
fstat(0, &statbuf);
buf = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED|MAP_NORESERVE,
0, 0);
if(buf != (uchar*)(-1)){
int t;
in.lim = &(in.cur = buf)[statbuf.st_size];
in.pos = NULL;
in.eof = NULL;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
munmap(buf, statbuf.st_size);
}
}

239
tools/re2c/examples/cnokw.re Normal file
View file

@ -0,0 +1,239 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
close(in.fd);
}

258
tools/re2c/examples/cunroll.re Normal file
View file

@ -0,0 +1,258 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
I = L|D;
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
X = any\[*/];
*/
/*!re2c
"/*" { goto comment; }
L { RET(ID); }
L I { RET(ID); }
L I I { RET(ID); }
L I I I { RET(ID); }
L I I I I { RET(ID); }
L I I I I I { RET(ID); }
L I I I I I I { RET(ID); }
L I I I I I I I { RET(ID); }
L I* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
X { goto comment; }
X X { goto comment; }
X X X { goto comment; }
X X X X { goto comment; }
X X X X X { goto comment; }
X X X X X X { goto comment; }
X X X X X X X { goto comment; }
X X X X X X X X { goto comment; }
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
close(in.fd);
}

202
tools/re2c/examples/modula.re Normal file
View file

@ -0,0 +1,202 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL {cursor = fill(s, cursor);}
#define RETURN(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
uint depth;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
digit = [0-9];
letter = [a-zA-Z];
*/
/*!re2c
"(*" { depth = 1; goto comment; }
digit + {RETURN(1);}
digit + / ".." {RETURN(1);}
[0-7] + "B" {RETURN(2);}
[0-7] + "C" {RETURN(3);}
digit [0-9A-F] * "H" {RETURN(4);}
digit + "." digit * ("E" ([+-]) ? digit +) ? {RETURN(5);}
['] (any\[\n']) * ['] | ["] (any\[\n"]) * ["] {RETURN(6);}
"#" {RETURN(7);}
"&" {RETURN(8);}
"(" {RETURN(9);}
")" {RETURN(10);}
"*" {RETURN(11);}
"+" {RETURN(12);}
"," {RETURN(13);}
"-" {RETURN(14);}
"." {RETURN(15);}
".." {RETURN(16);}
"/" {RETURN(17);}
":" {RETURN(18);}
":=" {RETURN(19);}
";" {RETURN(20);}
"<" {RETURN(21);}
"<=" {RETURN(22);}
"<>" {RETURN(23);}
"=" {RETURN(24);}
">" {RETURN(25);}
">=" {RETURN(26);}
"[" {RETURN(27);}
"]" {RETURN(28);}
"^" {RETURN(29);}
"{" {RETURN(30);}
"|" {RETURN(31);}
"}" {RETURN(32);}
"~" {RETURN(33);}
"AND" {RETURN(34);}
"ARRAY" {RETURN(35);}
"BEGIN" {RETURN(36);}
"BY" {RETURN(37);}
"CASE" {RETURN(38);}
"CONST" {RETURN(39);}
"DEFINITION" {RETURN(40);}
"DIV" {RETURN(41);}
"DO" {RETURN(42);}
"ELSE" {RETURN(43);}
"ELSIF" {RETURN(44);}
"END" {RETURN(45);}
"EXIT" {RETURN(46);}
"EXPORT" {RETURN(47);}
"FOR" {RETURN(48);}
"FROM" {RETURN(49);}
"IF" {RETURN(50);}
"IMPLEMENTATION" {RETURN(51);}
"IMPORT" {RETURN(52);}
"IN" {RETURN(53);}
"LOOP" {RETURN(54);}
"MOD" {RETURN(55);}
"MODULE" {RETURN(56);}
"NOT" {RETURN(57);}
"OF" {RETURN(58);}
"OR" {RETURN(59);}
"POINTER" {RETURN(60);}
"PROCEDURE" {RETURN(61);}
"QUALIFIED" {RETURN(62);}
"RECORD" {RETURN(63);}
"REPEAT" {RETURN(64);}
"RETURN" {RETURN(65);}
"SET" {RETURN(66);}
"THEN" {RETURN(67);}
"TO" {RETURN(68);}
"TYPE" {RETURN(69);}
"UNTIL" {RETURN(70);}
"VAR" {RETURN(71);}
"WHILE" {RETURN(72);}
"WITH" {RETURN(73);}
letter (letter | digit) * {RETURN(74);}
[ \t]+ { goto std; }
"\n"
{
if(cursor == s->eof) RETURN(0);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*)"
{
if(--depth == 0)
goto std;
else
goto comment;
}
"(*" { ++depth; goto comment; }
"\n"
{
if(cursor == s->eof) RETURN(0);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
/*
void putStr(FILE *o, char *s, uint l){
while(l-- > 0)
putc(*s++, o);
}
*/
main(){
Scanner in;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while(scan(&in)){
/*
putc('<', stdout);
putStr(stdout, (char*) in.tok, in.cur - in.tok);
putc('>', stdout);
putc('\n', stdout);
*/
}
}

1
tools/re2c/examples/rexx/README Normal file
View file

@ -0,0 +1 @@
Replacement modules for an existing REXX interpreter. Not standalone.

319
tools/re2c/examples/rexx/rexx.l Normal file
View file

@ -0,0 +1,319 @@
#include "scanio.h"
#include "scanner.h"
#define CURSOR ch
#define LOADCURSOR ch = *cursor;
#define ADVANCE cursor++;
#define BACK(n) cursor -= (n);
#define CHECK(n) if((ScanCB.lim - cursor) < (n)){cursor = ScanFill(cursor);}
#define MARK(n) ScanCB.ptr = cursor; sel = (n);
#define REVERT cursor = ScanCB.ptr;
#define MARKER sel
#define RETURN(i) {ScanCB.cur = cursor; return i;}
int ScanToken(){
uchar *cursor = ScanCB.cur;
unsigned sel;
uchar ch;
ScanCB.tok = cursor;
ScanCB.eot = NULL;
/*!re2c
all = [\000-\377];
eof = [\000];
any = all\eof;
letter = [a-z]|[A-Z];
digit = [0-9];
symchr = letter|digit|[.!?_];
const = (digit|[.])symchr*([eE][+-]?digit+)?;
simple = (symchr\(digit|[.]))(symchr\[.])*;
stem = simple [.];
symbol = symchr*;
sqstr = ['] ((any\['\n])|(['][']))* ['];
dqstr = ["] ((any\["\n])|(["]["]))* ["];
str = sqstr|dqstr;
ob = [ \t]*;
not = [\\~];
A = [aA];
B = [bB];
C = [cC];
D = [dD];
E = [eE];
F = [fF];
G = [gG];
H = [hH];
I = [iI];
J = [jJ];
K = [kK];
L = [lL];
M = [mM];
N = [nN];
O = [oO];
P = [pP];
Q = [qQ];
R = [rR];
S = [sS];
T = [tT];
U = [uU];
V = [vV];
W = [wW];
X = [xX];
Y = [yY];
Z = [zZ];
*/
scan:
/*!re2c
"\n"
{
++(ScanCB.lineNum);
ScanCB.linePos = ScanCB.pos + (cursor - ScanCB.mrk);
RETURN(SU_EOL);
}
"|" ob "|"
{ RETURN(OP_CONCAT); }
"+"
{ RETURN(OP_PLUS); }
"-"
{ RETURN(OP_MINUS); }
"*"
{ RETURN(OP_MULT); }
"/"
{ RETURN(OP_DIV); }
"%"
{ RETURN(OP_IDIV); }
"/" ob "/"
{ RETURN(OP_REMAIN); }
"*" ob "*"
{ RETURN(OP_POWER); }
"="
{ RETURN(OP_EQUAL); }
not ob "=" | "<" ob ">" | ">" ob "<"
{ RETURN(OP_EQUAL_N); }
">"
{ RETURN(OP_GT); }
"<"
{ RETURN(OP_LT); }
">" ob "=" | not ob "<"
{ RETURN(OP_GE); }
"<" ob "=" | not ob ">"
{ RETURN(OP_LE); }
"=" ob "="
{ RETURN(OP_EQUAL_EQ); }
not ob "=" ob "="
{ RETURN(OP_EQUAL_EQ_N); }
">" ob ">"
{ RETURN(OP_GT_STRICT); }
"<" ob "<"
{ RETURN(OP_LT_STRICT); }
">" ob ">" ob "=" | not ob "<" ob "<"
{ RETURN(OP_GE_STRICT); }
"<" ob "<" ob "=" | not ob ">" ob ">"
{ RETURN(OP_LE_STRICT); }
"&"
{ RETURN(OP_AND); }
"|"
{ RETURN(OP_OR); }
"&" ob "&"
{ RETURN(OP_XOR); }
not
{ RETURN(OP_NOT); }
":"
{ RETURN(SU_COLON); }
","
{ RETURN(SU_COMMA); }
"("
{ RETURN(SU_POPEN); }
")"
{ RETURN(SU_PCLOSE); }
";"
{ RETURN(SU_EOC); }
A D D R E S S
{ RETURN(RX_ADDRESS); }
A R G
{ RETURN(RX_ARG); }
C A L L
{ RETURN(RX_CALL); }
D O
{ RETURN(RX_DO); }
D R O P
{ RETURN(RX_DROP); }
E L S E
{ RETURN(RX_ELSE); }
E N D
{ RETURN(RX_END); }
E X I T
{ RETURN(RX_EXIT); }
I F
{ RETURN(RX_IF); }
I N T E R P R E T
{ RETURN(RX_INTERPRET); }
I T E R A T E
{ RETURN(RX_ITERATE); }
L E A V E
{ RETURN(RX_LEAVE); }
N O P
{ RETURN(RX_NOP); }
N U M E R I C
{ RETURN(RX_NUMERIC); }
O P T I O N S
{ RETURN(RX_OPTIONS); }
O T H E R W I S E
{ RETURN(RX_OTHERWISE); }
P A R S E
{ RETURN(RX_PARSE); }
P R O C E D U R E
{ RETURN(RX_PROCEDURE); }
P U L L
{ RETURN(RX_PULL); }
P U S H
{ RETURN(RX_PUSH); }
Q U E U E
{ RETURN(RX_QUEUE); }
R E T U R N
{ RETURN(RX_RETURN); }
S A Y
{ RETURN(RX_SAY); }
S E L E C T
{ RETURN(RX_SELECT); }
S I G N A L
{ RETURN(RX_SIGNAL); }
T H E N
{ RETURN(RX_THEN); }
T R A C E
{ RETURN(RX_TRACE); }
W H E N
{ RETURN(RX_WHEN); }
O F F
{ RETURN(RXS_OFF); }
O N
{ RETURN(RXS_ON); }
B Y
{ RETURN(RXS_BY); }
D I G I T S
{ RETURN(RXS_DIGITS); }
E N G I N E E R I N G
{ RETURN(RXS_ENGINEERING); }
E R R O R
{ RETURN(RXS_ERROR); }
E X P O S E
{ RETURN(RXS_EXPOSE); }
F A I L U R E
{ RETURN(RXS_FAILURE); }
F O R
{ RETURN(RXS_FOR); }
F O R E V E R
{ RETURN(RXS_FOREVER); }
F O R M
{ RETURN(RXS_FORM); }
F U Z Z
{ RETURN(RXS_FUZZ); }
H A L T
{ RETURN(RXS_HALT); }
L I N E I N
{ RETURN(RXS_LINEIN); }
N A M E
{ RETURN(RXS_NAME); }
N O T R E A D Y
{ RETURN(RXS_NOTREADY); }
N O V A L U E
{ RETURN(RXS_NOVALUE); }
S C I E N T I F I C
{ RETURN(RXS_SCIENTIFIC); }
S O U R C E
{ RETURN(RXS_SOURCE); }
S Y N T A X
{ RETURN(RXS_SYNTAX); }
T O
{ RETURN(RXS_TO); }
U N T I L
{ RETURN(RXS_UNTIL); }
U P P E R
{ RETURN(RXS_UPPER); }
V A L U E
{ RETURN(RXS_VALUE); }
V A R
{ RETURN(RXS_VAR); }
V E R S I O N
{ RETURN(RXS_VERSION); }
W H I L E
{ RETURN(RXS_WHILE); }
W I T H
{ RETURN(RXS_WITH); }
const
{ RETURN(SU_CONST); }
simple
{ RETURN(SU_SYMBOL); }
stem
{ RETURN(SU_SYMBOL_STEM); }
symbol
{ RETURN(SU_SYMBOL_COMPOUND); }
str
{ RETURN(SU_LITERAL); }
str [bB] / (all\symchr)
{ RETURN(SU_LITERAL_BIN); }
str [xX] / (all\symchr)
{ RETURN(SU_LITERAL_HEX); }
eof
{ RETURN(SU_EOF); }
any
{ RETURN(SU_ERROR); }
*/
}
bool StripToken(){
uchar *cursor = ScanCB.cur;
unsigned depth;
uchar ch;
bool blanks = FALSE;
ScanCB.eot = cursor;
strip:
/*!re2c
"/*"
{
depth = 1;
goto comment;
}
"\r"
{ goto strip; }
[ \t]
{
blanks = TRUE;
goto strip;
}
[] / all
{ RETURN(blanks); }
*/
comment:
/*!re2c
"*/"
{
if(--depth == 0)
goto strip;
else
goto comment;
}
"\n"
{
++(ScanCB.lineNum);
ScanCB.linePos = ScanCB.pos + (cursor - ScanCB.mrk);
goto comment;
}
"/*"
{
++depth;
goto comment;
}
eof
{ RETURN(blanks); }
any
{
goto comment;
}
*/
}

41
tools/re2c/examples/rexx/scanio.c Normal file
View file

@ -0,0 +1,41 @@
uchar *ScanFill(uchar *cursor){
unsigned cnt = s->tok - s->bot;
s->pos += cursor - s->mrk;
if(cnt){
if(s->eot){
unsigned len = s->eot - s->tok;
memcpy(s->bot, s->tok, len);
s->eot = &s->bot[len];
if((len = s->lim - cursor) != 0)
memcpy(s->eot, cursor, len);
cursor = s->eot;
s->lim = &cursor[len];
} else {
memcpy(s->bot, s->tok, s->lim - s->tok);
cursor -= cnt;
s->lim -= cnt;
}
s->tok = s->bot;
s->ptr -= cnt;
}
if((s->top - s->lim) < 512){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + 512)*sizeof(uchar));
memcpy(buf, s->bot, s->lim - s->bot);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
if(s->eot)
s->eot = &buf[s->eot - s->bot];
cursor = &buf[cursor - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[512];
free(s->bot);
s->bot = buf;
}
s->mrk = cursor;
if(ScanCBIO.file){
if((cnt = read(ScanCBIO.u.f.fd, (char*) s->lim, 512)) != 512)
memset(&s->lim[cnt], 0, 512 - cnt);
s->lim += 512;
}
return cursor;
}

7
tools/re2c/examples/sample.re Normal file
View file

@ -0,0 +1,7 @@
/*!re2c
"print" {return PRINT;}
[a-z]+ {return ID;}
[0-9]+ {return DEC;}
"0x" [0-9a-f]+ {return HEX;}
[\000-\377] {return ERR;}
*/

13
tools/re2c/examples/simple.re Normal file
View file

@ -0,0 +1,13 @@
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\000-\377] {return NULL;}
*/
}

18
tools/re2c/globals.h Normal file
View file

@ -0,0 +1,18 @@
/* $Id: globals.h,v 1.4 2004/05/13 03:47:52 nuffer Exp $ */
#ifndef _globals_h
#define _globals_h
#include "basics.h"
extern char *fileName;
extern char *outputFileName;
extern bool sFlag;
extern bool bFlag;
extern unsigned int oline;
extern uchar asc2ebc[256];
extern uchar ebc2asc[256];
extern uchar *xlat, *talx;
#endif

41
tools/re2c/ins.h Normal file
View file

@ -0,0 +1,41 @@
/* $Id: ins.h,v 1.3 2004/05/13 02:58:17 nuffer Exp $ */
#ifndef _ins_h
#define _ins_h
#include "basics.h"
const uint nChars = 256;
typedef uchar Char;
const uint CHAR = 0;
const uint GOTO = 1;
const uint FORK = 2;
const uint TERM = 3;
const uint CTXT = 4;
union Ins {
struct {
byte tag;
byte marked;
void *link;
} i;
struct {
ushort value;
ushort bump;
void *link;
} c;
};
inline bool isMarked(Ins *i){
return i->i.marked != 0;
}
inline void mark(Ins *i){
i->i.marked = true;
}
inline void unmark(Ins *i){
i->i.marked = false;
}
#endif

251
tools/re2c/install-sh Normal file
View file

@ -0,0 +1,251 @@
#!/bin/sh
#
# install - install a program, script, or datafile
# This comes from X11R5 (mit/util/scripts/install.sh).
#
# Copyright 1991 by the Massachusetts Institute of Technology
#
# Permission to use, copy, modify, distribute, and sell this software and its
# documentation for any purpose is hereby granted without fee, provided that
# the above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear in supporting
# documentation, and that the name of M.I.T. not be used in advertising or
# publicity pertaining to distribution of the software without specific,
# written prior permission. M.I.T. makes no representations about the
# suitability of this software for any purpose. It is provided "as is"
# without express or implied warranty.
#
# Calling this script install-sh is preferred over install.sh, to prevent
# `make' implicit rules from creating a file called install from it
# when there is no Makefile.
#
# This script is compatible with the BSD install script, but was written
# from scratch. It can only install one file at a time, a restriction
# shared with many OS's install programs.
# set DOITPROG to echo to test this script
# Don't use :- since 4.3BSD and earlier shells don't like it.
doit="${DOITPROG-}"
# put in absolute paths if you don't have them in your path; or use env. vars.
mvprog="${MVPROG-mv}"
cpprog="${CPPROG-cp}"
chmodprog="${CHMODPROG-chmod}"
chownprog="${CHOWNPROG-chown}"
chgrpprog="${CHGRPPROG-chgrp}"
stripprog="${STRIPPROG-strip}"
rmprog="${RMPROG-rm}"
mkdirprog="${MKDIRPROG-mkdir}"
transformbasename=""
transform_arg=""
instcmd="$mvprog"
chmodcmd="$chmodprog 0755"
chowncmd=""
chgrpcmd=""
stripcmd=""
rmcmd="$rmprog -f"
mvcmd="$mvprog"
src=""
dst=""
dir_arg=""
while [ x"$1" != x ]; do
case $1 in
-c) instcmd="$cpprog"
shift
continue;;
-d) dir_arg=true
shift
continue;;
-m) chmodcmd="$chmodprog $2"
shift
shift
continue;;
-o) chowncmd="$chownprog $2"
shift
shift
continue;;
-g) chgrpcmd="$chgrpprog $2"
shift
shift
continue;;
-s) stripcmd="$stripprog"
shift
continue;;
-t=*) transformarg=`echo $1 | sed 's/-t=//'`
shift
continue;;
-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
shift
continue;;
*) if [ x"$src" = x ]
then
src=$1
else
# this colon is to work around a 386BSD /bin/sh bug
:
dst=$1
fi
shift
continue;;
esac
done
if [ x"$src" = x ]
then
echo "install: no input file specified"
exit 1
else
true
fi
if [ x"$dir_arg" != x ]; then
dst=$src
src=""
if [ -d $dst ]; then
instcmd=:
chmodcmd=""
else
instcmd=mkdir
fi
else
# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
# might cause directories to be created, which would be especially bad
# if $src (and thus $dsttmp) contains '*'.
if [ -f $src -o -d $src ]
then
true
else
echo "install: $src does not exist"
exit 1
fi
if [ x"$dst" = x ]
then
echo "install: no destination specified"
exit 1
else
true
fi
# If destination is a directory, append the input filename; if your system
# does not like double slashes in filenames, you may need to add some logic
if [ -d $dst ]
then
dst="$dst"/`basename $src`
else
true
fi
fi
## this sed command emulates the dirname command
dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
# Make sure that the destination directory exists.
# this part is taken from Noah Friedman's mkinstalldirs script
# Skip lots of stat calls in the usual case.
if [ ! -d "$dstdir" ]; then
defaultIFS='
'
IFS="${IFS-${defaultIFS}}"
oIFS="${IFS}"
# Some sh's can't handle IFS=/ for some reason.
IFS='%'
set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
IFS="${oIFS}"
pathcomp=''
while [ $# -ne 0 ] ; do
pathcomp="${pathcomp}${1}"
shift
if [ ! -d "${pathcomp}" ] ;
then
$mkdirprog "${pathcomp}"
else
true
fi
pathcomp="${pathcomp}/"
done
fi
if [ x"$dir_arg" != x ]
then
$doit $instcmd $dst &&
if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
else
# If we're going to rename the final executable, determine the name now.
if [ x"$transformarg" = x ]
then
dstfile=`basename $dst`
else
dstfile=`basename $dst $transformbasename |
sed $transformarg`$transformbasename
fi
# don't allow the sed command to completely eliminate the filename
if [ x"$dstfile" = x ]
then
dstfile=`basename $dst`
else
true
fi
# Make a temp file name in the proper directory.
dsttmp=$dstdir/#inst.$$#
# Move or copy the file name to the temp name
$doit $instcmd $src $dsttmp &&
trap "rm -f ${dsttmp}" 0 &&
# and set any options; do chmod last to preserve setuid bits
# If any of these fail, we abort the whole thing. If we want to
# ignore errors from any of these, just make sure not to ignore
# errors from the above "$doit $instcmd $src $dsttmp" command.
if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
# Now rename the file to the real destination.
$doit $rmcmd -f $dstdir/$dstfile &&
$doit $mvcmd $dsttmp $dstdir/$dstfile
fi &&
exit 0

149
tools/re2c/main.cc Normal file
View file

@ -0,0 +1,149 @@
/* $Id: main.cc,v 1.10 2004/05/13 03:47:52 nuffer Exp $ */
#ifdef HAVE_CONFIG_H
#include "config.h"
#elif _WIN32
#include "configwin.h"
#endif
#include <fstream>
#include <iostream>
#include <stdlib.h>
#include <string.h>
#include "globals.h"
#include "parser.h"
#include "dfa.h"
#include "mbo_getopt.h"
char *fileName = 0;
char *outputFileName = 0;
bool sFlag = false;
bool bFlag = false;
unsigned int oline = 1;
using namespace std;
static char *opt_arg = NULL;
static int opt_ind = 1;
static const mbo_opt_struct OPTIONS[] = {
{'?', 0, "help"},
{'b', 0, "bit-vectors"},
{'e', 0, "ecb"},
{'h', 0, "help"},
{'s', 0, "nested-ifs"},
{'o', 1, "output"},
{'v', 0, "version"}
};
static void usage()
{
cerr << "usage: re2c [-esbvh] file\n"
"\n"
"-? -h --help Display this info.\n"
"\n"
"-b --bit-vectors Implies -s. Use bit vectors as well in the attempt to\n"
" coax better code out of the compiler. Most useful for\n"
" specifications with more than a few keywords (e.g. for\n"
" most programming languages).\n"
"\n"
"-e --ecb Cross-compile from an ASCII platform to\n"
" an EBCDIC one.\n"
"\n"
"-s --nested-ifs Generate nested ifs for some switches. Many compilers\n"
" need this assist to generate better code.\n"
"\n"
"-o --output=output Specify the output file instead of stdout\n"
"\n"
"-v --version Show version information.\n";
}
int main(int argc, char *argv[])
{
int c;
fileName = NULL;
if (argc == 1) {
usage();
return 2;
}
while ((c = mbo_getopt(argc, argv, OPTIONS, &opt_arg, &opt_ind, 0))!=-1) {
switch (c) {
case 'b':
sFlag = true;
bFlag = true;
break;
case 'e':
xlat = asc2ebc;
talx = ebc2asc;
break;
case 's':
sFlag = true;
break;
case 'o':
outputFileName = opt_arg;
break;
case 'v':
cerr << "re2c " << PACKAGE_VERSION << "\n";
return 2;
case 'h':
case '?':
default:
usage();
return 2;
}
}
if (argc == opt_ind + 1)
{
fileName = argv[opt_ind];
}
else
{
usage();
return 2;
}
// set up the input stream
istream* input = 0;
ifstream inputFile;
if (fileName[0] == '-' && fileName[1] == '\0')
{
fileName = "<stdin>";
input = &cin;
}
else
{
inputFile.open(fileName);
if (!inputFile)
{
cerr << "can't open " << fileName << "\n";
return 1;
}
input = &inputFile;
}
// set up the output stream
ostream* output = 0;
ofstream outputFile;
if (outputFileName == 0 || (fileName[0] == '-' && fileName[1] == '\0'))
{
outputFileName = "<stdout>";
output = &cout;
}
else
{
outputFile.open(outputFileName);
if (!outputFile)
{
cerr << "can't open " << outputFileName << "\n";
return 1;
}
output = &outputFile;
}
parse(*input, *output);
return 0;
}

45
tools/re2c/makerpm.in Normal file
View file

@ -0,0 +1,45 @@
#! /bin/sh
# $Id: makerpm.in,v 1.7 2004/04/19 23:31:34 helly Exp $
# M.Boerger <re2c@somabo.de>
PREFIX="@PACKAGE_NAME@"
VERSION="@PACKAGE_VERSION@"
TARDIR="`basename \`pwd\``"
RELEASE=${1:-1}
echo "Usage:"
echo "$0 <release>"
echo
echo "e.g.:"
echo "$0"
echo -n "Building RPM version $VERSION, release: $RELEASE "
sleep 1 ; echo -n . ; sleep 1 ; echo -n . ; sleep 1 ; echo -n .
echo
TAR=@PACKAGE_TARNAME@-$VERSION.tar.gz
SPEC=re2c-${VERSION}.spec
RPMBASE=/usr/src/redhat
for i in /usr/src/redhat /usr/src/packages /usr/src/RPM; do
if test -d $i; then
RPMBASE=$i
break
fi
done
RPMDIR=${RPMBASE}/RPMS
SPECDIR=${RPMBASE}/SPECS
SRCDIR=${RPMBASE}/SOURCES
make dist
cp $TAR $SRCDIR
cat re2c.spec | sed -e "s/RPM_RELEASE/${RELEASE}/g" > ${SPEC}
echo "CP: (`pwd`) cp -a $SPEC $SPECDIR/${SPEC}"
cp -a $SPEC $SPECDIR/${SPEC}
#cp -a *.patch $SRCDIR
cd $SPECDIR
echo "RPM: rpm -ba ${SPEC}"
rpmbuild -ba ${SPEC}

142
tools/re2c/mbo_getopt.cc Normal file
View file

@ -0,0 +1,142 @@
/*
Author: Marcus Boerger <helly@users.sourceforge.net>
*/
/* $Id: mbo_getopt.cc,v 1.1 2004/04/19 21:14:46 helly Exp $ */
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include "mbo_getopt.h"
#define OPTERRCOLON (1)
#define OPTERRNF (2)
#define OPTERRARG (3)
static int mbo_opt_error(int argc, char * const *argv, int oint, int optchr, int err, int show_err)
{
if (show_err)
{
fprintf(stderr, "Error in argument %d, char %d: ", oint, optchr+1);
switch(err)
{
case OPTERRCOLON:
fprintf(stderr, ": in flags\n");
break;
case OPTERRNF:
fprintf(stderr, "option not found %c\n", argv[oint][optchr]);
break;
case OPTERRARG:
fprintf(stderr, "no argument for option %c\n", argv[oint][optchr]);
break;
default:
fprintf(stderr, "unknown\n");
break;
}
}
return('?');
}
int mbo_getopt(int argc, char* const *argv, const mbo_opt_struct opts[], char **optarg, int *optind, int show_err)
{
static int optchr = 0;
static int dash = 0; /* have already seen the - */
int arg_start = 2;
int opts_idx = -1;
if (*optind >= argc) {
return(EOF);
}
if (!dash) {
if ((argv[*optind][0] != '-')) {
return(EOF);
} else {
if (!argv[*optind][1])
{
/*
* use to specify stdin. Need to let pgm process this and
* the following args
*/
return(EOF);
}
}
}
if ((argv[*optind][0] == '-') && (argv[*optind][1] == '-')) {
/* '--' indicates end of args if not followed by a known long option name */
while (1) {
opts_idx++;
if (opts[opts_idx].opt_char == '-') {
(*optind)++;
return(EOF);
} else if (opts[opts_idx].opt_name && !strcmp(&argv[*optind][2], opts[opts_idx].opt_name)) {
break;
}
}
optchr = 0;
dash = 1;
arg_start = 2 + strlen(opts[opts_idx].opt_name);
}
if (!dash) {
dash = 1;
optchr = 1;
}
/* Check if the guy tries to do a -: kind of flag */
if (argv[*optind][optchr] == ':') {
dash = 0;
(*optind)++;
return (mbo_opt_error(argc, argv, *optind-1, optchr, OPTERRCOLON, show_err));
}
if (opts_idx < 0) {
while (1) {
opts_idx++;
if (opts[opts_idx].opt_char == '-') {
int errind = *optind;
int errchr = optchr;
if (!argv[*optind][optchr+1]) {
dash = 0;
(*optind)++;
} else {
optchr++;
}
return(mbo_opt_error(argc, argv, errind, errchr, OPTERRNF, show_err));
} else if (argv[*optind][optchr] == opts[opts_idx].opt_char) {
break;
}
}
}
if (opts[opts_idx].need_param) {
/* Check for cases where the value of the argument
is in the form -<arg> <val> or in the form -<arg><val> */
dash = 0;
if(!argv[*optind][arg_start]) {
(*optind)++;
if (*optind == argc) {
return(mbo_opt_error(argc, argv, *optind-1, optchr, OPTERRARG, show_err));
}
*optarg = argv[(*optind)++];
} else {
*optarg = &argv[*optind][arg_start];
(*optind)++;
}
return opts[opts_idx].opt_char;
} else {
if (arg_start == 2) {
if (!argv[*optind][optchr+1])
{
dash = 0;
(*optind)++;
} else {
optchr++;
}
} else {
(*optind)++;
}
return opts[opts_idx].opt_char;
}
assert(0);
return(0); /* never reached */
}

16
tools/re2c/mbo_getopt.h Normal file
View file

@ -0,0 +1,16 @@
/*
Author: Marcus Boerger <helly@users.sourceforge.net>
*/
/* $Id: mbo_getopt.h,v 1.1 2004/04/19 21:14:46 helly Exp $ */
/* Define structure for one recognized option (both single char and long name).
* If short_open is '-' this is the last option.
*/
typedef struct _mbo_opt_struct {
const char opt_char;
const int need_param;
const char * opt_name;
} mbo_opt_struct;
int mbo_getopt(int argc, char* const *argv, const mbo_opt_struct opts[], char **optarg, int *optind, int show_err);

198
tools/re2c/missing Normal file
View file

@ -0,0 +1,198 @@
#! /bin/sh
# Common stub for a few missing GNU programs while installing.
# Copyright (C) 1996, 1997, 2001, 2002 Free Software Foundation, Inc.
# Franc,ois Pinard <pinard@iro.umontreal.ca>, 1996.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
if test $# -eq 0; then
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
fi
# In the cases where this matters, `missing' is being run in the
# srcdir already.
if test -f configure.in; then
configure_ac=configure.ac
else
configure_ac=configure.in
fi
case "$1" in
-h|--h|--he|--hel|--help)
echo "\
$0 [OPTION]... PROGRAM [ARGUMENT]...
Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
error status if there is no known handling for PROGRAM.
Options:
-h, --help display this help and exit
-v, --version output version information and exit
Supported PROGRAM values:
aclocal touch file \`aclocal.m4'
autoconf touch file \`configure'
autoheader touch file \`config.h.in'
automake touch all \`Makefile.in' files
bison create \`y.tab.[ch]', if possible, from existing .[ch]
flex create \`lex.yy.c', if possible, from existing .c
lex create \`lex.yy.c', if possible, from existing .c
makeinfo touch the output file
yacc create \`y.tab.[ch]', if possible, from existing .[ch]"
;;
-v|--v|--ve|--ver|--vers|--versi|--versio|--version)
echo "missing - GNU libit 0.0"
;;
-*)
echo 1>&2 "$0: Unknown \`$1' option"
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
;;
aclocal*)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`acinclude.m4' or \`$configure_ac'. You might want
to install the \`Automake' and \`Perl' packages. Grab them from
any GNU archive site."
touch aclocal.m4
;;
autoconf)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`$configure_ac'. You might want to install the
\`Autoconf' and \`GNU m4' packages. Grab them from any GNU
archive site."
touch configure
;;
autoheader)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`acconfig.h' or \`$configure_ac'. You might want
to install the \`Autoconf' and \`GNU m4' packages. Grab them
from any GNU archive site."
files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' $configure_ac`
test -z "$files" && files="config.h"
touch_files=
for f in $files; do
case "$f" in
*:*) touch_files="$touch_files "`echo "$f" |
sed -e 's/^[^:]*://' -e 's/:.*//'`;;
*) touch_files="$touch_files $f.in";;
esac
done
touch $touch_files
;;
automake*)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`Makefile.am', \`acinclude.m4' or \`$configure_ac'.
You might want to install the \`Automake' and \`Perl' packages.
Grab them from any GNU archive site."
find . -type f -name Makefile.am -print |
sed 's/\.am$/.in/' |
while read f; do touch "$f"; done
;;
bison|yacc)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified a \`.y' file. You may need the \`Bison' package
in order for those modifications to take effect. You can get
\`Bison' from any GNU archive site."
rm -f y.tab.c y.tab.h
if [ $# -ne 1 ]; then
eval LASTARG="\${$#}"
case "$LASTARG" in
*.y)
SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
if [ -f "$SRCFILE" ]; then
cp "$SRCFILE" y.tab.c
fi
SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
if [ -f "$SRCFILE" ]; then
cp "$SRCFILE" y.tab.h
fi
;;
esac
fi
if [ ! -f y.tab.h ]; then
echo >y.tab.h
fi
if [ ! -f y.tab.c ]; then
echo 'main() { return 0; }' >y.tab.c
fi
;;
lex|flex)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified a \`.l' file. You may need the \`Flex' package
in order for those modifications to take effect. You can get
\`Flex' from any GNU archive site."
rm -f lex.yy.c
if [ $# -ne 1 ]; then
eval LASTARG="\${$#}"
case "$LASTARG" in
*.l)
SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
if [ -f "$SRCFILE" ]; then
cp "$SRCFILE" lex.yy.c
fi
;;
esac
fi
if [ ! -f lex.yy.c ]; then
echo 'main() { return 0; }' >lex.yy.c
fi
;;
makeinfo)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified a \`.texi' or \`.texinfo' file, or any other file
indirectly affecting the aspect of the manual. The spurious
call might also be the consequence of using a buggy \`make' (AIX,
DU, IRIX). You might want to install the \`Texinfo' package or
the \`GNU make' package. Grab either from any GNU archive site."
file=`echo "$*" | sed -n 's/.*-o \([^ ]*\).*/\1/p'`
if test -z "$file"; then
file=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
file=`sed -n '/^@setfilename/ { s/.* \([^ ]*\) *$/\1/; p; q; }' $file`
fi
touch $file
;;
*)
echo 1>&2 "\
WARNING: \`$1' is needed, and you do not seem to have it handy on your
system. You might have modified some files without having the
proper tools for further handling them. Check the \`README' file,
it often tells you about the needed prerequirements for installing
this package. You may also peek at any GNU archive site, in case
some other package would contain this missing \`$1' program."
exit 1
;;
esac
exit 0

40
tools/re2c/mkinstalldirs Normal file
View file

@ -0,0 +1,40 @@
#! /bin/sh
# mkinstalldirs --- make directory hierarchy
# Author: Noah Friedman <friedman@prep.ai.mit.edu>
# Created: 1993-05-16
# Public domain
# $Id: mkinstalldirs,v 1.13 1999/01/05 03:18:55 bje Exp $
errstatus=0
for file
do
set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
shift
pathcomp=
for d
do
pathcomp="$pathcomp$d"
case "$pathcomp" in
-* ) pathcomp=./$pathcomp ;;
esac
if test ! -d "$pathcomp"; then
echo "mkdir $pathcomp"
mkdir "$pathcomp" || lasterr=$?
if test ! -d "$pathcomp"; then
errstatus=$lasterr
fi
fi
pathcomp="$pathcomp/"
done
done
exit $errstatus
# mkinstalldirs ends here

1433
tools/re2c/parser.cc Normal file
View file

File diff suppressed because it is too large Load diff

23
tools/re2c/parser.h Normal file
View file

@ -0,0 +1,23 @@
/* $Id: parser.h,v 1.4 2004/05/13 02:58:17 nuffer Exp $ */
#ifndef _parser_h
#define _parser_h
#include "scanner.h"
#include "re.h"
#include <iosfwd>
class Symbol {
public:
static Symbol *first;
Symbol *next;
Str name;
RegExp *re;
public:
Symbol(const SubStr&);
static Symbol *find(const SubStr&);
};
void line_source(unsigned int, std::ostream&);
void parse(std::istream&, std::ostream&);
#endif

211
tools/re2c/parser.y Normal file
View file

@ -0,0 +1,211 @@
%{
/* $Id: parser.y,v 1.9 2004/05/13 14:26:01 nuffer Exp $ */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <time.h>
#include <string.h>
#include <stdlib.h>
#include <iostream>
#include "globals.h"
#include "parser.h"
extern "C"
{
int yyparse();
int yylex();
void yyerror(char*);
}
static uint accept;
static RegExp *spec;
static Scanner *in;
/* Bison version 1.875 emits a definition that is not working
* with several g++ version. Hence we disable it here.
*/
#if defined(__GNUC__)
#define __attribute__(x)
#endif
/* strdup() isn't standard C, so if we don't have it, we'll create our
* own version
*/
#if !defined(HAVE_STRDUP)
static char* strdup(const char* s)
{
char* rv = (char*)malloc(strlen(s) + 1);
if (rv == NULL)
return NULL;
strcpy(rv, s);
return rv;
}
#endif
%}
%start spec
%union {
Symbol *symbol;
RegExp *regexp;
Token *token;
char op;
ExtOp extop;
};
%token CLOSESIZE CLOSE ID CODE RANGE STRING
%type <op> CLOSE
%type <op> close
%type <extop> CLOSESIZE
%type <symbol> ID
%type <token> CODE
%type <regexp> RANGE STRING
%type <regexp> rule look expr diff term factor primary
%%
spec :
{ accept = 0;
spec = NULL; }
| spec rule
{ spec = spec? mkAlt(spec, $2) : $2; }
| spec decl
;
decl : ID '=' expr ';'
{ if($1->re)
in->fatal("sym already defined");
$1->re = $3; }
;
rule : expr look CODE
{ $$ = new RuleOp($1, $2, $3, accept++); }
;
look :
{ $$ = new NullOp; }
| '/' expr
{ $$ = $2; }
;
expr : diff
{ $$ = $1; }
| expr '|' diff
{ $$ = mkAlt($1, $3); }
;
diff : term
{ $$ = $1; }
| diff '\\' term
{ $$ = mkDiff($1, $3);
if(!$$)
in->fatal("can only difference char sets");
}
;
term : factor
{ $$ = $1; }
| term factor
{ $$ = new CatOp($1, $2); }
;
factor : primary
{ $$ = $1; }
| primary close
{
switch($2){
case '*':
$$ = mkAlt(new CloseOp($1), new NullOp());
break;
case '+':
$$ = new CloseOp($1);
break;
case '?':
$$ = mkAlt($1, new NullOp());
break;
}
}
| primary CLOSESIZE
{
$$ = new CloseVOp($1, $2.minsize, $2.maxsize);
}
;
close : CLOSE
{ $$ = $1; }
| close CLOSE
{ $$ = ($1 == $2) ? $1 : '*'; }
;
primary : ID
{ if(!$1->re)
in->fatal("can't find symbol");
$$ = $1->re; }
| RANGE
{ $$ = $1; }
| STRING
{ $$ = $1; }
| '(' expr ')'
{ $$ = $2; }
;
%%
extern "C" {
void yyerror(char* s){
in->fatal(s);
}
int yylex(){
return in->scan();
}
} // end extern "C"
void line_source(unsigned int line, std::ostream& o)
{
char * fnamebuf;
char * token;
o << "#line " << line << " \"";
if( fileName != NULL ) {
fnamebuf = strdup( fileName );
} else {
fnamebuf = strdup( "<stdin>" );
}
token = strtok( fnamebuf, "\\" );
for(;;) {
o << token;
token = strtok( NULL, "\\" );
if( token == NULL ) break;
o << "\\\\";
}
o << "\"\n";
++oline;
free( fnamebuf );
}
void parse(std::istream& i, std::ostream &o){
o << "/* Generated by re2c " PACKAGE_VERSION " on ";
time_t now = time(&now);
o.write(ctime(&now), 24);
o << " */\n";
oline += 2;
in = new Scanner(i);
line_source(in->line(), o);
while(in->echo(o)){
yyparse();
if(spec)
genCode(o, spec);
line_source(in->line(), o);
}
}

206
tools/re2c/re.h Normal file
View file

@ -0,0 +1,206 @@
/* $Id: re.h,v 1.5 2004/05/13 02:58:18 nuffer Exp $ */
#ifndef _re_h
#define _re_h
#include <iostream>
#include "token.h"
#include "ins.h"
typedef struct extop {
char op;
int minsize;
int maxsize;
} ExtOp;
struct CharPtn {
uint card;
CharPtn *fix;
CharPtn *nxt;
};
struct CharSet {
CharPtn *fix;
CharPtn *freeHead, **freeTail;
CharPtn *rep[nChars];
CharPtn ptn[nChars];
};
class Range {
public:
Range *next;
uint lb, ub; // [lb,ub)
public:
Range(uint l, uint u) : next(NULL), lb(l), ub(u)
{ }
Range(Range &r) : next(NULL), lb(r.lb), ub(r.ub)
{ }
friend std::ostream& operator<<(std::ostream&, const Range&);
friend std::ostream& operator<<(std::ostream&, const Range*);
};
inline std::ostream& operator<<(std::ostream &o, const Range *r){
return r? o << *r : o;
}
class RegExp {
public:
uint size;
public:
virtual char *typeOf() = 0;
RegExp *isA(char *t)
{ return typeOf() == t? this : NULL; }
virtual void split(CharSet&) = 0;
virtual void calcSize(Char*) = 0;
virtual uint fixedLength();
virtual void compile(Char*, Ins*) = 0;
virtual void display(std::ostream&) const = 0;
friend std::ostream& operator<<(std::ostream&, const RegExp&);
friend std::ostream& operator<<(std::ostream&, const RegExp*);
};
inline std::ostream& operator<<(std::ostream &o, const RegExp &re){
re.display(o);
return o;
}
inline std::ostream& operator<<(std::ostream &o, const RegExp *re){
return o << *re;
}
class NullOp: public RegExp {
public:
static char *type;
public:
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << "_";
}
};
class MatchOp: public RegExp {
public:
static char *type;
Range *match;
public:
MatchOp(Range *m) : match(m)
{ }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream&) const;
};
class RuleOp: public RegExp {
private:
RegExp *exp;
public:
RegExp *ctx;
static char *type;
Ins *ins;
uint accept;
Token *code;
uint line;
public:
RuleOp(RegExp*, RegExp*, Token*, uint);
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp << "/" << ctx << ";";
}
};
class AltOp: public RegExp {
private:
RegExp *exp1, *exp2;
public:
static char *type;
public:
AltOp(RegExp *e1, RegExp *e2)
{ exp1 = e1; exp2 = e2; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp1 << "|" << exp2;
}
friend RegExp *mkAlt(RegExp*, RegExp*);
};
class CatOp: public RegExp {
private:
RegExp *exp1, *exp2;
public:
static char *type;
public:
CatOp(RegExp *e1, RegExp *e2)
{ exp1 = e1; exp2 = e2; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp1 << exp2;
}
};
class CloseOp: public RegExp {
private:
RegExp *exp;
public:
static char *type;
public:
CloseOp(RegExp *e)
{ exp = e; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp << "+";
}
};
class CloseVOp: public RegExp {
private:
RegExp *exp;
int min;
int max;
public:
static char *type;
public:
CloseVOp(RegExp *e, int lb, int ub)
{ exp = e; min = lb; max = ub; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp << "+";
}
};
extern void genCode(std::ostream&, RegExp*);
extern RegExp *mkDiff(RegExp*, RegExp*);
extern RegExp *strToRE(SubStr);
extern RegExp *ranToRE(SubStr);
extern RegExp *strToCaseInsensitiveRE(SubStr s);
#endif

601
tools/re2c/re2c.1 Normal file
View file

@ -0,0 +1,601 @@
./"
./" $Id: re2c.1.in,v 1.9 2004/05/26 13:33:25 nuffer Exp $
./"
.TH RE2C 1 "14 March 1994" "Version 0.9.3"
.ds re \fBre2c\fP
.ds le \fBlex\fP
.ds rx regular expression
.ds lx \fIl\fP-expression
\"$Log: re2c.1.in,v $
\"Revision 1.9 2004/05/26 13:33:25 nuffer
\"Added description of -o option.
\"
\"Revision 1.8 2004/04/19 22:32:48 helly
\"Update
\"
\"Revision 1.7 2004/04/19 02:13:48 helly
\"Featurerequest #869298 (Add case insensitive string literals)
\"
\"Revision 1.6 2004/04/17 15:49:13 helly
\"Fix example, cur must be set to make the uncommented printf's working
\"
\"Revision 1.5 2004/03/30 01:02:45 helly
\"Update docu
\"
\"Revision 1.4 2004/03/14 14:23:40 helly
\"Update
\"
\"Revision 1.3 2004/03/14 12:54:19 helly
\"Next step of autogen patch
\"
\"Revision 1.2 2004/03/13 20:35:12 helly
\"Updated configure stuff
\"
\"Revision 1.1 2004/01/31 15:44:39 nuffer
\"Applied patch from Marcus Boerger
\"
\"Revision 1.2 1994/04/16 15:50:32 peter
\"Fix bug in simple example.
\"
\"Revision 1.1 1994/04/08 15:39:09 peter
\"Initial revision
\"
.SH NAME
re2c \- convert regular expressions to C/C++
.SH SYNOPSIS
\*(re [\fB-esbvh\fP] [\fB-o output\fP] file\fP
.SH DESCRIPTION
\*(re is a preprocessor that generates C-based recognizers from regular
expressions.
The input to \*(re consists of C/C++ source interleaved with
comments of the form \fC/*!re2c\fP ... \fC*/\fP which contain
scanner specifications.
In the output these comments are replaced with code that, when
executed, will find the next input token and then execute
some user-supplied token-specific code.
For example, given the following code
.in +3
.nf
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\\000-\\377] {return NULL;}
*/
}
.fi
.in -3
\*(re will generate
.in +3
.nf
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
.fi
.in -3
.SH OPTIONS
\*(re provides the following options:
.TP
\fB-e\fP
Cross-compile from an ASCII platform to an EBCDIC one.
.TP
\fB-s\fP
Generate nested \fCif\fPs for some \fCswitch\fPes. Many compilers need this
assist to generate better code.
.TP
\fB-b\fP
Implies \fB-s\fP. Use bit vectors as well in the attempt to coax better
code out of the compiler. Most useful for specifications with more than a
few keywords (e.g. for most programming languages).
.TP
\fB-h\fP
\fB-?\fP
Invoke a short help.
.TP
\fB-v\fP
Show version information.
.TP
\fB-o output\fP
Specify the output file.
.SH "INTERFACE CODE"
Unlike other scanner generators, \*(re does not generate complete scanners:
the user must supply some interface code.
In particular, the user must define the following macros:
.TP
\fCYYCHAR\fP
Type used to hold an input symbol.
Usually \fCchar\fP or \fCunsigned char\fP.
.TP
\fCYYCURSOR\fP
\*(lx of type \fC*YYCHAR\fP that points to the current input symbol.
The generated code advances \fCYYCURSOR\fP as symbols are matched.
On entry, \fCYYCURSOR\fP is assumed to point to the first character of the
current token. On exit, \fCYYCURSOR\fP will point to the first character of
the following token.
.TP
\fCYLIMIT\fP
Expression of type \fC*YYCHAR\fP that marks the end of the buffer
(\fCYLIMIT[-1]\fP is the last character in the buffer).
The generated code repeatedly compares \fCYYCURSOR\fP to \fCYLIMIT\fP
to determine when the buffer needs (re)filling.
.TP
\fCYYMARKER\fP
\*(lx of type \fC*YYCHAR\fP.
The generated code saves backtracking information in \fCYYMARKER\fP.
.TP
\fCYYFILL(\fP\fIn\fP\fC)\fP
The generated code "calls" \fCYYFILL\fP when the buffer needs
(re)filling: at least \fIn\fP additional characters should
be provided. \fCYYFILL\fP should adjust \fCYYCURSOR\fP, \fCYYLIMIT\fP and
\fCYYMARKER\fP as needed. Note that for typical programming languages
\fIn\fP will be the length of the longest keyword plus one.
.SH "SCANNER SPECIFICATIONS"
Each scanner specification consists of a set of \fIrules\fP and name
definitions.
Rules consist of a regular expression along with a block of C/C++ code that
is to be executed when the associated regular expression is matched.
Name definitions are of the form
``\fIname\fP \fC=\fP \fIregular expression\fP\fC;\fP''.
.SH "SUMMARY OF RE2C REGULAR EXPRESSIONS"
.TP
\fC"foo"\fP
the literal string \fCfoo\fP.
ANSI-C escape sequences can be used.
.TP
\fC'foo'\fP
the literal string \fCfoo\fP (characters [a-zA-Z] treated case-insensitive).
ANSI-C escape sequences can be used.
.TP
\fC[xyz]\fP
a "character class"; in this case,
the \*(rx matches either an '\fCx\fP', a '\fCy\fP', or a '\fCz\fP'.
.TP
\fC[abj-oZ]\fP
a "character class" with a range in it;
matches an '\fCa\fP', a '\fCb\fP', any letter from '\fCj\fP' through '\fCo\fP',
or a '\fCZ\fP'.
.TP
\fIr\fP\fC\e\fP\fIs\fP
match any \fIr\fP which isn't an \fIs\fP. \fIr\fP and \fIs\fP must be regular expressions
which can be expressed as character classes.
.TP
\fIr\fP\fC*\fP
zero or more \fIr\fP's, where \fIr\fP is any regular expression
.TP
\fC\fIr\fP\fC+\fP
one or more \fIr\fP's
.TP
\fC\fIr\fP\fC?\fP
zero or one \fIr\fP's (that is, "an optional \fIr\fP")
.TP
name
the expansion of the "name" definition (see above)
.TP
\fC(\fP\fIr\fP\fC)\fP
an \fIr\fP; parentheses are used to override precedence
(see below)
.TP
\fIrs\fP
an \fIr\fP followed by an \fIs\fP ("concatenation")
.TP
\fIr\fP\fC|\fP\fIs\fP
either an \fIr\fP or an \fIs\fP
.TP
\fIr\fP\fC/\fP\fIs\fP
an \fIr\fP but only if it is followed by an \fIs\fP. The s is not part of
the matched text. This type of \*(rx is called "trailing context".
.TP
\fIr\fP\fC{\fP\fIn\fP\fC}\fP
matches \fIr\fP exactly \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,}\fP
matches \fIr\fP at least \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,\fP\fIm\fP\fC}\fP
matches \fIr\fP at least \fIn\fP but not more than \fIm\fP times.
.LP
The regular expressions listed above are grouped according to
precedence, from highest precedence at the top to lowest at the bottom.
Those grouped together have equal precedence.
.SH "A LARGER EXAMPLE"
.LP
.in +3
.nf
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*)
malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\\n';
}
s->lim += cnt;
}
s->cur = cursor;
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\\000-\\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\\\] ([abfnrtv?'"\\\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\\[\\n\\\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\\[\\n\\\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \\t\\v\\f]+ { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\\t%.*s\\n", t, in.cur - in.tok, in.tok);
printf("%d\\n", t);
*/
}
close(in.fd);
}
.fi
.in -3
.SH FEATURES
.LP
\*(re does not provide a default action:
the generated code assumes that the input
will consist of a sequence of tokens.
Typically this can be dealt with by adding a rule such as the one for
unexpected characters in the example above.
.LP
The user must arrange for a sentinel token to appear at the end of input
(and provide a rule for matching it):
\*(re does not provide an \fC<<EOF>>\fP expression.
If the source is from a null-byte terminated string, a
rule matching a null character will suffice. If the source is from a
file then the approach taken in the example can be used: pad the input with
a newline (or some other character that can't appear within another token);
upon recognizing such a character check to see if it is the sentinel
and act accordingly.
.LP
\*(re does not provide start conditions: use a separate scanner
specification for each start condition (as illustrated in the above example).
.LP
No [^x]. Use difference instead.
.SH BUGS
.LP
Only fixed length trailing context can be handled.
.LP
The maximum value appearing as a parameter \fIn\fP to \fCYYFILL\fP is not
provided to the generated code (this value is needed for constructing
the interface code).
Note that this value is usually relatively small: for
typical programming languages \fIn\fP will be the length of the longest
keyword plus one.
.LP
Difference only works for character sets.
.LP
The \*(re internal algorithms need documentation.
.SH "SEE ALSO"
.LP
flex(1), lex(1).
.P
More information on \fBre2c\fP can be found here:
.PD 0
.P
.B http://sourceforge.net/projects/re2c/
.PD 1
.SH AUTHORS
.PD 0
.P
Peter Bumbulis <peter@csg.uwaterloo.ca>
.P
Brian Young <bayoung@acm.org>
.P
Dan Nuffer <nuffer@users.sourceforge.net>
.P
Marcus Boerger <helly@users.sourceforge.net>
.P
Hartmut Kaiser <hkaiser@users.sourceforge.net>
.P
.PD 1
.SH VERSION INFORMATION
This manpage describes \fBre2c\fP, version 0.9.3.
.fi

601
tools/re2c/re2c.1.in Normal file
View file

@ -0,0 +1,601 @@
./"
./" $Id: re2c.1.in,v 1.9 2004/05/26 13:33:25 nuffer Exp $
./"
.TH RE2C 1 "14 March 1994" "Version @PACKAGE_VERSION@"
.ds re \fBre2c\fP
.ds le \fBlex\fP
.ds rx regular expression
.ds lx \fIl\fP-expression
\"$Log: re2c.1.in,v $
\"Revision 1.9 2004/05/26 13:33:25 nuffer
\"Added description of -o option.
\"
\"Revision 1.8 2004/04/19 22:32:48 helly
\"Update
\"
\"Revision 1.7 2004/04/19 02:13:48 helly
\"Featurerequest #869298 (Add case insensitive string literals)
\"
\"Revision 1.6 2004/04/17 15:49:13 helly
\"Fix example, cur must be set to make the uncommented printf's working
\"
\"Revision 1.5 2004/03/30 01:02:45 helly
\"Update docu
\"
\"Revision 1.4 2004/03/14 14:23:40 helly
\"Update
\"
\"Revision 1.3 2004/03/14 12:54:19 helly
\"Next step of autogen patch
\"
\"Revision 1.2 2004/03/13 20:35:12 helly
\"Updated configure stuff
\"
\"Revision 1.1 2004/01/31 15:44:39 nuffer
\"Applied patch from Marcus Boerger
\"
\"Revision 1.2 1994/04/16 15:50:32 peter
\"Fix bug in simple example.
\"
\"Revision 1.1 1994/04/08 15:39:09 peter
\"Initial revision
\"
.SH NAME
re2c \- convert regular expressions to C/C++
.SH SYNOPSIS
\*(re [\fB-esbvh\fP] [\fB-o output\fP] file\fP
.SH DESCRIPTION
\*(re is a preprocessor that generates C-based recognizers from regular
expressions.
The input to \*(re consists of C/C++ source interleaved with
comments of the form \fC/*!re2c\fP ... \fC*/\fP which contain
scanner specifications.
In the output these comments are replaced with code that, when
executed, will find the next input token and then execute
some user-supplied token-specific code.
For example, given the following code
.in +3
.nf
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\\000-\\377] {return NULL;}
*/
}
.fi
.in -3
\*(re will generate
.in +3
.nf
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
.fi
.in -3
.SH OPTIONS
\*(re provides the following options:
.TP
\fB-e\fP
Cross-compile from an ASCII platform to an EBCDIC one.
.TP
\fB-s\fP
Generate nested \fCif\fPs for some \fCswitch\fPes. Many compilers need this
assist to generate better code.
.TP
\fB-b\fP
Implies \fB-s\fP. Use bit vectors as well in the attempt to coax better
code out of the compiler. Most useful for specifications with more than a
few keywords (e.g. for most programming languages).
.TP
\fB-h\fP
\fB-?\fP
Invoke a short help.
.TP
\fB-v\fP
Show version information.
.TP
\fB-o output\fP
Specify the output file.
.SH "INTERFACE CODE"
Unlike other scanner generators, \*(re does not generate complete scanners:
the user must supply some interface code.
In particular, the user must define the following macros:
.TP
\fCYYCHAR\fP
Type used to hold an input symbol.
Usually \fCchar\fP or \fCunsigned char\fP.
.TP
\fCYYCURSOR\fP
\*(lx of type \fC*YYCHAR\fP that points to the current input symbol.
The generated code advances \fCYYCURSOR\fP as symbols are matched.
On entry, \fCYYCURSOR\fP is assumed to point to the first character of the
current token. On exit, \fCYYCURSOR\fP will point to the first character of
the following token.
.TP
\fCYLIMIT\fP
Expression of type \fC*YYCHAR\fP that marks the end of the buffer
(\fCYLIMIT[-1]\fP is the last character in the buffer).
The generated code repeatedly compares \fCYYCURSOR\fP to \fCYLIMIT\fP
to determine when the buffer needs (re)filling.
.TP
\fCYYMARKER\fP
\*(lx of type \fC*YYCHAR\fP.
The generated code saves backtracking information in \fCYYMARKER\fP.
.TP
\fCYYFILL(\fP\fIn\fP\fC)\fP
The generated code "calls" \fCYYFILL\fP when the buffer needs
(re)filling: at least \fIn\fP additional characters should
be provided. \fCYYFILL\fP should adjust \fCYYCURSOR\fP, \fCYYLIMIT\fP and
\fCYYMARKER\fP as needed. Note that for typical programming languages
\fIn\fP will be the length of the longest keyword plus one.
.SH "SCANNER SPECIFICATIONS"
Each scanner specification consists of a set of \fIrules\fP and name
definitions.
Rules consist of a regular expression along with a block of C/C++ code that
is to be executed when the associated regular expression is matched.
Name definitions are of the form
``\fIname\fP \fC=\fP \fIregular expression\fP\fC;\fP''.
.SH "SUMMARY OF RE2C REGULAR EXPRESSIONS"
.TP
\fC"foo"\fP
the literal string \fCfoo\fP.
ANSI-C escape sequences can be used.
.TP
\fC'foo'\fP
the literal string \fCfoo\fP (characters [a-zA-Z] treated case-insensitive).
ANSI-C escape sequences can be used.
.TP
\fC[xyz]\fP
a "character class"; in this case,
the \*(rx matches either an '\fCx\fP', a '\fCy\fP', or a '\fCz\fP'.
.TP
\fC[abj-oZ]\fP
a "character class" with a range in it;
matches an '\fCa\fP', a '\fCb\fP', any letter from '\fCj\fP' through '\fCo\fP',
or a '\fCZ\fP'.
.TP
\fIr\fP\fC\e\fP\fIs\fP
match any \fIr\fP which isn't an \fIs\fP. \fIr\fP and \fIs\fP must be regular expressions
which can be expressed as character classes.
.TP
\fIr\fP\fC*\fP
zero or more \fIr\fP's, where \fIr\fP is any regular expression
.TP
\fC\fIr\fP\fC+\fP
one or more \fIr\fP's
.TP
\fC\fIr\fP\fC?\fP
zero or one \fIr\fP's (that is, "an optional \fIr\fP")
.TP
name
the expansion of the "name" definition (see above)
.TP
\fC(\fP\fIr\fP\fC)\fP
an \fIr\fP; parentheses are used to override precedence
(see below)
.TP
\fIrs\fP
an \fIr\fP followed by an \fIs\fP ("concatenation")
.TP
\fIr\fP\fC|\fP\fIs\fP
either an \fIr\fP or an \fIs\fP
.TP
\fIr\fP\fC/\fP\fIs\fP
an \fIr\fP but only if it is followed by an \fIs\fP. The s is not part of
the matched text. This type of \*(rx is called "trailing context".
.TP
\fIr\fP\fC{\fP\fIn\fP\fC}\fP
matches \fIr\fP exactly \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,}\fP
matches \fIr\fP at least \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,\fP\fIm\fP\fC}\fP
matches \fIr\fP at least \fIn\fP but not more than \fIm\fP times.
.LP
The regular expressions listed above are grouped according to
precedence, from highest precedence at the top to lowest at the bottom.
Those grouped together have equal precedence.
.SH "A LARGER EXAMPLE"
.LP
.in +3
.nf
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*)
malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\\n';
}
s->lim += cnt;
}
s->cur = cursor;
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\\000-\\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\\\] ([abfnrtv?'"\\\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\\[\\n\\\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\\[\\n\\\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \\t\\v\\f]+ { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\\t%.*s\\n", t, in.cur - in.tok, in.tok);
printf("%d\\n", t);
*/
}
close(in.fd);
}
.fi
.in -3
.SH FEATURES
.LP
\*(re does not provide a default action:
the generated code assumes that the input
will consist of a sequence of tokens.
Typically this can be dealt with by adding a rule such as the one for
unexpected characters in the example above.
.LP
The user must arrange for a sentinel token to appear at the end of input
(and provide a rule for matching it):
\*(re does not provide an \fC<<EOF>>\fP expression.
If the source is from a null-byte terminated string, a
rule matching a null character will suffice. If the source is from a
file then the approach taken in the example can be used: pad the input with
a newline (or some other character that can't appear within another token);
upon recognizing such a character check to see if it is the sentinel
and act accordingly.
.LP
\*(re does not provide start conditions: use a separate scanner
specification for each start condition (as illustrated in the above example).
.LP
No [^x]. Use difference instead.
.SH BUGS
.LP
Only fixed length trailing context can be handled.
.LP
The maximum value appearing as a parameter \fIn\fP to \fCYYFILL\fP is not
provided to the generated code (this value is needed for constructing
the interface code).
Note that this value is usually relatively small: for
typical programming languages \fIn\fP will be the length of the longest
keyword plus one.
.LP
Difference only works for character sets.
.LP
The \*(re internal algorithms need documentation.
.SH "SEE ALSO"
.LP
flex(1), lex(1).
.P
More information on \fBre2c\fP can be found here:
.PD 0
.P
.B http://sourceforge.net/projects/re2c/
.PD 1
.SH AUTHORS
.PD 0
.P
Peter Bumbulis <peter@csg.uwaterloo.ca>
.P
Brian Young <bayoung@acm.org>
.P
Dan Nuffer <nuffer@users.sourceforge.net>
.P
Marcus Boerger <helly@users.sourceforge.net>
.P
Hartmut Kaiser <hkaiser@users.sourceforge.net>
.P
.PD 1
.SH VERSION INFORMATION
This manpage describes \fBre2c\fP, version @PACKAGE_VERSION@.
.fi

51
tools/re2c/re2c.spec.in Normal file
View file

@ -0,0 +1,51 @@
Summary: re2c - A tool for generating C-based recognizers from regular expressions
Name: @PACKAGE_NAME@
Version: @PACKAGE_VERSION@
Release: RPM_RELEASE
Group: Development
License: public domain
URL: http://sourceforge.net/projects/re2c/
Source: %{name}-%{version}.tar.gz
BuildRoot: %{_tmppath}/%{name}-%{version}-root
%description
re2c is a great tool for writing fast and flexible lexers. It has
served many people well for many years and it deserves to be
maintained more actively. re2c is on the order of 2-3 times faster
than a flex based scanner, and its input model is much more
flexible.
%prep
%setup -q -n @PACKAGE_NAME@-@PACKAGE_VERSION@
%build
./configure \
--prefix=%{_prefix}
make re2c
#regenerate file scanner.cc
rm -f scanner.cc
./re2c scanner.re > scanner.cc
rm -f re2c scanner.o
make
%install
rm -rf $RPM_BUILD_ROOT
mkdir -p $RPM_BUILD_ROOT%{_bindir}
install -m 0755 re2c $RPM_BUILD_ROOT%{_bindir}
mkdir -p $RPM_BUILD_ROOT%{_mandir}/man1
install -m 0755 re2c.1 $RPM_BUILD_ROOT%{_mandir}/man1
%clean
rm -rf $RPM_BUILD_ROOT
%changelog
* Sun Jan 04 2003 Marcus Boerger <re2c@somabo.de>
- Initial version.
%files
%defattr(-,root,root)
%{_bindir}/re2c
%{_mandir}/man1/re2c.1*
%doc README examples doc/*

199
tools/re2c/re2c.vcproj Normal file
View file

@ -0,0 +1,199 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="7.10"
Name="re2c"
ProjectGUID="{667D2EE7-C357-49E2-9BAB-0A4A45F0F76E}"
Keyword="Win32Proj">
<Platforms>
<Platform
Name="Win32"/>
</Platforms>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="."
IntermediateDirectory="Build"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="1"
OptimizeForProcessor="3"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
StringPooling="TRUE"
ExceptionHandling="TRUE"
RuntimeLibrary="2"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="FALSE"
DebugInformationFormat="0"
CallingConvention="0"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/re2c.exe"
LinkIncremental="1"
GenerateDebugInformation="FALSE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
OptimizeForWindows98="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="."
IntermediateDirectory="Build"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="1"
OptimizeForProcessor="3"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
StringPooling="TRUE"
ExceptionHandling="TRUE"
RuntimeLibrary="2"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="FALSE"
DebugInformationFormat="0"
CallingConvention="0"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/re2c.exe"
LinkIncremental="1"
GenerateDebugInformation="FALSE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
OptimizeForWindows98="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
<File
RelativePath=".\actions.cc">
</File>
<File
RelativePath=".\code.cc">
</File>
<File
RelativePath=".\dfa.cc">
</File>
<File
RelativePath=".\main.cc">
</File>
<File
RelativePath=".\mbo_getopt.cc">
</File>
<File
RelativePath=".\parser.cc">
</File>
<File
RelativePath=".\scanner.cc">
</File>
<File
RelativePath=".\substr.cc">
</File>
<File
RelativePath=".\translate.cc">
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
<File
RelativePath=".\basics.h">
</File>
<File
RelativePath=".\dfa.h">
</File>
<File
RelativePath=".\globals.h">
</File>
<File
RelativePath=".\ins.h">
</File>
<File
RelativePath=".\mbo_getopt.h">
</File>
<File
RelativePath=".\parser.h">
</File>
<File
RelativePath=".\re.h">
</File>
<File
RelativePath=".\scanner.h">
</File>
<File
RelativePath=".\substr.h">
</File>
<File
RelativePath=".\token.h">
</File>
<File
RelativePath=".\y.tab.h">
</File>
</Filter>
<Filter
Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

639
tools/re2c/scanner.cc Normal file
View file

@ -0,0 +1,639 @@
/* Generated by re2c 0.9.3 on Wed May 26 17:58:01 2004 */
#line 1 "scanner.re"
/* $Id: scanner.re,v 1.10 2004/05/13 02:58:18 nuffer Exp $ */
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include "scanner.h"
#include "parser.h"
#include "y.tab.h"
extern YYSTYPE yylval;
#ifndef MAX
#define MAX(a,b) (((a)>(b))?(a):(b))
#endif
#define BSIZE 8192
#define YYCTYPE char
#define YYCURSOR cursor
#define YYLIMIT lim
#define YYMARKER ptr
#define YYFILL(n) {cursor = fill(cursor);}
#define RETURN(i) {cur = cursor; return i;}
Scanner::Scanner(std::istream& i) : in(i),
bot(NULL), tok(NULL), ptr(NULL), cur(NULL), pos(NULL), lim(NULL),
top(NULL), eof(NULL), tchar(0), tline(0), cline(1) {
;
}
char *Scanner::fill(char *cursor){
if(!eof){
uint cnt = tok - bot;
if(cnt){
memcpy(bot, tok, lim - tok);
tok = bot;
ptr -= cnt;
cursor -= cnt;
pos -= cnt;
lim -= cnt;
}
if((top - lim) < BSIZE){
char *buf = new char[(lim - bot) + BSIZE];
memcpy(buf, tok, lim - tok);
tok = buf;
ptr = &buf[ptr - bot];
cursor = &buf[cursor - bot];
pos = &buf[pos - bot];
lim = &buf[lim - bot];
top = &lim[BSIZE];
delete [] bot;
bot = buf;
}
if((cnt = in.rdbuf()->sgetn((char*) lim, BSIZE)) != BSIZE){
eof = &lim[cnt]; *eof++ = '\n';
}
lim += cnt;
}
return cursor;
}
#line 72 "scanner.re"
int Scanner::echo(std::ostream &out){
char *cursor = cur;
// Catch EOF
if (eof && cursor == eof)
return 0;
tok = cursor;
echo:
#line 7 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 7) YYFILL(7);
yych = *YYCURSOR;
if(yych == '\n') goto yy4;
if(yych != '/') goto yy6;
goto yy2;
yy2: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych == '*') goto yy7;
goto yy3;
yy3:
#line 91 "scanner.re"
{ goto echo; }
#line 26 "scanner.cc"
yy4: ++YYCURSOR;
goto yy5;
yy5:
#line 87 "scanner.re"
{ if(cursor == eof) RETURN(0);
out.write((const char*)(tok), (const char*)(cursor) - (const char*)(tok));
tok = pos = cursor; cline++;
goto echo; }
#line 35 "scanner.cc"
yy6: yych = *++YYCURSOR;
goto yy3;
yy7: yych = *++YYCURSOR;
if(yych == '!') goto yy9;
goto yy8;
yy8: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy3;
}
yy9: yych = *++YYCURSOR;
if(yych != 'r') goto yy8;
goto yy10;
yy10: yych = *++YYCURSOR;
if(yych != 'e') goto yy8;
goto yy11;
yy11: yych = *++YYCURSOR;
if(yych != '2') goto yy8;
goto yy12;
yy12: yych = *++YYCURSOR;
if(yych != 'c') goto yy8;
goto yy13;
yy13: ++YYCURSOR;
goto yy14;
yy14:
#line 84 "scanner.re"
{ out.write((const char*)(tok), (const char*)(&cursor[-7]) - (const char*)(tok));
tok = cursor;
RETURN(1); }
#line 64 "scanner.cc"
}
#line 92 "scanner.re"
}
int Scanner::scan(){
char *cursor = cur;
uint depth;
scan:
tchar = cursor - pos;
tline = cline;
tok = cursor;
#line 68 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy15;
yy16: ++YYCURSOR;
yy15:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/'){
if(yych <= '"'){
if(yych <= '\n'){
if(yych <= '\b') goto yy37;
if(yych <= '\t') goto yy33;
goto yy35;
} else {
if(yych == ' ') goto yy33;
if(yych <= '!') goto yy37;
goto yy23;
}
} else {
if(yych <= ')'){
if(yych <= '&') goto yy37;
if(yych <= '\'') goto yy25;
goto yy29;
} else {
if(yych <= '*') goto yy21;
if(yych <= '+') goto yy30;
if(yych <= '.') goto yy37;
goto yy19;
}
}
} else {
if(yych <= '@'){
if(yych <= '<'){
if(yych == ';') goto yy29;
goto yy37;
} else {
if(yych <= '=') goto yy29;
if(yych == '?') goto yy30;
goto yy37;
}
} else {
if(yych <= '`'){
if(yych <= 'Z') goto yy31;
if(yych <= '[') goto yy27;
if(yych <= '\\') goto yy29;
goto yy37;
} else {
if(yych <= 'z') goto yy31;
if(yych <= '{') goto yy17;
if(yych <= '|') goto yy29;
goto yy37;
}
}
}
yy17: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych <= '/') goto yy18;
if(yych <= '9') goto yy63;
goto yy18;
yy18:
#line 105 "scanner.re"
{ depth = 1;
goto code;
}
#line 134 "scanner.cc"
yy19: ++YYCURSOR;
if((yych = *YYCURSOR) == '*') goto yy61;
goto yy20;
yy20:
#line 131 "scanner.re"
{ RETURN(*tok); }
#line 141 "scanner.cc"
yy21: ++YYCURSOR;
if((yych = *YYCURSOR) == '/') goto yy59;
goto yy22;
yy22:
#line 133 "scanner.re"
{ yylval.op = *tok;
RETURN(CLOSE); }
#line 149 "scanner.cc"
yy23: yyaccept = 1;
yych = *(YYMARKER = ++YYCURSOR);
if(yych != '\n') goto yy55;
goto yy24;
yy24:
#line 122 "scanner.re"
{ fatal("unterminated string constant (missing \")"); }
#line 157 "scanner.cc"
yy25: yyaccept = 2;
yych = *(YYMARKER = ++YYCURSOR);
if(yych != '\n') goto yy50;
goto yy26;
yy26:
#line 123 "scanner.re"
{ fatal("unterminated string constant (missing ')"); }
#line 165 "scanner.cc"
yy27: yyaccept = 3;
yych = *(YYMARKER = ++YYCURSOR);
if(yych != '\n') goto yy44;
goto yy28;
yy28:
#line 129 "scanner.re"
{ fatal("unterminated range (missing ])"); }
#line 173 "scanner.cc"
yy29: yych = *++YYCURSOR;
goto yy20;
yy30: yych = *++YYCURSOR;
goto yy22;
yy31: ++YYCURSOR;
yych = *YYCURSOR;
goto yy42;
yy32:
#line 148 "scanner.re"
{ cur = cursor;
yylval.symbol = Symbol::find(token());
return ID; }
#line 185 "scanner.cc"
yy33: ++YYCURSOR;
yych = *YYCURSOR;
goto yy40;
yy34:
#line 152 "scanner.re"
{ goto scan; }
#line 191 "scanner.cc"
yy35: ++YYCURSOR;
goto yy36;
yy36:
#line 154 "scanner.re"
{ if(cursor == eof) RETURN(0);
pos = cursor; cline++;
goto scan;
}
#line 200 "scanner.cc"
yy37: ++YYCURSOR;
goto yy38;
yy38:
#line 159 "scanner.re"
{ std::cerr << "unexpected character: " << *tok << std::endl;
goto scan;
}
#line 208 "scanner.cc"
yy39: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy40;
yy40: if(yych == '\t') goto yy39;
if(yych == ' ') goto yy39;
goto yy34;
yy41: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy42;
yy42: if(yych <= '@'){
if(yych <= '/') goto yy32;
if(yych <= '9') goto yy41;
goto yy32;
} else {
if(yych <= 'Z') goto yy41;
if(yych <= '`') goto yy32;
if(yych <= 'z') goto yy41;
goto yy32;
}
yy43: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy44;
yy44: if(yych <= '['){
if(yych != '\n') goto yy43;
goto yy45;
} else {
if(yych <= '\\') goto yy46;
if(yych <= ']') goto yy47;
goto yy43;
}
yy45: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy18;
case 1: goto yy24;
case 2: goto yy26;
case 3: goto yy28;
}
yy46: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy45;
goto yy43;
yy47: ++YYCURSOR;
goto yy48;
yy48:
#line 125 "scanner.re"
{ cur = cursor;
yylval.regexp = ranToRE(token());
return RANGE; }
#line 265 "scanner.cc"
yy49: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy50;
yy50: if(yych <= '&'){
if(yych == '\n') goto yy45;
goto yy49;
} else {
if(yych <= '\'') goto yy52;
if(yych != '\\') goto yy49;
goto yy51;
}
yy51: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy45;
goto yy49;
yy52: ++YYCURSOR;
goto yy53;
yy53:
#line 118 "scanner.re"
{ cur = cursor;
yylval.regexp = strToCaseInsensitiveRE(token());
return STRING; }
#line 292 "scanner.cc"
yy54: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy55;
yy55: if(yych <= '!'){
if(yych == '\n') goto yy45;
goto yy54;
} else {
if(yych <= '"') goto yy57;
if(yych != '\\') goto yy54;
goto yy56;
}
yy56: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy45;
goto yy54;
yy57: ++YYCURSOR;
goto yy58;
yy58:
#line 114 "scanner.re"
{ cur = cursor;
yylval.regexp = strToRE(token());
return STRING; }
#line 319 "scanner.cc"
yy59: ++YYCURSOR;
goto yy60;
yy60:
#line 111 "scanner.re"
{ tok = cursor;
RETURN(0); }
#line 326 "scanner.cc"
yy61: ++YYCURSOR;
goto yy62;
yy62:
#line 108 "scanner.re"
{ depth = 1;
goto comment; }
#line 333 "scanner.cc"
yy63: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy64;
yy64: if(yych <= '/'){
if(yych == ',') goto yy67;
goto yy45;
} else {
if(yych <= '9') goto yy63;
if(yych != '}') goto yy45;
goto yy65;
}
yy65: ++YYCURSOR;
goto yy66;
yy66:
#line 136 "scanner.re"
{ yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = atoi((char *)tok+1);
RETURN(CLOSESIZE); }
#line 354 "scanner.cc"
yy67: yych = *++YYCURSOR;
if(yych != '}') goto yy71;
goto yy68;
yy68: ++YYCURSOR;
goto yy69;
yy69:
#line 144 "scanner.re"
{ yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = -1;
RETURN(CLOSESIZE); }
#line 365 "scanner.cc"
yy70: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy71;
yy71: if(yych <= '/') goto yy45;
if(yych <= '9') goto yy70;
if(yych != '}') goto yy45;
goto yy72;
yy72: ++YYCURSOR;
goto yy73;
yy73:
#line 140 "scanner.re"
{ yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = MAX(yylval.extop.minsize,atoi(strchr((char *)tok, ',')+1));
RETURN(CLOSESIZE); }
#line 382 "scanner.cc"
}
#line 162 "scanner.re"
code:
#line 386 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy74;
yy75: ++YYCURSOR;
yy74:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '&'){
if(yych <= '\n'){
if(yych <= '\t') goto yy82;
goto yy80;
} else {
if(yych == '"') goto yy84;
goto yy82;
}
} else {
if(yych <= '{'){
if(yych <= '\'') goto yy85;
if(yych <= 'z') goto yy82;
goto yy78;
} else {
if(yych != '}') goto yy82;
goto yy76;
}
}
yy76: ++YYCURSOR;
goto yy77;
yy77:
#line 166 "scanner.re"
{ if(--depth == 0){
cur = cursor;
yylval.token = new Token(token(), tline);
return CODE;
}
goto code; }
#line 423 "scanner.cc"
yy78: ++YYCURSOR;
goto yy79;
yy79:
#line 172 "scanner.re"
{ ++depth;
goto code; }
#line 430 "scanner.cc"
yy80: ++YYCURSOR;
goto yy81;
yy81:
#line 174 "scanner.re"
{ if(cursor == eof) fatal("missing '}'");
pos = cursor; cline++;
goto code;
}
#line 439 "scanner.cc"
yy82: ++YYCURSOR;
goto yy83;
yy83:
#line 178 "scanner.re"
{ goto code; }
#line 445 "scanner.cc"
yy84: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych == '\n') goto yy83;
goto yy91;
yy85: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych == '\n') goto yy83;
goto yy87;
yy86: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy87;
yy87: if(yych <= '&'){
if(yych != '\n') goto yy86;
goto yy88;
} else {
if(yych <= '\'') goto yy82;
if(yych == '\\') goto yy89;
goto yy86;
}
yy88: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy83;
}
yy89: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy88;
goto yy86;
yy90: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy91;
yy91: if(yych <= '!'){
if(yych == '\n') goto yy88;
goto yy90;
} else {
if(yych <= '"') goto yy82;
if(yych != '\\') goto yy90;
goto yy92;
}
yy92: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy88;
goto yy90;
}
#line 179 "scanner.re"
comment:
#line 499 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy93;
yy94: ++YYCURSOR;
yy93:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= ')'){
if(yych == '\n') goto yy98;
goto yy100;
} else {
if(yych <= '*') goto yy95;
if(yych == '/') goto yy97;
goto yy100;
}
yy95: ++YYCURSOR;
if((yych = *YYCURSOR) == '/') goto yy103;
goto yy96;
yy96:
#line 193 "scanner.re"
{ goto comment; }
#line 522 "scanner.cc"
yy97: yych = *++YYCURSOR;
if(yych == '*') goto yy101;
goto yy96;
yy98: ++YYCURSOR;
goto yy99;
yy99:
#line 189 "scanner.re"
{ if(cursor == eof) RETURN(0);
tok = pos = cursor; cline++;
goto comment;
}
#line 534 "scanner.cc"
yy100: yych = *++YYCURSOR;
goto yy96;
yy101: ++YYCURSOR;
goto yy102;
yy102:
#line 187 "scanner.re"
{ ++depth;
goto comment; }
#line 543 "scanner.cc"
yy103: ++YYCURSOR;
goto yy104;
yy104:
#line 183 "scanner.re"
{ if(--depth == 0)
goto scan;
else
goto comment; }
#line 552 "scanner.cc"
}
#line 194 "scanner.re"
}
void Scanner::fatal(char *msg){
std::cerr << "line " << tline << ", column " << (tchar + 1) << ": "
<< msg << std::endl;
exit(1);
}

34
tools/re2c/scanner.h Normal file
View file

@ -0,0 +1,34 @@
/* $Id: scanner.h,v 1.5 2004/05/13 02:58:18 nuffer Exp $ */
#ifndef _scanner_h
#define _scanner_h
#include <iosfwd>
#include "token.h"
class Scanner {
private:
std::istream& in;
char *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint tchar, tline, cline;
private:
char *fill(char*);
Scanner(const Scanner&); //unimplemented
Scanner& operator=(const Scanner&); //unimplemented
public:
Scanner(std::istream&);
int echo(std::ostream&);
int scan();
void fatal(char*);
SubStr token();
uint line();
};
inline SubStr Scanner::token(){
return SubStr(tok, cur - tok);
}
inline uint Scanner::line(){
return cline;
}
#endif

201
tools/re2c/scanner.re Normal file
View file

@ -0,0 +1,201 @@
/* $Id: scanner.re,v 1.10 2004/05/13 02:58:18 nuffer Exp $ */
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include "scanner.h"
#include "parser.h"
#include "y.tab.h"
extern YYSTYPE yylval;
#ifndef MAX
#define MAX(a,b) (((a)>(b))?(a):(b))
#endif
#define BSIZE 8192
#define YYCTYPE char
#define YYCURSOR cursor
#define YYLIMIT lim
#define YYMARKER ptr
#define YYFILL(n) {cursor = fill(cursor);}
#define RETURN(i) {cur = cursor; return i;}
Scanner::Scanner(std::istream& i) : in(i),
bot(NULL), tok(NULL), ptr(NULL), cur(NULL), pos(NULL), lim(NULL),
top(NULL), eof(NULL), tchar(0), tline(0), cline(1) {
;
}
char *Scanner::fill(char *cursor){
if(!eof){
uint cnt = tok - bot;
if(cnt){
memcpy(bot, tok, lim - tok);
tok = bot;
ptr -= cnt;
cursor -= cnt;
pos -= cnt;
lim -= cnt;
}
if((top - lim) < BSIZE){
char *buf = new char[(lim - bot) + BSIZE];
memcpy(buf, tok, lim - tok);
tok = buf;
ptr = &buf[ptr - bot];
cursor = &buf[cursor - bot];
pos = &buf[pos - bot];
lim = &buf[lim - bot];
top = &lim[BSIZE];
delete [] bot;
bot = buf;
}
if((cnt = in.rdbuf()->sgetn((char*) lim, BSIZE)) != BSIZE){
eof = &lim[cnt]; *eof++ = '\n';
}
lim += cnt;
}
return cursor;
}
/*!re2c
any = [\000-\377];
dot = any \ [\n];
esc = dot \ [\\];
cstring = "[" ((esc \ [\]]) | "\\" dot)* "]" ;
dstring = "\"" ((esc \ ["] ) | "\\" dot)* "\"";
sstring = "'" ((esc \ ['] ) | "\\" dot)* "'" ;
letter = [a-zA-Z];
digit = [0-9];
*/
int Scanner::echo(std::ostream &out){
char *cursor = cur;
// Catch EOF
if (eof && cursor == eof)
return 0;
tok = cursor;
echo:
/*!re2c
"/*!re2c" { out.write((const char*)(tok), (const char*)(&cursor[-7]) - (const char*)(tok));
tok = cursor;
RETURN(1); }
"\n" { if(cursor == eof) RETURN(0);
out.write((const char*)(tok), (const char*)(cursor) - (const char*)(tok));
tok = pos = cursor; cline++;
goto echo; }
any { goto echo; }
*/
}
int Scanner::scan(){
char *cursor = cur;
uint depth;
scan:
tchar = cursor - pos;
tline = cline;
tok = cursor;
/*!re2c
"{" { depth = 1;
goto code;
}
"/*" { depth = 1;
goto comment; }
"*/" { tok = cursor;
RETURN(0); }
dstring { cur = cursor;
yylval.regexp = strToRE(token());
return STRING; }
sstring { cur = cursor;
yylval.regexp = strToCaseInsensitiveRE(token());
return STRING; }
"\"" { fatal("unterminated string constant (missing \")"); }
"'" { fatal("unterminated string constant (missing ')"); }
cstring { cur = cursor;
yylval.regexp = ranToRE(token());
return RANGE; }
"[" { fatal("unterminated range (missing ])"); }
[()|=;/\\] { RETURN(*tok); }
[*+?] { yylval.op = *tok;
RETURN(CLOSE); }
"{" [0-9]+ "}" { yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = atoi((char *)tok+1);
RETURN(CLOSESIZE); }
"{" [0-9]+ "," [0-9]+ "}" { yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = MAX(yylval.extop.minsize,atoi(strchr((char *)tok, ',')+1));
RETURN(CLOSESIZE); }
"{" [0-9]+ ",}" { yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = -1;
RETURN(CLOSESIZE); }
letter (letter|digit)* { cur = cursor;
yylval.symbol = Symbol::find(token());
return ID; }
[ \t]+ { goto scan; }
"\n" { if(cursor == eof) RETURN(0);
pos = cursor; cline++;
goto scan;
}
any { std::cerr << "unexpected character: " << *tok << std::endl;
goto scan;
}
*/
code:
/*!re2c
"}" { if(--depth == 0){
cur = cursor;
yylval.token = new Token(token(), tline);
return CODE;
}
goto code; }
"{" { ++depth;
goto code; }
"\n" { if(cursor == eof) fatal("missing '}'");
pos = cursor; cline++;
goto code;
}
dstring | sstring | any { goto code; }
*/
comment:
/*!re2c
"*/" { if(--depth == 0)
goto scan;
else
goto comment; }
"/*" { ++depth;
goto comment; }
"\n" { if(cursor == eof) RETURN(0);
tok = pos = cursor; cline++;
goto comment;
}
any { goto comment; }
*/
}
void Scanner::fatal(char *msg){
std::cerr << "line " << tline << ", column " << (tchar + 1) << ": "
<< msg << std::endl;
exit(1);
}

0
tools/re2c/stamp-h.in Normal file
View file

37
tools/re2c/substr.cc Normal file
View file

@ -0,0 +1,37 @@
/* $Id: substr.cc,v 1.4 2004/05/13 02:58:18 nuffer Exp $ */
#include <string.h>
#include "substr.h"
#include "globals.h"
void SubStr::out(std::ostream& o) const {
o.write(str, len);
for (size_t i = 0; i < (size_t)len; ++i)
{
if (str[i] == '\n')
++oline;
}
}
bool operator==(const SubStr &s1, const SubStr &s2){
return (bool) (s1.len == s2.len && memcmp(s1.str, s2.str, s1.len) == 0);
}
Str::Str(const SubStr& s) : SubStr(new char[s.len], s.len) {
memcpy(str, s.str, s.len);
}
Str::Str(Str& s) : SubStr(s.str, s.len) {
s.str = NULL;
s.len = 0;
}
Str::Str() : SubStr((char*) NULL, 0) {
;
}
Str::~Str() {
delete str;
str = (char*)-1;
len = (uint)-1;
}

46
tools/re2c/substr.h Normal file
View file

@ -0,0 +1,46 @@
/* $Id: substr.h,v 1.3 2004/05/13 02:58:18 nuffer Exp $ */
#ifndef _substr_h
#define _substr_h
#include <iostream>
#include "basics.h"
class SubStr {
public:
char *str;
uint len;
public:
friend bool operator==(const SubStr &, const SubStr &);
SubStr(uchar*, uint);
SubStr(char*, uint);
SubStr(const SubStr&);
void out(std::ostream&) const;
};
class Str: public SubStr {
public:
Str(const SubStr&);
Str(Str&);
Str();
~Str();
};
inline std::ostream& operator<<(std::ostream& o, const SubStr &s){
s.out(o);
return o;
}
inline std::ostream& operator<<(std::ostream& o, const SubStr* s){
return o << *s;
}
inline SubStr::SubStr(uchar *s, uint l)
: str((char*) s), len(l) { }
inline SubStr::SubStr(char *s, uint l)
: str(s), len(l) { }
inline SubStr::SubStr(const SubStr &s)
: str(s.str), len(s.len) { }
#endif

162
tools/re2c/test.c Normal file
View file

@ -0,0 +1,162 @@
/* Generated by re2c 0.9.3 on Mon Feb 14 13:46:36 2005 */
#line 1 "test.re"
#line 6 "test.c"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 4) YYFILL(4);
yych = *YYCURSOR;
switch(yych){
case '.': goto yy4;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy2;
default: goto yy6;
}
yy2: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
goto yy16;
yy3:
#line 4 "test.re"
{ return true; }
#line 35 "test.c"
yy4: ++YYCURSOR;
switch((yych = *YYCURSOR)) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy7;
default: goto yy5;
}
yy5:
#line 5 "test.re"
{ return false; }
#line 53 "test.c"
yy6: yych = *++YYCURSOR;
goto yy5;
yy7: yyaccept = 0;
YYMARKER = ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy8;
yy8: switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy7;
case 'D':
case 'E': case 'd':
case 'e': goto yy9;
default: goto yy3;
}
yy9: yych = *++YYCURSOR;
switch(yych){
case '+': case '-': goto yy11;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy12;
default: goto yy10;
}
yy10: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy3;
}
yy11: yych = *++YYCURSOR;
switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy12;
default: goto yy10;
}
yy12: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy13;
yy13: switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy12;
default: goto yy3;
}
yy14: yych = *++YYCURSOR;
switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy7;
default: goto yy10;
}
yy15: yyaccept = 0;
YYMARKER = ++YYCURSOR;
if((YYLIMIT - YYCURSOR) < 3) YYFILL(3);
yych = *YYCURSOR;
goto yy16;
yy16: switch(yych){
case '.': goto yy14;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy15;
case 'D':
case 'E': case 'd':
case 'e': goto yy9;
default: goto yy3;
}
}
#line 6 "test.re"

6
tools/re2c/test.re Normal file
View file

@ -0,0 +1,6 @@
/*!re2c
digits = [0-9];
(digits+ | digits* "." digits+) ([dDeE] [+-]? digits+)? { return true; }
[\000-\377] { return false; }
*/

19
tools/re2c/token.h Normal file
View file

@ -0,0 +1,19 @@
/* $Id: token.h,v 1.2 2004/01/31 15:44:39 nuffer Exp $ */
#ifndef _token_h
#define _token_h
#include "substr.h"
class Token {
public:
Str text;
uint line;
public:
Token(SubStr, uint);
};
inline Token::Token(SubStr t, uint l) : text(t), line(l) {
;
}
#endif

62
tools/re2c/translate.cc Normal file
View file

@ -0,0 +1,62 @@
/* $Id: translate.cc,v 1.2 2004/01/31 15:44:39 nuffer Exp $ */
#include "globals.h"
uchar asc2asc[256] = {
0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,
0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,
0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,
0x40,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,
0x50,0x51,0x52,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,
0x60,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,
0x80,0x81,0x82,0x83,0x84,0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,
0xa0,0xa1,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,
0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,
0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,
0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,
0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,
0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff
};
uchar *xlat = asc2asc;
uchar *talx = asc2asc;
uchar asc2ebc[256] = { /* Based on ISO 8859/1 and Code Page 37 */
0x00,0x01,0x02,0x03,0x37,0x2d,0x2e,0x2f,0x16,0x05,0x25,0x0b,0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13,0x3c,0x3d,0x32,0x26,0x18,0x19,0x3f,0x27,0x1c,0x1d,0x1e,0x1f,
0x40,0x5a,0x7f,0x7b,0x5b,0x6c,0x50,0x7d,0x4d,0x5d,0x5c,0x4e,0x6b,0x60,0x4b,0x61,
0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0x7a,0x5e,0x4c,0x7e,0x6e,0x6f,
0x7c,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,
0xd7,0xd8,0xd9,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xba,0xe0,0xbb,0xb0,0x6d,
0x79,0x81,0x82,0x83,0x84,0x85,0x86,0x87,0x88,0x89,0x91,0x92,0x93,0x94,0x95,0x96,
0x97,0x98,0x99,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xc0,0x4f,0xd0,0xa1,0x07,
0x20,0x21,0x22,0x23,0x24,0x15,0x06,0x17,0x28,0x29,0x2a,0x2b,0x2c,0x09,0x0a,0x1b,
0x30,0x31,0x1a,0x33,0x34,0x35,0x36,0x08,0x38,0x39,0x3a,0x3b,0x04,0x14,0x3e,0xff,
0x41,0xaa,0x4a,0xb1,0x9f,0xb2,0x6a,0xb5,0xbd,0xb4,0x9a,0x8a,0x5f,0xca,0xaf,0xbc,
0x90,0x8f,0xea,0xfa,0xbe,0xa0,0xb6,0xb3,0x9d,0xda,0x9b,0x8b,0xb7,0xb8,0xb9,0xab,
0x64,0x65,0x62,0x66,0x63,0x67,0x9e,0x68,0x74,0x71,0x72,0x73,0x78,0x75,0x76,0x77,
0xac,0x69,0xed,0xee,0xeb,0xef,0xec,0xbf,0x80,0xfd,0xfe,0xfb,0xfc,0xad,0x8e,0x59,
0x44,0x45,0x42,0x46,0x43,0x47,0x9c,0x48,0x54,0x51,0x52,0x53,0x58,0x55,0x56,0x57,
0x8c,0x49,0xcd,0xce,0xcb,0xcf,0xcc,0xe1,0x70,0xdd,0xde,0xdb,0xdc,0x8d,0xae,0xdf
};
uchar ebc2asc[256] = { /* Based on ISO 8859/1 and Code Page 37 */
0x00,0x01,0x02,0x03,0x9c,0x09,0x86,0x7f,0x97,0x8d,0x8e,0x0b,0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13,0x9d,0x85,0x08,0x87,0x18,0x19,0x92,0x8f,0x1c,0x1d,0x1e,0x1f,
0x80,0x81,0x82,0x83,0x84,0x0a,0x17,0x1b,0x88,0x89,0x8a,0x8b,0x8c,0x05,0x06,0x07,
0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9a,0x9b,0x14,0x15,0x9e,0x1a,
0x20,0xa0,0xe2,0xe4,0xe0,0xe1,0xe3,0xe5,0xe7,0xf1,0xa2,0x2e,0x3c,0x28,0x2b,0x7c,
0x26,0xe9,0xea,0xeb,0xe8,0xed,0xee,0xef,0xec,0xdf,0x21,0x24,0x2a,0x29,0x3b,0xac,
0x2d,0x2f,0xc2,0xc4,0xc0,0xc1,0xc3,0xc5,0xc7,0xd1,0xa6,0x2c,0x25,0x5f,0x3e,0x3f,
0xf8,0xc9,0xca,0xcb,0xc8,0xcd,0xce,0xcf,0xcc,0x60,0x3a,0x23,0x40,0x27,0x3d,0x22,
0xd8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xab,0xbb,0xf0,0xfd,0xde,0xb1,
0xb0,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0xaa,0xba,0xe6,0xb8,0xc6,0xa4,
0xb5,0x7e,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0xa1,0xbf,0xd0,0xdd,0xfe,0xae,
0x5e,0xa3,0xa5,0xb7,0xa9,0xa7,0xb6,0xbc,0xbd,0xbe,0x5b,0x5d,0xaf,0xa8,0xb4,0xd7,
0x7b,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xad,0xf4,0xf6,0xf2,0xf3,0xf5,
0x7d,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0xb9,0xfb,0xfc,0xf9,0xfa,0xff,
0x5c,0xf7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0xb2,0xd4,0xd6,0xd2,0xd3,0xd5,
0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xb3,0xdb,0xdc,0xd9,0xda,0x9f
};

69
tools/re2c/y.tab.h Normal file
View file

@ -0,0 +1,69 @@
/* A Bison parser, made by GNU Bison 1.875. */
/* Skeleton parser for Yacc-like parsing with Bison,
Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002 Free Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA. */
/* As a special exception, when this file is copied by Bison into a
Bison output file, you may use that output file without restriction.
This special exception was added by the Free Software Foundation
in version 1.24 of Bison. */
/* Tokens. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
/* Put the tokens into the symbol table, so that GDB and other debuggers
know about them. */
enum yytokentype {
CLOSESIZE = 258,
CLOSE = 259,
ID = 260,
CODE = 261,
RANGE = 262,
STRING = 263
};
#endif
#define CLOSESIZE 258
#define CLOSE 259
#define ID 260
#define CODE 261
#define RANGE 262
#define STRING 263
#if ! defined (YYSTYPE) && ! defined (YYSTYPE_IS_DECLARED)
#line 53 "parser.y"
typedef union YYSTYPE {
Symbol *symbol;
RegExp *regexp;
Token *token;
char op;
ExtOp extop;
} YYSTYPE;
/* Line 1240 of yacc.c. */
#line 60 "y.tab.h"
# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
# define YYSTYPE_IS_TRIVIAL 1
#endif
extern YYSTYPE yylval;

30
tools/xlatcc/Makefile.mgw Normal file
View file

@ -0,0 +1,30 @@
EXE = xlatcc.exe
CCDV = @../../ccdv
CC = gcc
CFLAGS = $(LOC) -Os -Wall -fomit-frame-pointer
BISON = bison
BISONFLAGS =
OBJS = gen.o xlat-parse.tab.o
all: $(EXE)
$(EXE): $(OBJS)
$(CCDV) $(CC) -o $(EXE) $(OBJS)
.PHONY: clean
clean:
-del /q /f $(EXE) 2>nul
-del /q /f *.o 2>nul
xlat-parse.tab.o: xlat-parse.tab.c xlat.h
$(CCDV) $(CC) $(CFLAGS) -c -o $@ $<
xlat-parse.tab.c: xlat-parse.y
$(CCDV) $(BISON) $(BISONFLAGS) -o $@ $<
gen.o: gen.c xlat.h
$(CCDV) $(CC) $(CFLAGS) -c -o $@ $<

272
tools/xlatcc/gen.c Normal file
View file

@ -0,0 +1,272 @@
#include <stdio.h>
#include <string.h>
#include <malloc.h>
#include "xlat.h"
#define FILE_STACK 32
SimpleTranslator Simple[65536];
BoomTranslator Boomish[MAX_BOOMISH];
int NumBoomish;
FILE *Source;
char *SourceName;
int SourceLine;
static FILE *SourceStack[FILE_STACK];
static char *NameStack[FILE_STACK];
static int LineStack[FILE_STACK];
static int SourceStackSize;
int yyparse ();
int CountSimpleTranslators (void)
{
int i, count;
for (i = 1, count = 0; i < 65536; i++)
{
if (Simple[i].NewSpecial != 0)
{
count++;
for ( ; i < 65536 && Simple[i].NewSpecial != 0; i++)
;
}
}
return count;
}
void WriteSimpleTranslators (FILE *file)
{
int i, start, count;
for (count = 0, i = 1; i < 65536; i++)
{
if (Simple[i].NewSpecial != 0)
{
start = i;
for ( ; i < 65536; i++)
{
if (Simple[i].NewSpecial == 0)
{
break;
}
}
i--;
// start = first linetype #
// i = last linetype #
putc (start >> 8, file);
putc (start & 255, file);
putc (i >> 8, file);
putc (i & 255, file);
for (; start <= i; start++)
{
putc (Simple[start].Flags, file);
putc (Simple[start].NewSpecial, file);
putc (Simple[start].Args[0], file);
putc (Simple[start].Args[1], file);
putc (Simple[start].Args[2], file);
putc (Simple[start].Args[3], file);
putc (Simple[start].Args[4], file);
count++;
}
}
}
printf ("Wrote %d normal linetypes\n", count);
}
// BOOM translators are stored on disk as:
//
// WORD <first linetype in range>
// WORD <last linetype in range>
// BYTE <new special>
// repeat [BYTE op BYTES parms]
//
// op consists of some bits:
//
// 76543210
// ||||||++-- Dest is arg[(op&3)+1] (arg[0] is always tag)
// |||||+---- 0 = store, 1 = or with existing value
// ||||+----- 0 = this is normal, 1 = x-op in next byte
// ++++------ # of elements in list, or 0 to always use a constant value
//
// If a constant value is used, parms is a single byte containing that value.
// Otherwise, parms has the format:
//
// WORD <value to AND with linetype>
// repeat [WORD <if result is this> BYTE <use this>]
//
// These x-ops are defined:
//
// 0 = end of this BOOM translator
// 1 = dest is flags
void WriteBoomTranslators (FILE *file)
{
int i, j, k;
for (i = 0; i < NumBoomish; i++)
{
putc (Boomish[i].FirstLinetype >> 8, file);
putc (Boomish[i].FirstLinetype & 255, file);
putc (Boomish[i].LastLinetype >> 8, file);
putc (Boomish[i].LastLinetype & 255, file);
putc (Boomish[i].NewSpecial, file);
for (j = 0; j < MAX_BOOMISH_EXEC; j++)
{
if (Boomish[i].Args[j].bDefined)
{
BYTE op;
if (Boomish[i].Args[j].ArgNum < 4)
{
op = Boomish[i].Args[j].ArgNum;
}
else
{
op = 8;
}
if (Boomish[i].Args[j].bOrExisting)
{
op |= 4;
}
if (!Boomish[i].Args[j].bUseConstant)
{
op |= Boomish[i].Args[j].ListSize << 4;
}
putc (op, file);
if (op & 8)
{
putc (Boomish[i].Args[j].ArgNum - 3, file);
}
if (Boomish[i].Args[j].bUseConstant)
{
putc (Boomish[i].Args[j].ConstantValue, file);
}
else
{
putc (Boomish[i].Args[j].AndValue >> 8, file);
putc (Boomish[i].Args[j].AndValue & 255, file);
for (k = 0; k < Boomish[i].Args[j].ListSize; k++)
{
putc (Boomish[i].Args[j].ResultFilter[k] >> 8, file);
putc (Boomish[i].Args[j].ResultFilter[k] & 255, file);
putc (Boomish[i].Args[j].ResultValue[k], file);
}
}
}
}
putc (8, file);
putc (0, file);
}
printf ("Wrote %d BOOM-style generalized linetypes\n", NumBoomish);
}
void WriteTranslatorFile (FILE *file, int count)
{
putc ('N', file);
putc ('O', file);
putc ('R', file);
putc ('M', file);
putc (count >> 8, file);
putc (count & 255, file);
WriteSimpleTranslators (file);
putc ('B', file);
putc ('O', file);
putc ('O', file);
putc ('M', file);
putc (NumBoomish >> 8, file);
putc (NumBoomish & 255, file);
WriteBoomTranslators (file);
}
void IncludeFile (const char *name)
{
FILE *newsource;
if (Source != NULL)
{
if (SourceStackSize == FILE_STACK)
{
printf ("Too many files included. Skipping %s\n", name);
return;
}
SourceStack[SourceStackSize] = Source;
LineStack[SourceStackSize] = SourceLine;
NameStack[SourceStackSize] = SourceName;
SourceStackSize++;
}
newsource = fopen (name, "r");
if (newsource == NULL)
{
printf ("Cannot open %s\n", name);
if (Source != NULL)
{
SourceStackSize--;
}
return;
}
Source = newsource;
SourceLine = 1;
SourceName = strdup (name);
}
bool EndFile ()
{
if (Source != NULL)
{
fclose (Source);
}
if (SourceName != NULL)
{
free (SourceName);
}
if (SourceStackSize)
{
Source = SourceStack[--SourceStackSize];
SourceName = NameStack[SourceStackSize];
SourceLine = LineStack[SourceStackSize];
return 1;
}
return 0;
}
int main (int argc, char **argv)
{
FILE *output;
int count;
if (argc != 3)
{
printf ("Usage: %s <source file> <output file>\n", argv[0]);
return -1;
}
IncludeFile (argv[1]);
yyparse ();
count = CountSimpleTranslators ();
if ((count | NumBoomish) != 0)
{
output = fopen (argv[2], "wb");
if (output == NULL)
{
printf ("Could not open %s\n", argv[2]);
return -2;
}
WriteTranslatorFile (output, count);
fclose (output);
}
else
{
printf ("No linedef types to translate\n");
}
return 0;
}

2169
tools/xlatcc/xlat-parse.tab.c Normal file
View file

File diff suppressed because it is too large Load diff

850
tools/xlatcc/xlat-parse.y Normal file
View file

@ -0,0 +1,850 @@
/* You need Bison (or maybe Yacc) to rebuild the .c file from this grammar.
* Or you could rewrite it to use Lemon, since the ZDoom source comes with Lemon.
* Since I have Bison, and the existing code works fine, there's no motivation
* for me to rewrite it.
*/
%{
#include "xlat.h"
#include <malloc.h>
#include <string.h>
// verbose doesn't seem to help all that much
//#define YYERROR_VERBOSE 1
int yyerror (char *s);
int yylex (void);
typedef struct _Symbol
{
struct _Symbol *Next;
int Value;
char Sym[1];
} Symbol;
static bool FindToken (char *tok, int *type);
static void AddSym (char *sym, int val);
static bool FindSym (char *sym, Symbol **val);
static int EnumVal;
struct ListFilter
{
WORD filter;
BYTE value;
};
typedef struct _morefilters
{
struct _morefilters *next;
struct ListFilter filter;
} MoreFilters;
typedef struct _morelines
{
struct _morelines *next;
BoomArg arg;
} MoreLines;
%}
%union {
int val;
char sym[80];
char string[80];
struct
{
BYTE addflags;
BYTE args[5];
} args;
struct ListFilter filter;
MoreFilters *filter_p;
struct
{
BYTE constant;
WORD mask;
MoreFilters *filters;
} boomarg;
Symbol *symval;
BoomArg boomline;
MoreLines *boomlines;
}
%token <val> NUM
%token <symval> SYMNUM
%token <sym> SYM
%token <string> STRING
%type <val> exp
%type <args> special_args
%type <filter> list_val
%type <filter_p> arg_list
%type <boomarg> boom_args
%type <val> boom_op
%type <val> boom_selector
%type <boomline> boom_line
%type <boomlines> boom_body
%type <val> maybe_argcount
%token DEFINE
%token INCLUDE
%token TAG
%token LINEID
%token SPECIAL
%token FLAGS
%token ARG2
%token ARG3
%token ARG4
%token ARG5
%token OR_EQUAL
%token ENUM
%token PRINT
%token ENDL
%left '|'
%left '^'
%left '&'
%left '-' '+'
%left '*' '/'
%left NEG
%start translation_unit
%%
exp:
NUM
| SYMNUM { $$ = $1->Value; }
| exp '+' exp { $$ = $1 + $3; }
| exp '-' exp { $$ = $1 - $3; }
| exp '*' exp { $$ = $1 * $3; }
| exp '/' exp { $$ = $1 / $3; }
| exp '|' exp { $$ = $1 | $3; }
| exp '&' exp { $$ = $1 & $3; }
| exp '^' exp { $$ = $1 ^ $3; }
| '-' exp %prec NEG { $$ = -$2 }
| '(' exp ')' { $$ = $2; }
;
translation_unit: /* empty line */
| translation_unit external_declaration
;
external_declaration:
define_statement
| include_statement
| print_statement
| enum_statement
| linetype_declaration
| boom_declaration
| special_declaration
;
print_statement:
PRINT '(' print_list ')'
{
printf ("\n");
}
;
print_list:
/* EMPTY */
| print_item
| print_item ',' print_list
;
print_item:
STRING { printf ("%s", $1); }
| exp { printf ("%d", $1); }
| ENDL { printf ("\n"); }
;
define_statement:
DEFINE SYM '(' exp ')' { AddSym ($2, $4); }
;
include_statement:
INCLUDE STRING { IncludeFile ($2); }
;
enum_statement:
ENUM '{' { EnumVal = 0; } enum_list '}'
;
enum_list:
/* empty */
| single_enum
| single_enum ',' enum_list
;
single_enum:
SYM { AddSym ($1, EnumVal++); }
| SYM '=' exp { AddSym ($1, EnumVal = $3); }
;
/* special declarations work just like they do for ACS, so
* specials can be defined just by including zspecial.acs
*/
special_declaration:
SPECIAL special_list ';'
;
special_list:
/* empty */
| special_def
| special_def ',' special_list
;
special_def:
exp ':' SYM '(' maybe_argcount ')' { AddSym ($3, $1); }
| exp ':' SYMNUM
{ printf ("%s, line %d: %s is already defined\n", SourceName, SourceLine, $3->Sym); }
'(' maybe_argcount ')'
;
maybe_argcount:
/* empty */ { $$ = 0; }
| exp
| exp ',' exp
;
linetype_declaration:
exp '=' exp ',' exp '(' special_args ')'
{
Simple[$1].NewSpecial = $5;
Simple[$1].Flags = $3 | $7.addflags;
Simple[$1].Args[0] = $7.args[0];
Simple[$1].Args[1] = $7.args[1];
Simple[$1].Args[2] = $7.args[2];
Simple[$1].Args[3] = $7.args[3];
Simple[$1].Args[4] = $7.args[4];
}
| exp '=' exp ',' SYM '(' special_args ')'
{
printf ("%s, line %d: %s is undefined\n", SourceName, SourceLine, $5);
}
;
boom_declaration:
'[' exp ']' '(' exp ',' exp ')' '{' boom_body '}'
{
if (NumBoomish == MAX_BOOMISH)
{
MoreLines *probe = $10;
while (probe != NULL)
{
MoreLines *next = probe->next;
free (probe);
probe = next;
}
printf ("%s, line %d: Too many BOOM translators\n", SourceName, SourceLine);
}
else
{
int i;
MoreLines *probe;
Boomish[NumBoomish].FirstLinetype = $5;
Boomish[NumBoomish].LastLinetype = $7;
Boomish[NumBoomish].NewSpecial = $2;
for (i = 0, probe = $10; probe != NULL; i++)
{
MoreLines *next = probe->next;;
if (i < MAX_BOOMISH_EXEC)
{
Boomish[NumBoomish].Args[i] = probe->arg;
}
else if (i == MAX_BOOMISH_EXEC)
{
printf ("%s, line %d: Too many commands for this BOOM translator\n", SourceName, SourceLine);
}
free (probe);
probe = next;
}
if (i < MAX_BOOMISH_EXEC)
{
Boomish[NumBoomish].Args[i].bDefined = 0;
}
NumBoomish++;
}
}
;
boom_body:
/* empty */
{
$$ = NULL;
}
| boom_line boom_body
{
$$ = malloc (sizeof(MoreLines));
$$->next = $2;
$$->arg = $1;
}
;
boom_line:
boom_selector boom_op boom_args
{
$$.bDefined = 1;
$$.bOrExisting = ($2 == OR_EQUAL);
$$.bUseConstant = ($3.filters == NULL);
$$.ArgNum = $1;
$$.ConstantValue = $3.constant;
$$.AndValue = $3.mask;
if ($3.filters != NULL)
{
int i;
MoreFilters *probe;
for (i = 0, probe = $3.filters; probe != NULL; i++)
{
MoreFilters *next = probe->next;
if (i < 15)
{
$$.ResultFilter[i] = probe->filter.filter;
$$.ResultValue[i] = probe->filter.value;
}
else if (i == 15)
{
yyerror ("Lists can only have 15 elements");
}
free (probe);
probe = next;
}
$$.ListSize = i > 15 ? 15 : i;
}
}
;
boom_selector:
FLAGS { $$ = 4; }
| ARG2 { $$ = 0; }
| ARG3 { $$ = 1; }
| ARG4 { $$ = 2; }
| ARG5 { $$ = 3; }
;
boom_op:
'=' { $$ = '='; }
| OR_EQUAL { $$ = OR_EQUAL; }
;
boom_args:
exp
{
$$.constant = $1;
$$.filters = NULL;
}
| exp '[' arg_list ']'
{
$$.mask = $1;
$$.filters = $3;
}
;
arg_list:
list_val
{
$$ = malloc (sizeof (MoreFilters));
$$->next = NULL;
$$->filter = $1;
}
| list_val ',' arg_list
{
$$ = malloc (sizeof (MoreFilters));
$$->next = $3;
$$->filter = $1;
}
;
list_val:
exp ':' exp { $$.filter = $1; $$.value = $3; }
;
special_args:
/* EMPTY LINE */
{
$$.addflags = 0;
memset ($$.args, 0, 5);
}
| TAG
{
$$.addflags = SIMPLE_HASTAGAT1;
memset ($$.args, 0, 5);
}
| TAG ',' exp
{
$$.addflags = SIMPLE_HASTAGAT1;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| TAG ',' exp ',' exp
{
$$.addflags = SIMPLE_HASTAGAT1;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = 0;
}
| TAG ',' exp ',' exp ',' exp
{
$$.addflags = SIMPLE_HASTAGAT1;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = 0;
}
| TAG ',' exp ',' exp ',' exp ',' exp
{
$$.addflags = SIMPLE_HASTAGAT1;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = $9;
}
| TAG ',' TAG
{
$$.addflags = SIMPLE_HAS2TAGS;
$$.args[0] = $$.args[1] = 0;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| TAG ',' TAG ',' exp
{
$$.addflags = SIMPLE_HAS2TAGS;
$$.args[0] = $$.args[1] = 0;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = 0;
}
| TAG ',' TAG ',' exp ',' exp
{
$$.addflags = SIMPLE_HAS2TAGS;
$$.args[0] = $$.args[1] = 0;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = 0;
}
| TAG ',' TAG ',' exp ',' exp ',' exp
{
$$.addflags = SIMPLE_HAS2TAGS;
$$.args[0] = $$.args[1] = 0;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = $9;
}
| LINEID
{
$$.addflags = SIMPLE_HASLINEID;
memset ($$.args, 0, 5);
}
| LINEID ',' exp
{
$$.addflags = SIMPLE_HASLINEID;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| LINEID ',' exp ',' exp
{
$$.addflags = SIMPLE_HASLINEID;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = 0;
}
| LINEID ',' exp ',' exp ',' exp
{
$$.addflags = SIMPLE_HASLINEID;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = 0;
}
| LINEID ',' exp ',' exp ',' exp ',' exp
{
$$.addflags = SIMPLE_HASLINEID;
$$.args[0] = 0;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = $9;
}
| exp
{
$$.addflags = 0;
$$.args[0] = $1;
$$.args[1] = 0;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' exp
{
$$.addflags = 0;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' exp ',' exp
{
$$.addflags = 0;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' exp ',' exp ',' exp
{
$$.addflags = 0;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = 0;
}
| exp ',' exp ',' exp ',' exp ',' exp
{
$$.addflags = 0;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = $9;
}
| exp ',' TAG
{
$$.addflags = SIMPLE_HASTAGAT2;
$$.args[0] = $1;
$$.args[1] = 0;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' TAG ',' exp
{
$$.addflags = SIMPLE_HASTAGAT2;
$$.args[0] = $1;
$$.args[1] = 0;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' TAG ',' exp ',' exp
{
$$.addflags = SIMPLE_HASTAGAT2;
$$.args[0] = $1;
$$.args[1] = 0;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = 0;
}
| exp ',' TAG ',' exp ',' exp ',' exp
{
$$.addflags = SIMPLE_HASTAGAT2;
$$.args[0] = $1;
$$.args[1] = 0;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = $9;
}
| exp ',' exp ',' TAG
{
$$.addflags = SIMPLE_HASTAGAT3;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = 0;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' exp ',' TAG ',' exp
{
$$.addflags = SIMPLE_HASTAGAT3;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = 0;
$$.args[3] = $7;
$$.args[4] = 0;
}
| exp ',' exp ',' TAG ',' exp ',' exp
{
$$.addflags = SIMPLE_HASTAGAT3;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = 0;
$$.args[3] = $7;
$$.args[4] = $9;
}
| exp ',' exp ',' exp ',' TAG
{
$$.addflags = SIMPLE_HASTAGAT4;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = 0;
}
| exp ',' exp ',' exp ',' TAG ',' exp
{
$$.addflags = SIMPLE_HASTAGAT4;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = 0;
$$.args[4] = $9;
}
| exp ',' exp ',' exp ',' exp ',' TAG
{
$$.addflags = SIMPLE_HASTAGAT5;
$$.args[0] = $1;
$$.args[1] = $3;
$$.args[2] = $5;
$$.args[3] = $7;
$$.args[4] = 0;
}
;
%%
#include <ctype.h>
#include <stdio.h>
int yylex (void)
{
char token[80];
int toksize;
int buildup;
int c;
loop:
while (Source == NULL)
{
if (!EndFile ())
return 0;
}
while (isspace (c = fgetc (Source)) && c != EOF)
{
if (c == '\n')
SourceLine++;
}
if (c == EOF)
{
if (EndFile ())
goto loop;
return 0;
}
if (isdigit (c))
{
buildup = c - '0';
if (c == '0')
{
c = fgetc (Source);
if (c == 'x' || c == 'X')
{
for (;;)
{
c = fgetc (Source);
if (isdigit (c))
{
buildup = (buildup<<4) + c - '0';
}
else if (c >= 'a' && c <= 'f')
{
buildup = (buildup<<4) + c - 'a' + 10;
}
else if (c >= 'A' && c <= 'F')
{
buildup = (buildup<<4) + c - 'A' + 10;
}
else
{
ungetc (c, Source);
yylval.val = buildup;
return NUM;
}
}
}
else
{
ungetc (c, Source);
}
}
while (isdigit (c = fgetc (Source)))
{
buildup = buildup*10 + c - '0';
}
ungetc (c, Source);
yylval.val = buildup;
return NUM;
}
if (isalpha (c))
{
token[0] = c;
toksize = 1;
while (toksize < 79 && (isalnum (c = fgetc (Source)) || c == '_'))
{
token[toksize++] = c;
}
token[toksize] = 0;
if (toksize == 79 && isalnum (c))
{
while (isalnum (c = fgetc (Source)))
;
}
ungetc (c, Source);
if (FindToken (token, &buildup))
{
return buildup;
}
if (FindSym (token, &yylval.symval))
{
return SYMNUM;
}
strcpy (yylval.sym, token);
return SYM;
}
if (c == '/')
{
c = fgetc (Source);
if (c == '*')
{
for (;;)
{
while ((c = fgetc (Source)) != '*' && c != EOF)
{
if (c == '\n')
SourceLine++;
}
if (c == EOF)
return 0;
if ((c = fgetc (Source)) == '/')
goto loop;
if (c == EOF)
return 0;
ungetc (c, Source);
}
}
else if (c == '/')
{
while ((c = fgetc (Source)) != '\n' && c != EOF)
;
if (c == '\n')
SourceLine++;
else if (c == EOF)
return 0;
goto loop;
}
else
{
ungetc (c, Source);
return '/';
}
}
if (c == '"')
{
int tokensize = 0;
while ((c = fgetc (Source)) != '"' && c != EOF)
{
yylval.string[tokensize++] = c;
}
yylval.string[tokensize] = 0;
return STRING;
}
if (c == '|')
{
c = fgetc (Source);
if (c == '=')
return OR_EQUAL;
ungetc (c, Source);
return '|';
}
return c;
}
static Symbol *FirstSym;
static void AddSym (char *sym, int val)
{
Symbol *syme = malloc (strlen (sym) + sizeof(Symbol));
syme->Next = FirstSym;
syme->Value = val;
strcpy (syme->Sym, sym);
FirstSym = syme;
}
static bool FindSym (char *sym, Symbol **val)
{
Symbol *syme = FirstSym;
while (syme != NULL)
{
if (strcmp (syme->Sym, sym) == 0)
{
*val = syme;
return 1;
}
syme = syme->Next;
}
return 0;
}
static bool FindToken (char *tok, int *type)
{
static const char tokens[][8] =
{
"endl", "print", "include", "special", "define", "enum",
"arg5", "arg4", "arg3", "arg2", "flags", "lineid", "tag"
};
static const short types[] =
{
ENDL, PRINT, INCLUDE, SPECIAL, DEFINE, ENUM,
ARG5, ARG4, ARG3, ARG2, FLAGS, LINEID, TAG
};
int i;
for (i = sizeof(tokens)/sizeof(tokens[0])-1; i >= 0; i--)
{
if (strcmp (tok, tokens[i]) == 0)
{
*type = types[i];
return 1;
}
}
return 0;
}
int yyerror (char *s)
{
if (SourceName != NULL)
printf ("%s, line %d: %s\n", SourceName, SourceLine, s);
else
printf ("%s\n", s);
return 0;
}

59
tools/xlatcc/xlat.h Normal file
View file

@ -0,0 +1,59 @@
#include <stdio.h>
typedef unsigned char BYTE;
typedef unsigned short WORD;
typedef unsigned char bool;
#define MAX_BOOMISH 16
#define MAX_BOOMISH_EXEC 32
#define SIMPLE_HASTAGAT1 (1<<5) // (tag, x, x, x, x)
#define SIMPLE_HASTAGAT2 (2<<5) // (x, tag, x, x, x)
#define SIMPLE_HASTAGAT3 (3<<5) // (x, x, tag, x, x)
#define SIMPLE_HASTAGAT4 (4<<5) // (x, x, x, tag, x)
#define SIMPLE_HASTAGAT5 (5<<5) // (x, x, x, x, tag)
#define SIMPLE_HASLINEID (6<<5) // (tag, lineid, x, x, x, x)
#define SIMPLE_HAS2TAGS (7<<5) // (tag, tag, x, x, x)
typedef struct
{
BYTE Flags;
BYTE NewSpecial;
BYTE Args[5];
} SimpleTranslator;
typedef struct
{
bool bDefined;
bool bOrExisting;
bool bUseConstant;
BYTE ListSize;
BYTE ArgNum;
BYTE ConstantValue;
WORD AndValue;
WORD ResultFilter[15];
BYTE ResultValue[15];
} BoomArg;
typedef struct
{
WORD FirstLinetype;
WORD LastLinetype;
BYTE NewSpecial;
BoomArg Args[MAX_BOOMISH_EXEC];
} BoomTranslator;
extern SimpleTranslator Simple[65536];
extern BoomTranslator Boomish[MAX_BOOMISH];
extern int NumBoomish;
extern FILE *Source;
extern char *SourceName;
extern int SourceLine;
void IncludeFile (const char *name);
bool EndFile ();
int yyerror (char *s);

149
tools/xlatcc/xlatcc.vcproj Normal file
View file

@ -0,0 +1,149 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="7.10"
Name="xlatcc"
ProjectGUID="{3FFA68B3-9449-4B03-ADEE-194C3638623B}"
Keyword="Win32Proj">
<Platforms>
<Platform
Name="Win32"/>
</Platforms>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="Debug"
IntermediateDirectory="Debug"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="5"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="4"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/xlatcc.exe"
LinkIncremental="2"
GenerateDebugInformation="TRUE"
ProgramDatabaseFile="$(OutDir)/xlatcc.pdb"
SubSystem="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="copy &quot;$(TargetPath)&quot; &quot;$(ProjectDir)\$(TargetFileName)&quot;"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="Release"
IntermediateDirectory="Release"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="4"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="TRUE"
DebugInformationFormat="3"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/xlatcc.exe"
LinkIncremental="1"
GenerateDebugInformation="TRUE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"
CommandLine="copy &quot;$(TargetPath)&quot; &quot;$(ProjectDir)\$(TargetFileName)&quot;"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
<File
RelativePath=".\gen.c">
</File>
<File
RelativePath=".\xlat-parse.tab.c">
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
<File
RelativePath=".\xlat.h">
</File>
</Filter>
<Filter
Name="Grammar"
Filter="">
<File
RelativePath=".\xlat-parse.y">
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCustomBuildTool"
CommandLine="bison xlat-parse.y"
Outputs="xlat-parse.tab.c"/>
</FileConfiguration>
</File>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>