zdoom/qzdoom
0
0
Fork 0
mirror of https://github.com/ZDoom/qzdoom.git synced 2025-05-30 08:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

Directory restructuring to make it easier to version projects that don't build zdoom.exe.

Browse source 
SVN r4 (trunk)
This commit is contained in:
Randy Heit 2006-02-24 04:48:15 +00:00
commit cf11cbdb30
821 changed files with 361202 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

47
tools/re2c/Makefile.am Normal file
View file

@ -0,0 +1,47 @@
# $Id: Makefile.am,v 1.15 2004/05/26 23:59:15 nuffer Exp $
bin_PROGRAMS = re2c
re2c_SOURCES = code.cc dfa.cc main.cc parser.cc actions.cc scanner.re substr.cc\
translate.cc scanner.cc mbo_getopt.cc \
basics.h dfa.h globals.h ins.h parser.h re.h scanner.h \
substr.h token.h mbo_getopt.h
BUILT_SOURCES = parser.cc scanner.cc
man_MANS = re2c.1
#CXXFLAGS = -O2 -Wall -I. -Wno-unused -Wno-parentheses -Wno-deprecated
YFLAGS = -d
RE2C = re2c
RE2CFLAGS = -s
CLEANFILES = parser.cc y.tab.c y.tab.h parser.cc re2c.1 .version
DISTCLEANFILES = makerpm re2c.spec README scanner.cc re2c$(EXEEXT)
EXTRA_DIST = $(man_MANS) README parser.y scanner.re makerpm.in re2c.spec.in \
README.in y.tab.h \
examples/basemmap.c examples/c.re examples/cmmap.re \
examples/cnokw.re examples/cunroll.re examples/modula.re \
examples/sample.re examples/simple.re examples/rexx/README \
examples/rexx/rexx.l examples/rexx/scanio.c doc/loplas.ps \
doc/sample.bib
rpm-files: $(bin_PROGRAMS) $(EXTRA_DIST)
parser.cc: $(top_srcdir)/parser.y
$(YACC) $(YFLAGS) $<
mv -f y.tab.c $(top_srcdir)/parser.cc
if cmp -s y.tab.h $(top_srcdir)/y.tab.h; then :; else mv -f y.tab.h $(top_srcdir)/y.tab.h; fi
scanner.cc: $(top_srcdir)/scanner.re
@if test -x ./re2c$(EXEEXT); then \
echo "re2c $(RE2CFLAGS) -o $@ $<"; \
./re2c $(RE2CFLAGS) -o $@ $< && cp $@ $(top_srcdir)/bootstrap/; \
else \
echo "cp -f $(top_srcdir)/bootstrap/$@ $@"; \
cp -f $(top_srcdir)/bootstrap/$@ $@; \
fi
.version:
echo $(PACKAGE_VERSION) > .version

472
tools/re2c/Makefile.in Normal file
View file

@ -0,0 +1,472 @@
# Makefile.in generated automatically by automake 1.4-p6 from Makefile.am
# Copyright (C) 1994, 1995-8, 1999, 2001 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
# $Id: Makefile.am,v 1.15 2004/05/26 23:59:15 nuffer Exp $
SHELL = @SHELL@
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
prefix = @prefix@
exec_prefix = @exec_prefix@
bindir = @bindir@
sbindir = @sbindir@
libexecdir = @libexecdir@
datadir = @datadir@
sysconfdir = @sysconfdir@
sharedstatedir = @sharedstatedir@
localstatedir = @localstatedir@
libdir = @libdir@
infodir = @infodir@
mandir = @mandir@
includedir = @includedir@
oldincludedir = /usr/include
DESTDIR =
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = .
ACLOCAL = @ACLOCAL@
AUTOCONF = @AUTOCONF@
AUTOMAKE = @AUTOMAKE@
AUTOHEADER = @AUTOHEADER@
INSTALL = @INSTALL@
INSTALL_PROGRAM = @INSTALL_PROGRAM@ $(AM_INSTALL_PROGRAM_FLAGS)
INSTALL_DATA = @INSTALL_DATA@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
transform = @program_transform_name@
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
CC = @CC@
CXX = @CXX@
MAKEINFO = @MAKEINFO@
PACKAGE = @PACKAGE@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
VERSION = @VERSION@
YACC = @YACC@
bin_PROGRAMS = re2c
re2c_SOURCES = code.cc dfa.cc main.cc parser.cc actions.cc scanner.re substr.cc translate.cc scanner.cc mbo_getopt.cc basics.h dfa.h globals.h ins.h parser.h re.h scanner.h substr.h token.h mbo_getopt.h
BUILT_SOURCES = parser.cc scanner.cc
man_MANS = re2c.1
#CXXFLAGS = -O2 -Wall -I. -Wno-unused -Wno-parentheses -Wno-deprecated
YFLAGS = -d
RE2C = re2c
RE2CFLAGS = -s
CLEANFILES = parser.cc y.tab.c y.tab.h parser.cc re2c.1 .version
DISTCLEANFILES = makerpm re2c.spec README scanner.cc re2c$(EXEEXT)
EXTRA_DIST = $(man_MANS) README parser.y scanner.re makerpm.in re2c.spec.in README.in y.tab.h examples/basemmap.c examples/c.re examples/cmmap.re examples/cnokw.re examples/cunroll.re examples/modula.re examples/sample.re examples/simple.re examples/rexx/README examples/rexx/rexx.l examples/rexx/scanio.c doc/loplas.ps doc/sample.bib
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
CONFIG_HEADER = config.h
CONFIG_CLEAN_FILES = README makerpm re2c.1 re2c.spec
PROGRAMS = $(bin_PROGRAMS)
DEFS = @DEFS@ -I. -I$(srcdir) -I.
CPPFLAGS = @CPPFLAGS@
LDFLAGS = @LDFLAGS@
LIBS = @LIBS@
re2c_OBJECTS = code.o dfa.o main.o parser.o actions.o substr.o \
translate.o scanner.o mbo_getopt.o
re2c_LDADD = $(LDADD)
re2c_DEPENDENCIES =
re2c_LDFLAGS =
CXXFLAGS = @CXXFLAGS@
CXXCOMPILE = $(CXX) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
CXXLD = $(CXX)
CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(LDFLAGS) -o $@
CFLAGS = @CFLAGS@
COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
CCLD = $(CC)
LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@
man1dir = $(mandir)/man1
MANS = $(man_MANS)
NROFF = nroff
DIST_COMMON = README ./stamp-h.in Makefile.am Makefile.in README.in \
aclocal.m4 config.h.in configure configure.in install-sh makerpm.in \
missing mkinstalldirs re2c.1.in re2c.spec.in
DISTFILES = $(DIST_COMMON) $(SOURCES) $(HEADERS) $(TEXINFOS) $(EXTRA_DIST)
TAR = gtar
GZIP_ENV = --best
SOURCES = $(re2c_SOURCES)
OBJECTS = $(re2c_OBJECTS)
all: all-redirect
.SUFFIXES:
.SUFFIXES: .S .c .cc .o .s
$(srcdir)/Makefile.in: Makefile.am $(top_srcdir)/configure.in $(ACLOCAL_M4)
cd $(top_srcdir) && $(AUTOMAKE) --foreign --include-deps Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
$(ACLOCAL_M4): configure.in
cd $(srcdir) && $(ACLOCAL)
config.status: $(srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
$(SHELL) ./config.status --recheck
$(srcdir)/configure: $(srcdir)/configure.in $(ACLOCAL_M4) $(CONFIGURE_DEPENDENCIES)
cd $(srcdir) && $(AUTOCONF)
config.h: stamp-h
@if test ! -f $@; then \
rm -f stamp-h; \
$(MAKE) stamp-h; \
else :; fi
stamp-h: $(srcdir)/config.h.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES= CONFIG_HEADERS=config.h \
$(SHELL) ./config.status
@echo timestamp > stamp-h 2> /dev/null
$(srcdir)/config.h.in: $(srcdir)/stamp-h.in
@if test ! -f $@; then \
rm -f $(srcdir)/stamp-h.in; \
$(MAKE) $(srcdir)/stamp-h.in; \
else :; fi
$(srcdir)/stamp-h.in: $(top_srcdir)/configure.in $(ACLOCAL_M4)
cd $(top_srcdir) && $(AUTOHEADER)
@echo timestamp > $(srcdir)/stamp-h.in 2> /dev/null
mostlyclean-hdr:
clean-hdr:
distclean-hdr:
-rm -f config.h
maintainer-clean-hdr:
README: $(top_builddir)/config.status README.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
makerpm: $(top_builddir)/config.status makerpm.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
re2c.1: $(top_builddir)/config.status re2c.1.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
re2c.spec: $(top_builddir)/config.status re2c.spec.in
cd $(top_builddir) && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
mostlyclean-binPROGRAMS:
clean-binPROGRAMS:
-test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS)
distclean-binPROGRAMS:
maintainer-clean-binPROGRAMS:
install-binPROGRAMS: $(bin_PROGRAMS)
@$(NORMAL_INSTALL)
$(mkinstalldirs) $(DESTDIR)$(bindir)
@list='$(bin_PROGRAMS)'; for p in $$list; do \
if test -f $$p; then \
echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/`echo $$p|sed 's/$(EXEEXT)$$//'|sed '$(transform)'|sed 's/$$/$(EXEEXT)/'`"; \
$(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/`echo $$p|sed 's/$(EXEEXT)$$//'|sed '$(transform)'|sed 's/$$/$(EXEEXT)/'`; \
else :; fi; \
done
uninstall-binPROGRAMS:
@$(NORMAL_UNINSTALL)
list='$(bin_PROGRAMS)'; for p in $$list; do \
rm -f $(DESTDIR)$(bindir)/`echo $$p|sed 's/$(EXEEXT)$$//'|sed '$(transform)'|sed 's/$$/$(EXEEXT)/'`; \
done
.c.o:
$(COMPILE) -c $<
.s.o:
$(COMPILE) -c $<
.S.o:
$(COMPILE) -c $<
mostlyclean-compile:
-rm -f *.o core *.core
clean-compile:
distclean-compile:
-rm -f *.tab.c
maintainer-clean-compile:
re2c: $(re2c_OBJECTS) $(re2c_DEPENDENCIES)
@rm -f re2c
$(CXXLINK) $(re2c_LDFLAGS) $(re2c_OBJECTS) $(re2c_LDADD) $(LIBS)
.cc.o:
$(CXXCOMPILE) -c $<
install-man1:
$(mkinstalldirs) $(DESTDIR)$(man1dir)
@list='$(man1_MANS)'; \
l2='$(man_MANS)'; for i in $$l2; do \
case "$$i" in \
*.1*) list="$$list $$i" ;; \
esac; \
done; \
for i in $$list; do \
if test -f $(srcdir)/$$i; then file=$(srcdir)/$$i; \
else file=$$i; fi; \
ext=`echo $$i | sed -e 's/^.*\\.//'`; \
inst=`echo $$i | sed -e 's/\\.[0-9a-z]*$$//'`; \
inst=`echo $$inst | sed '$(transform)'`.$$ext; \
echo " $(INSTALL_DATA) $$file $(DESTDIR)$(man1dir)/$$inst"; \
$(INSTALL_DATA) $$file $(DESTDIR)$(man1dir)/$$inst; \
done
uninstall-man1:
@list='$(man1_MANS)'; \
l2='$(man_MANS)'; for i in $$l2; do \
case "$$i" in \
*.1*) list="$$list $$i" ;; \
esac; \
done; \
for i in $$list; do \
ext=`echo $$i | sed -e 's/^.*\\.//'`; \
inst=`echo $$i | sed -e 's/\\.[0-9a-z]*$$//'`; \
inst=`echo $$inst | sed '$(transform)'`.$$ext; \
echo " rm -f $(DESTDIR)$(man1dir)/$$inst"; \
rm -f $(DESTDIR)$(man1dir)/$$inst; \
done
install-man: $(MANS)
@$(NORMAL_INSTALL)
$(MAKE) $(AM_MAKEFLAGS) install-man1
uninstall-man:
@$(NORMAL_UNINSTALL)
$(MAKE) $(AM_MAKEFLAGS) uninstall-man1
tags: TAGS
ID: $(HEADERS) $(SOURCES) $(LISP)
list='$(SOURCES) $(HEADERS)'; \
unique=`for i in $$list; do echo $$i; done | \
awk ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
here=`pwd` && cd $(srcdir) \
&& mkid -f$$here/ID $$unique $(LISP)
TAGS: $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS)'; \
unique=`for i in $$list; do echo $$i; done | \
awk ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(ETAGS_ARGS)config.h.in$$unique$(LISP)$$tags" \
|| (cd $(srcdir) && etags $(ETAGS_ARGS) $$tags config.h.in $$unique $(LISP) -o $$here/TAGS)
mostlyclean-tags:
clean-tags:
distclean-tags:
-rm -f TAGS ID
maintainer-clean-tags:
distdir = $(PACKAGE)-$(VERSION)
top_distdir = $(distdir)
# This target untars the dist file and tries a VPATH configuration. Then
# it guarantees that the distribution is self-contained by making another
# tarfile.
distcheck: dist
-rm -rf $(distdir)
GZIP=$(GZIP_ENV) $(TAR) zxf $(distdir).tar.gz
mkdir $(distdir)/=build
mkdir $(distdir)/=inst
dc_install_base=`cd $(distdir)/=inst && pwd`; \
cd $(distdir)/=build \
&& ../configure --srcdir=.. --prefix=$$dc_install_base \
&& $(MAKE) $(AM_MAKEFLAGS) \
&& $(MAKE) $(AM_MAKEFLAGS) dvi \
&& $(MAKE) $(AM_MAKEFLAGS) check \
&& $(MAKE) $(AM_MAKEFLAGS) install \
&& $(MAKE) $(AM_MAKEFLAGS) installcheck \
&& $(MAKE) $(AM_MAKEFLAGS) dist
-rm -rf $(distdir)
@banner="$(distdir).tar.gz is ready for distribution"; \
dashes=`echo "$$banner" | sed s/./=/g`; \
echo "$$dashes"; \
echo "$$banner"; \
echo "$$dashes"
dist: distdir
-chmod -R a+r $(distdir)
GZIP=$(GZIP_ENV) $(TAR) chozf $(distdir).tar.gz $(distdir)
-rm -rf $(distdir)
dist-all: distdir
-chmod -R a+r $(distdir)
GZIP=$(GZIP_ENV) $(TAR) chozf $(distdir).tar.gz $(distdir)
-rm -rf $(distdir)
distdir: $(DISTFILES)
-rm -rf $(distdir)
mkdir $(distdir)
-chmod 777 $(distdir)
$(mkinstalldirs) $(distdir)/doc $(distdir)/examples \
$(distdir)/examples/rexx
@for file in $(DISTFILES); do \
d=$(srcdir); \
if test -d $$d/$$file; then \
cp -pr $$d/$$file $(distdir)/$$file; \
else \
test -f $(distdir)/$$file \
|| ln $$d/$$file $(distdir)/$$file 2> /dev/null \
|| cp -p $$d/$$file $(distdir)/$$file || :; \
fi; \
done
actions.o: actions.cc globals.h basics.h config.h parser.h scanner.h \
token.h substr.h re.h ins.h dfa.h
code.o: code.cc substr.h basics.h config.h globals.h dfa.h re.h token.h \
ins.h parser.h scanner.h
dfa.o: dfa.cc globals.h basics.h config.h substr.h dfa.h re.h token.h \
ins.h
main.o: main.cc config.h globals.h basics.h parser.h scanner.h token.h \
substr.h re.h ins.h dfa.h mbo_getopt.h
mbo_getopt.o: mbo_getopt.cc mbo_getopt.h
parser.o: parser.cc config.h globals.h basics.h parser.h scanner.h \
token.h substr.h re.h ins.h
scanner.o: scanner.cc scanner.h token.h substr.h basics.h config.h \
parser.h re.h ins.h y.tab.h
substr.o: substr.cc substr.h basics.h config.h globals.h
translate.o: translate.cc globals.h basics.h config.h
info-am:
info: info-am
dvi-am:
dvi: dvi-am
check-am: all-am
check: check-am
installcheck-am:
installcheck: installcheck-am
all-recursive-am: config.h
$(MAKE) $(AM_MAKEFLAGS) all-recursive
install-exec-am: install-binPROGRAMS
install-exec: install-exec-am
install-data-am: install-man
install-data: install-data-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
install: install-am
uninstall-am: uninstall-binPROGRAMS uninstall-man
uninstall: uninstall-am
all-am: Makefile $(PROGRAMS) $(MANS) config.h
all-redirect: all-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) AM_INSTALL_PROGRAM_FLAGS=-s install
installdirs:
$(mkinstalldirs) $(DESTDIR)$(bindir) $(DESTDIR)$(mandir)/man1
mostlyclean-generic:
clean-generic:
-test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
distclean-generic:
-rm -f Makefile $(CONFIG_CLEAN_FILES)
-rm -f config.cache config.log stamp-h stamp-h[0-9]*
-test -z "$(DISTCLEANFILES)" || rm -f $(DISTCLEANFILES)
maintainer-clean-generic:
-test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES)
mostlyclean-am: mostlyclean-hdr mostlyclean-binPROGRAMS \
mostlyclean-compile mostlyclean-tags \
mostlyclean-generic
mostlyclean: mostlyclean-am
clean-am: clean-hdr clean-binPROGRAMS clean-compile clean-tags \
clean-generic mostlyclean-am
clean: clean-am
distclean-am: distclean-hdr distclean-binPROGRAMS distclean-compile \
distclean-tags distclean-generic clean-am
distclean: distclean-am
-rm -f config.status
maintainer-clean-am: maintainer-clean-hdr maintainer-clean-binPROGRAMS \
maintainer-clean-compile maintainer-clean-tags \
maintainer-clean-generic distclean-am
@echo "This command is intended for maintainers to use;"
@echo "it deletes files that may require special tools to rebuild."
maintainer-clean: maintainer-clean-am
-rm -f config.status
.PHONY: mostlyclean-hdr distclean-hdr clean-hdr maintainer-clean-hdr \
mostlyclean-binPROGRAMS distclean-binPROGRAMS clean-binPROGRAMS \
maintainer-clean-binPROGRAMS uninstall-binPROGRAMS install-binPROGRAMS \
mostlyclean-compile distclean-compile clean-compile \
maintainer-clean-compile install-man1 uninstall-man1 install-man \
uninstall-man tags mostlyclean-tags distclean-tags clean-tags \
maintainer-clean-tags distdir info-am info dvi-am dvi check check-am \
installcheck-am installcheck all-recursive-am install-exec-am \
install-exec install-data-am install-data install-am install \
uninstall-am uninstall all-redirect all-am all installdirs \
mostlyclean-generic distclean-generic clean-generic \
maintainer-clean-generic clean mostlyclean distclean maintainer-clean
rpm-files: $(bin_PROGRAMS) $(EXTRA_DIST)
parser.cc: $(top_srcdir)/parser.y
$(YACC) $(YFLAGS) $<
mv -f y.tab.c $(top_srcdir)/parser.cc
if cmp -s y.tab.h $(top_srcdir)/y.tab.h; then :; else mv -f y.tab.h $(top_srcdir)/y.tab.h; fi
scanner.cc: $(top_srcdir)/scanner.re
@if test -x ./re2c$(EXEEXT); then \
echo "re2c $(RE2CFLAGS) -o $@ $<"; \
./re2c $(RE2CFLAGS) -o $@ $< && cp $@ $(top_srcdir)/bootstrap/; \
else \
echo "cp -f $(top_srcdir)/bootstrap/$@ $@"; \
cp -f $(top_srcdir)/bootstrap/$@ $@; \
fi
.version:
echo $(PACKAGE_VERSION) > .version
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

31
tools/re2c/Makefile.mgw Normal file
View file

@ -0,0 +1,31 @@
EXE = re2c.exe
CCDV = @../../ccdv
CXX = g++
CXXFLAGS = $(LOC) -D_WIN32 -DNDEBUG -Os -Wall -Wno-unused
OBJS = actions.o code.o dfa.o main.o mbo_getopt.o parser.o scanner.o substr.o translate.o
all: $(EXE)
.cc.o:
$(CCDV) $(CXX) $(CXXFLAGS) -c -o $@ $<
$(EXE): $(OBJS)
$(CCDV) $(CXX) -s -o $(EXE) $(OBJS)
.PHONY: clean
clean:
-del /q /f $(EXE) 2>nul
-del /q /f *.o 2>nul
actions.o: actions.cc
code.o: code.cc
dfa.o: dfa.cc
main.o: main.cc
mbo_getopt.o: mbo_getopt.cc
parser.o: parser.cc
scanner.o: scanner.cc
substr.o: substr.cc
translate.o: translate.cc

165
tools/re2c/README Normal file
View file

@ -0,0 +1,165 @@
re2c Version 0.9.3
------------------
Originally written by Peter Bumbulis (peter@csg.uwaterloo.ca)
Currently maintained by:
Dan Nuffer <nuffer at users.sourceforge.net>
Marcus Boerger <helly at users.sourceforge.net>
Hartmut Kaiser <hkaiser at users.sourceforge.net>
The re2c distribution can be found at:
http://sourceforge.net/projects/re2c/
This distribution is a cleaned up version of the 0.5 release. Several
bugs were fixed as well as code cleanup for warning free compilation.
It has been developed and tested with egcs 1.0.2 and gcc 2.7.2.3, 2.96
and 3.3.1 on Linux x86. You can compile your own version with other gcc
version if you have yacc or any working bison version (tested up to bison
1.875).
You can build this software by simply typing the following commands:
./autogen.sh
./configure
make
The above version will be based on the pregenerated scanner.cc file.
If you want to build that file yourself (recommended when installing
re2c) you need the following steps:
./autogen.sh
./configure
make
rm -f scanner.cc
make install
Or you can create a rpm package and install it by the following commands:
./autogen.sh
./configure
./makerpm <release>
rpm -Uhv <packagedir>/re2c-0.9.3-<release>.rpm
Here <realease> should be a number like 1. And <packagedir> must equal
the directory where the makerpm step has written the generated rpm to.
re2c is a great tool for writing fast and flexible lexers. It has
served many people well for many years. re2c is on the order of 2-3
times faster than a flex based scanner, and its input model is much
more flexible.
Peter's original version 0.5 ANNOUNCE and README follows.
--
re2c is a tool for generating C-based recognizers from regular
expressions. re2c-based scanners are efficient: for programming
languages, given similar specifications, an re2c-based scanner is
typically almost twice as fast as a flex-based scanner with little or no
increase in size (possibly a decrease on cisc architectures). Indeed,
re2c-based scanners are quite competitive with hand-crafted ones.
Unlike flex, re2c does not generate complete scanners: the user must
supply some interface code. While this code is not bulky (about 50-100
lines for a flex-like scanner; see the man page and examples in the
distribution) careful coding is required for efficiency (and
correctness). One advantage of this arrangement is that the generated
code is not tied to any particular input model. For example, re2c
generated code can be used to scan data from a null-byte terminated
buffer as illustrated below.
Given the following source
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\000-\377] {return NULL;}
*/
}
re2c will generate
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
Note that most compilers will perform dead-code elimination to remove
all YYCURSOR, YYLIMIT comparisions.
re2c was developed for a particular project (constructing a fast REXX
scanner of all things!) and so while it has some rough edges, it should
be quite usable. More information about re2c can be found in the
(admittedly skimpy) man page; the algorithms and heuristics used are
described in an upcoming LOPLAS article (included in the distribution).
Probably the best way to find out more about re2c is to try the supplied
examples. re2c is written in C++, and is currently being developed
under Linux using gcc 2.5.8.
Peter
--
re2c is distributed with no warranty whatever. The code is certain to
contain errors. Neither the author nor any contributor takes
responsibility for any consequences of its use.
re2c is in the public domain. The data structures and algorithms used
in re2c are all either taken from documents available to the general
public or are inventions of the author. Programs generated by re2c may
be distributed freely. re2c itself may be distributed freely, in source
or binary, unchanged or modified. Distributors may charge whatever fees
they can obtain for re2c.
If you do make use of re2c, or incorporate it into a larger project an
acknowledgement somewhere (documentation, research report, etc.) would
be appreciated.
Please send bug reports and feedback (including suggestions for
improving the distribution) to
peter@csg.uwaterloo.ca
Include a small example and the banner from parser.y with bug reports.

165
tools/re2c/README.in Normal file
View file

@ -0,0 +1,165 @@
re2c Version @PACKAGE_VERSION@
------------------
Originally written by Peter Bumbulis (peter@csg.uwaterloo.ca)
Currently maintained by:
Dan Nuffer <nuffer at users.sourceforge.net>
Marcus Boerger <helly at users.sourceforge.net>
Hartmut Kaiser <hkaiser at users.sourceforge.net>
The re2c distribution can be found at:
http://sourceforge.net/projects/re2c/
This distribution is a cleaned up version of the 0.5 release. Several
bugs were fixed as well as code cleanup for warning free compilation.
It has been developed and tested with egcs 1.0.2 and gcc 2.7.2.3, 2.96
and 3.3.1 on Linux x86. You can compile your own version with other gcc
version if you have yacc or any working bison version (tested up to bison
1.875).
You can build this software by simply typing the following commands:
./autogen.sh
./configure
make
The above version will be based on the pregenerated scanner.cc file.
If you want to build that file yourself (recommended when installing
re2c) you need the following steps:
./autogen.sh
./configure
make
rm -f scanner.cc
make install
Or you can create a rpm package and install it by the following commands:
./autogen.sh
./configure
./makerpm <release>
rpm -Uhv <packagedir>/re2c-@PACKAGE_VERSION@-<release>.rpm
Here <realease> should be a number like 1. And <packagedir> must equal
the directory where the makerpm step has written the generated rpm to.
re2c is a great tool for writing fast and flexible lexers. It has
served many people well for many years. re2c is on the order of 2-3
times faster than a flex based scanner, and its input model is much
more flexible.
Peter's original version 0.5 ANNOUNCE and README follows.
--
re2c is a tool for generating C-based recognizers from regular
expressions. re2c-based scanners are efficient: for programming
languages, given similar specifications, an re2c-based scanner is
typically almost twice as fast as a flex-based scanner with little or no
increase in size (possibly a decrease on cisc architectures). Indeed,
re2c-based scanners are quite competitive with hand-crafted ones.
Unlike flex, re2c does not generate complete scanners: the user must
supply some interface code. While this code is not bulky (about 50-100
lines for a flex-like scanner; see the man page and examples in the
distribution) careful coding is required for efficiency (and
correctness). One advantage of this arrangement is that the generated
code is not tied to any particular input model. For example, re2c
generated code can be used to scan data from a null-byte terminated
buffer as illustrated below.
Given the following source
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\000-\377] {return NULL;}
*/
}
re2c will generate
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
Note that most compilers will perform dead-code elimination to remove
all YYCURSOR, YYLIMIT comparisions.
re2c was developed for a particular project (constructing a fast REXX
scanner of all things!) and so while it has some rough edges, it should
be quite usable. More information about re2c can be found in the
(admittedly skimpy) man page; the algorithms and heuristics used are
described in an upcoming LOPLAS article (included in the distribution).
Probably the best way to find out more about re2c is to try the supplied
examples. re2c is written in C++, and is currently being developed
under Linux using gcc 2.5.8.
Peter
--
re2c is distributed with no warranty whatever. The code is certain to
contain errors. Neither the author nor any contributor takes
responsibility for any consequences of its use.
re2c is in the public domain. The data structures and algorithms used
in re2c are all either taken from documents available to the general
public or are inventions of the author. Programs generated by re2c may
be distributed freely. re2c itself may be distributed freely, in source
or binary, unchanged or modified. Distributors may charge whatever fees
they can obtain for re2c.
If you do make use of re2c, or incorporate it into a larger project an
acknowledgement somewhere (documentation, research report, etc.) would
be appreciated.
Please send bug reports and feedback (including suggestions for
improving the distribution) to
peter@csg.uwaterloo.ca
Include a small example and the banner from parser.y with bug reports.

157
tools/re2c/aclocal.m4 vendored Normal file
View file

@ -0,0 +1,157 @@
dnl aclocal.m4 generated automatically by aclocal 1.4-p6
dnl Copyright (C) 1994, 1995-8, 1999, 2001 Free Software Foundation, Inc.
dnl This file is free software; the Free Software Foundation
dnl gives unlimited permission to copy and/or distribute it,
dnl with or without modifications, as long as this notice is preserved.
dnl This program is distributed in the hope that it will be useful,
dnl but WITHOUT ANY WARRANTY, to the extent permitted by law; without
dnl even the implied warranty of MERCHANTABILITY or FITNESS FOR A
dnl PARTICULAR PURPOSE.
# Do all the work for Automake. This macro actually does too much --
# some checks are only needed if your package does certain things.
# But this isn't really a big deal.
# serial 1
dnl Usage:
dnl AM_INIT_AUTOMAKE(package,version, [no-define])
AC_DEFUN([AM_INIT_AUTOMAKE],
[AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl
AC_REQUIRE([AC_PROG_INSTALL])
PACKAGE=[$1]
AC_SUBST(PACKAGE)
VERSION=[$2]
AC_SUBST(VERSION)
dnl test to see if srcdir already configured
if test "`cd $srcdir && pwd`" != "`pwd`" && test -f $srcdir/config.status; then
AC_MSG_ERROR([source directory already configured; run "make distclean" there first])
fi
ifelse([$3],,
AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE", [Name of package])
AC_DEFINE_UNQUOTED(VERSION, "$VERSION", [Version number of package]))
AC_REQUIRE([AM_SANITY_CHECK])
AC_REQUIRE([AC_ARG_PROGRAM])
dnl FIXME This is truly gross.
missing_dir=`cd $ac_aux_dir && pwd`
AM_MISSING_PROG(ACLOCAL, aclocal-${am__api_version}, $missing_dir)
AM_MISSING_PROG(AUTOCONF, autoconf, $missing_dir)
AM_MISSING_PROG(AUTOMAKE, automake-${am__api_version}, $missing_dir)
AM_MISSING_PROG(AUTOHEADER, autoheader, $missing_dir)
AM_MISSING_PROG(MAKEINFO, makeinfo, $missing_dir)
AC_REQUIRE([AC_PROG_MAKE_SET])])
# Copyright 2002 Free Software Foundation, Inc.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# AM_AUTOMAKE_VERSION(VERSION)
# ----------------------------
# Automake X.Y traces this macro to ensure aclocal.m4 has been
# generated from the m4 files accompanying Automake X.Y.
AC_DEFUN([AM_AUTOMAKE_VERSION],[am__api_version="1.4"])
# AM_SET_CURRENT_AUTOMAKE_VERSION
# -------------------------------
# Call AM_AUTOMAKE_VERSION so it can be traced.
# This function is AC_REQUIREd by AC_INIT_AUTOMAKE.
AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
[AM_AUTOMAKE_VERSION([1.4-p6])])
#
# Check to make sure that the build environment is sane.
#
AC_DEFUN([AM_SANITY_CHECK],
[AC_MSG_CHECKING([whether build environment is sane])
# Just in case
sleep 1
echo timestamp > conftestfile
# Do `set' in a subshell so we don't clobber the current shell's
# arguments. Must try -L first in case configure is actually a
# symlink; some systems play weird games with the mod time of symlinks
# (eg FreeBSD returns the mod time of the symlink's containing
# directory).
if (
set X `ls -Lt $srcdir/configure conftestfile 2> /dev/null`
if test "[$]*" = "X"; then
# -L didn't work.
set X `ls -t $srcdir/configure conftestfile`
fi
if test "[$]*" != "X $srcdir/configure conftestfile" \
&& test "[$]*" != "X conftestfile $srcdir/configure"; then
# If neither matched, then we have a broken ls. This can happen
# if, for instance, CONFIG_SHELL is bash and it inherits a
# broken ls alias from the environment. This has actually
# happened. Such a system could not be considered "sane".
AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken
alias in your environment])
fi
test "[$]2" = conftestfile
)
then
# Ok.
:
else
AC_MSG_ERROR([newly created file is older than distributed files!
Check your system clock])
fi
rm -f conftest*
AC_MSG_RESULT(yes)])
dnl AM_MISSING_PROG(NAME, PROGRAM, DIRECTORY)
dnl The program must properly implement --version.
AC_DEFUN([AM_MISSING_PROG],
[AC_MSG_CHECKING(for working $2)
# Run test in a subshell; some versions of sh will print an error if
# an executable is not found, even if stderr is redirected.
# Redirect stdin to placate older versions of autoconf. Sigh.
if ($2 --version) < /dev/null > /dev/null 2>&1; then
$1=$2
AC_MSG_RESULT(found)
else
$1="$3/missing $2"
AC_MSG_RESULT(missing)
fi
AC_SUBST($1)])
# Like AC_CONFIG_HEADER, but automatically create stamp file.
AC_DEFUN([AM_CONFIG_HEADER],
[AC_PREREQ([2.12])
AC_CONFIG_HEADER([$1])
dnl When config.status generates a header, we must update the stamp-h file.
dnl This file resides in the same directory as the config header
dnl that is generated. We must strip everything past the first ":",
dnl and everything past the last "/".
AC_OUTPUT_COMMANDS(changequote(<<,>>)dnl
ifelse(patsubst(<<$1>>, <<[^ ]>>, <<>>), <<>>,
<<test -z "<<$>>CONFIG_HEADERS" || echo timestamp > patsubst(<<$1>>, <<^\([^:]*/\)?.*>>, <<\1>>)stamp-h<<>>dnl>>,
<<am_indx=1
for am_file in <<$1>>; do
case " <<$>>CONFIG_HEADERS " in
*" <<$>>am_file "*<<)>>
echo timestamp > `echo <<$>>am_file | sed -e 's%:.*%%' -e 's%[^/]*$%%'`stamp-h$am_indx
;;
esac
am_indx=`expr "<<$>>am_indx" + 1`
done<<>>dnl>>)
changequote([,]))])

568
tools/re2c/actions.cc Normal file
View file

@ -0,0 +1,568 @@
/* $Id: actions.cc,v 1.5 2004/05/13 02:58:17 nuffer Exp $ */
#include <time.h>
#include <string.h>
#include <iostream>
#include <iomanip>
#include "globals.h"
#include "parser.h"
#include "dfa.h"
Symbol *Symbol::first = NULL;
Symbol::Symbol(const SubStr &str) : next(first), name(str), re(NULL) {
first = this;
}
Symbol *Symbol::find(const SubStr &str){
for(Symbol *sym = first; sym; sym = sym->next)
if(sym->name == str) return sym;
return new Symbol(str);
}
void showIns(std::ostream &o, const Ins &i, const Ins &base){
o.width(3);
o << &i - &base << ": ";
switch(i.i.tag){
case CHAR: {
o << "match ";
for(const Ins *j = &(&i)[1]; j < (Ins*) i.i.link; ++j)
prtCh(o, j->c.value);
break;
} case GOTO:
o << "goto " << ((Ins*) i.i.link - &base);
break;
case FORK:
o << "fork " << ((Ins*) i.i.link - &base);
break;
case CTXT:
o << "term " << ((RuleOp*) i.i.link)->accept;
break;
case TERM:
o << "term " << ((RuleOp*) i.i.link)->accept;
break;
}
o << "\n";
}
uint RegExp::fixedLength(){
return ~0;
}
char *NullOp::type = "NullOp";
void NullOp::calcSize(Char*){
size = 0;
}
uint NullOp::fixedLength(){
return 0;
}
void NullOp::compile(Char*, Ins*){
;
}
void NullOp::split(CharSet&){
;
}
std::ostream& operator<<(std::ostream &o, const Range &r){
if((r.ub - r.lb) == 1){
prtCh(o, r.lb);
} else {
prtCh(o, r.lb); o << "-"; prtCh(o, r.ub-1);
}
return o << r.next;
}
Range *doUnion(Range *r1, Range *r2){
Range *r, **rP = &r;
for(;;){
Range *s;
if(r1->lb <= r2->lb){
s = new Range(*r1);
} else {
s = new Range(*r2);
}
*rP = s;
rP = &s->next;
for(;;){
if(r1->lb <= r2->lb){
if(r1->lb > s->ub)
break;
if(r1->ub > s->ub)
s->ub = r1->ub;
if(!(r1 = r1->next)){
uint ub = 0;
for(; r2 && r2->lb <= s->ub; r2 = r2->next)
ub = r2->ub;
if(ub > s->ub)
s->ub = ub;
*rP = r2;
return r;
}
} else {
if(r2->lb > s->ub)
break;
if(r2->ub > s->ub)
s->ub = r2->ub;
if(!(r2 = r2->next)){
uint ub = 0;
for(; r1 && r1->lb <= s->ub; r1 = r1->next)
ub = r1->ub;
if(ub > s->ub)
s->ub = ub;
*rP = r1;
return r;
}
}
}
}
*rP = NULL;
return r;
}
Range *doDiff(Range *r1, Range *r2){
Range *r, *s, **rP = &r;
for(; r1; r1 = r1->next){
uint lb = r1->lb;
for(; r2 && r2->ub <= r1->lb; r2 = r2->next);
for(; r2 && r2->lb < r1->ub; r2 = r2->next){
if(lb < r2->lb){
*rP = s = new Range(lb, r2->lb);
rP = &s->next;
}
if((lb = r2->ub) >= r1->ub)
goto noMore;
}
*rP = s = new Range(lb, r1->ub);
rP = &s->next;
noMore:;
}
*rP = NULL;
return r;
}
MatchOp *merge(MatchOp *m1, MatchOp *m2){
if(!m1)
return m2;
if(!m2)
return m1;
return new MatchOp(doUnion(m1->match, m2->match));
}
char *MatchOp::type = "MatchOp";
void MatchOp::display(std::ostream &o) const{
o << match;
}
void MatchOp::calcSize(Char *rep){
size = 1;
for(Range *r = match; r; r = r->next)
for(uint c = r->lb; c < r->ub; ++c)
if(rep[c] == c)
++size;
}
uint MatchOp::fixedLength(){
return 1;
}
void MatchOp::compile(Char *rep, Ins *i){
i->i.tag = CHAR;
i->i.link = &i[size];
Ins *j = &i[1];
uint bump = size;
for(Range *r = match; r; r = r->next){
for(uint c = r->lb; c < r->ub; ++c){
if(rep[c] == c){
j->c.value = c;
j->c.bump = --bump;
j++;
}
}
}
}
void MatchOp::split(CharSet &s){
for(Range *r = match; r; r = r->next){
for(uint c = r->lb; c < r->ub; ++c){
CharPtn *x = s.rep[c], *a = x->nxt;
if(!a){
if(x->card == 1)
continue;
x->nxt = a = s.freeHead;
if(!(s.freeHead = s.freeHead->nxt))
s.freeTail = &s.freeHead;
a->nxt = NULL;
x->fix = s.fix;
s.fix = x;
}
if(--(x->card) == 0){
*s.freeTail = x;
*(s.freeTail = &x->nxt) = NULL;
}
s.rep[c] = a;
++(a->card);
}
}
for(; s.fix; s.fix = s.fix->fix)
if(s.fix->card)
s.fix->nxt = NULL;
}
RegExp *mkDiff(RegExp *e1, RegExp *e2){
MatchOp *m1, *m2;
if(!(m1 = (MatchOp*) e1->isA(MatchOp::type)))
return NULL;
if(!(m2 = (MatchOp*) e2->isA(MatchOp::type)))
return NULL;
Range *r = doDiff(m1->match, m2->match);
return r? (RegExp*) new MatchOp(r) : (RegExp*) new NullOp;
}
RegExp *doAlt(RegExp *e1, RegExp *e2){
if(!e1)
return e2;
if(!e2)
return e1;
return new AltOp(e1, e2);
}
RegExp *mkAlt(RegExp *e1, RegExp *e2){
AltOp *a;
MatchOp *m1, *m2;
if((a = (AltOp*) e1->isA(AltOp::type))){
if((m1 = (MatchOp*) a->exp1->isA(MatchOp::type)))
e1 = a->exp2;
} else if((m1 = (MatchOp*) e1->isA(MatchOp::type))){
e1 = NULL;
}
if((a = (AltOp*) e2->isA(AltOp::type))){
if((m2 = (MatchOp*) a->exp1->isA(MatchOp::type)))
e2 = a->exp2;
} else if((m2 = (MatchOp*) e2->isA(MatchOp::type))){
e2 = NULL;
}
return doAlt(merge(m1, m2), doAlt(e1, e2));
}
char *AltOp::type = "AltOp";
void AltOp::calcSize(Char *rep){
exp1->calcSize(rep);
exp2->calcSize(rep);
size = exp1->size + exp2->size + 2;
}
uint AltOp::fixedLength(){
uint l1 = exp1->fixedLength();
uint l2 = exp1->fixedLength();
if(l1 != l2 || l1 == ~0u)
return ~0;
return l1;
}
void AltOp::compile(Char *rep, Ins *i){
i->i.tag = FORK;
Ins *j = &i[exp1->size + 1];
i->i.link = &j[1];
exp1->compile(rep, &i[1]);
j->i.tag = GOTO;
j->i.link = &j[exp2->size + 1];
exp2->compile(rep, &j[1]);
}
void AltOp::split(CharSet &s){
exp1->split(s);
exp2->split(s);
}
char *CatOp::type = "CatOp";
void CatOp::calcSize(Char *rep){
exp1->calcSize(rep);
exp2->calcSize(rep);
size = exp1->size + exp2->size;
}
uint CatOp::fixedLength(){
uint l1, l2;
if((l1 = exp1->fixedLength()) != ~0u )
if((l2 = exp2->fixedLength()) != ~0u)
return l1+l2;
return ~0;
}
void CatOp::compile(Char *rep, Ins *i){
exp1->compile(rep, &i[0]);
exp2->compile(rep, &i[exp1->size]);
}
void CatOp::split(CharSet &s){
exp1->split(s);
exp2->split(s);
}
char *CloseOp::type = "CloseOp";
void CloseOp::calcSize(Char *rep){
exp->calcSize(rep);
size = exp->size + 1;
}
void CloseOp::compile(Char *rep, Ins *i){
exp->compile(rep, &i[0]);
i += exp->size;
i->i.tag = FORK;
i->i.link = i - exp->size;
}
void CloseOp::split(CharSet &s){
exp->split(s);
}
char *CloseVOp::type = "CloseVOp";
void CloseVOp::calcSize(Char *rep){
exp->calcSize(rep);
if(max >= 0)
size = (exp->size * min) + ((1 + exp->size) * (max - min));
else
size = (exp->size * min) + 1;
}
void CloseVOp::compile(Char *rep, Ins *i){
Ins *jumppoint;
int st = 0;
jumppoint = i + ((1 + exp->size) * (max - min));
for(st = min; st < max; st++) {
i->i.tag = FORK;
i->i.link = jumppoint;
i+=1;
exp->compile(rep, &i[0]);
i += exp->size;
}
for(st = 0; st < min; st++) {
exp->compile(rep, &i[0]);
i += exp->size;
if(max < 0 && st == 0) {
i->i.tag = FORK;
i->i.link = i - exp->size;
i++;
}
}
}
void CloseVOp::split(CharSet &s){
exp->split(s);
}
RegExp *expr(Scanner &);
uchar unescape(SubStr &s){
s.len--;
uchar c;
if((c = *s.str++) != '\\' || s.len == 0)
return xlat[c];
s.len--;
switch(c = *s.str++){
case 'n':
return xlat['\n'];
case 't':
return xlat['\t'];
case 'v':
return xlat['\v'];
case 'b':
return xlat['\b'];
case 'r':
return xlat['\r'];
case 'f':
return xlat['\f'];
case 'a':
return xlat['\a'];
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7': {
uchar v = c - '0';
for(; s.len != 0 && '0' <= (c = *s.str) && c <= '7'; s.len--, s.str++)
v = v*8 + (c - '0');
return v;
} default:
return xlat[c];
}
}
Range *getRange(SubStr &s){
uchar lb = unescape(s), ub;
if(s.len < 2 || *s.str != '-'){
ub = lb;
} else {
s.len--; s.str++;
ub = unescape(s);
if(ub < lb){
uchar tmp;
tmp = lb; lb = ub; ub = tmp;
}
}
return new Range(lb, ub+1);
}
RegExp *matchChar(uint c){
return new MatchOp(new Range(c, c+1));
}
RegExp *strToRE(SubStr s){
s.len -= 2; s.str += 1;
if(s.len == 0)
return new NullOp;
RegExp *re = matchChar(unescape(s));
while(s.len > 0)
re = new CatOp(re, matchChar(unescape(s)));
return re;
}
RegExp *strToCaseInsensitiveRE(SubStr s){
s.len -= 2; s.str += 1;
if(s.len == 0)
return new NullOp;
uchar c = unescape(s);
RegExp *re, *reL, *reU;
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
reL = matchChar(tolower(c));
reU = matchChar(toupper(c));
re = mkAlt(reL, reU);
} else {
re = matchChar(c);
}
while(s.len > 0) {
uchar c = unescape(s);
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
reL = matchChar(tolower(c));
reU = matchChar(toupper(c));
re = new CatOp(re, mkAlt(reL, reU));
} else {
re = new CatOp(re, matchChar(c));
}
}
return re;
}
RegExp *ranToRE(SubStr s){
s.len -= 2; s.str += 1;
if(s.len == 0)
return new NullOp;
Range *r = getRange(s);
while(s.len > 0)
r = doUnion(r, getRange(s));
return new MatchOp(r);
}
char *RuleOp::type = "RuleOp";
RuleOp::RuleOp(RegExp *e, RegExp *c, Token *t, uint a)
: exp(e), ctx(c), ins(NULL), accept(a), code(t) {
;
}
void RuleOp::calcSize(Char *rep){
exp->calcSize(rep);
ctx->calcSize(rep);
size = exp->size + ctx->size + 1;
}
void RuleOp::compile(Char *rep, Ins *i){
ins = i;
exp->compile(rep, &i[0]);
i += exp->size;
ctx->compile(rep, &i[0]);
i += ctx->size;
i->i.tag = TERM;
i->i.link = this;
}
void RuleOp::split(CharSet &s){
exp->split(s);
ctx->split(s);
}
extern void printSpan(std::ostream&, uint, uint);
void optimize(Ins *i){
while(!isMarked(i)){
mark(i);
if(i->i.tag == CHAR){
i = (Ins*) i->i.link;
} else if(i->i.tag == GOTO || i->i.tag == FORK){
Ins *target = (Ins*) i->i.link;
optimize(target);
if(target->i.tag == GOTO)
i->i.link = target->i.link == target? i : target;
if(i->i.tag == FORK){
Ins *follow = (Ins*) &i[1];
optimize(follow);
if(follow->i.tag == GOTO && follow->i.link == follow){
i->i.tag = GOTO;
} else if(i->i.link == i){
i->i.tag = GOTO;
i->i.link = follow;
}
}
return;
} else {
++i;
}
}
}
void genCode(std::ostream& o, RegExp *re){
CharSet cs;
uint j;
memset(&cs, 0, sizeof(cs));
for(j = 0; j < nChars; ++j){
cs.rep[j] = &cs.ptn[0];
cs.ptn[j].nxt = &cs.ptn[j+1];
}
cs.freeHead = &cs.ptn[1];
*(cs.freeTail = &cs.ptn[nChars-1].nxt) = NULL;
cs.ptn[0].card = nChars;
cs.ptn[0].nxt = NULL;
re->split(cs);
/*
for(uint k = 0; k < nChars;){
for(j = k; ++k < nChars && cs.rep[k] == cs.rep[j];);
printSpan(cerr, j, k);
cerr << "\t" << cs.rep[j] - &cs.ptn[0] << endl;
}
*/
Char rep[nChars];
for(j = 0; j < nChars; ++j){
if(!cs.rep[j]->nxt)
cs.rep[j]->nxt = &cs.ptn[j];
rep[j] = (Char) (cs.rep[j]->nxt - &cs.ptn[0]);
}
re->calcSize(rep);
Ins *ins = new Ins[re->size+1];
memset(ins, 0, (re->size+1)*sizeof(Ins));
re->compile(rep, ins);
Ins *eoi = &ins[re->size];
eoi->i.tag = GOTO;
eoi->i.link = eoi;
optimize(ins);
for(j = 0; j < re->size;){
unmark(&ins[j]);
if(ins[j].i.tag == CHAR){
j = (Ins*) ins[j].i.link - ins;
} else {
j++;
}
}
DFA *dfa = new DFA(ins, re->size, 0, 256, rep);
dfa->emit(o);
delete dfa;
delete [] ins;
}

63
tools/re2c/basics.h Normal file
View file

@ -0,0 +1,63 @@
/* $Id: basics.h,v 1.3 2004/03/13 13:40:37 helly Exp $ */
#ifndef _basics_h
#define _basics_h
#ifdef HAVE_CONFIG_H
#include "config.h"
#elif _WIN32
#include "configwin.h"
#endif
#if SIZEOF_CHAR == 1
typedef unsigned char byte;
#elif SIZEOF_SHORT == 1
typedef unsigned short byte;
#elif SIZEOF_INT == 1
typedef unsigned int byte;
#elif SIZEOF_LONG == 1
typedef unsigned long byte;
#else
typedef unsigned char byte;
#endif
#if SIZEOF_CHAR == 2
typedef unsigned char word;
#elif SIZEOF_SHORT == 2
typedef unsigned short word;
#elif SIZEOF_INT == 2
typedef unsigned int word;
#elif SIZEOF_LONG == 2
typedef unsigned long word;
#else
typedef unsigned short word;
#endif
#if SIZEOF_CHAR == 4
typedef unsigned char dword;
#elif SIZEOF_SHORT == 4
typedef unsigned short dword;
#elif SIZEOF_INT == 4
typedef unsigned int dword;
#elif SIZEOF_LONG == 4
typedef unsigned long dword;
#else
typedef unsigned long dword;
#endif
#ifndef HAVE_UINT
typedef unsigned int uint;
#endif
#ifndef HAVE_UCHAR
typedef unsigned char uchar;
#endif
#ifndef HAVE_USHORT
typedef unsigned short ushort;
#endif
#ifndef HAVE_ULONG
typedef unsigned long ulong;
#endif
#endif

964
tools/re2c/code.cc Normal file
View file

@ -0,0 +1,964 @@
/* $Id: code.cc,v 1.10 2004/05/27 00:02:01 nuffer Exp $ */
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <iomanip>
#include <iostream>
#include "substr.h"
#include "globals.h"
#include "dfa.h"
#include "parser.h"
// there must be at least one span in list; all spans must cover
// same range
void Go::compact()
{
// arrange so that adjacent spans have different targets
uint i = 0;
for(uint j = 1; j < nSpans; ++j)
{
if(span[j].to != span[i].to)
{
++i; span[i].to = span[j].to;
}
span[i].ub = span[j].ub;
}
nSpans = i + 1;
}
void Go::unmap(Go *base, State *x)
{
Span *s = span, *b = base->span, *e = &b[base->nSpans];
uint lb = 0;
s->ub = 0;
s->to = NULL;
for(; b != e; ++b)
{
if(b->to == x)
{
if((s->ub - lb) > 1)
{
s->ub = b->ub;
}
} else {
if(b->to != s->to)
{
if(s->ub)
{
lb = s->ub; ++s;
}
s->to = b->to;
}
s->ub = b->ub;
}
}
s->ub = e[-1].ub; ++s;
nSpans = s - span;
}
void doGen(Go *g, State *s, uchar *bm, uchar m)
{
Span *b = g->span, *e = &b[g->nSpans];
uint lb = 0;
for(; b < e; ++b)
{
if(b->to == s)
{
for(; lb < b->ub; ++lb)
{
bm[lb] |= m;
}
}
lb = b->ub;
}
}
void prt(std::ostream& o, Go *g, State *s)
{
Span *b = g->span, *e = &b[g->nSpans];
uint lb = 0;
for(; b < e; ++b)
{
if(b->to == s)
{
printSpan(o, lb, b->ub);
}
lb = b->ub;
}
}
bool matches(Go *g1, State *s1, Go *g2, State *s2)
{
Span *b1 = g1->span, *e1 = &b1[g1->nSpans];
uint lb1 = 0;
Span *b2 = g2->span, *e2 = &b2[g2->nSpans];
uint lb2 = 0;
for(;;)
{
for(; b1 < e1 && b1->to != s1; ++b1)
{
lb1 = b1->ub;
}
for(; b2 < e2 && b2->to != s2; ++b2)
{
lb2 = b2->ub;
}
if(b1 == e1)
{
return b2 == e2;
}
if(b2 == e2){
return false;
}
if(lb1 != lb2 || b1->ub != b2->ub)
{
return false;
}
++b1; ++b2;
}
}
class BitMap {
public:
static BitMap *first;
Go *go;
State *on;
BitMap *next;
uint i;
uchar m;
public:
static BitMap *find(Go*, State*);
static BitMap *find(State*);
static void gen(std::ostream&, uint, uint);
static void stats();
BitMap(Go*, State*);
};
BitMap *BitMap::first = NULL;
BitMap::BitMap(Go *g, State *x) : go(g), on(x), next(first)
{
first = this;
}
BitMap *BitMap::find(Go *g, State *x)
{
for(BitMap *b = first; b; b = b->next)
{
if(matches(b->go, b->on, g, x))
{
return b;
}
}
return new BitMap(g, x);
}
BitMap *BitMap::find(State *x)
{
for(BitMap *b = first; b; b = b->next)
{
if(b->on == x)
{
return b;
}
}
return NULL;
}
void BitMap::gen(std::ostream &o, uint lb, uint ub)
{
BitMap *b = first;
if(b)
{
o << "\tstatic unsigned char yybm[] = {";
uint n = ub - lb;
uchar *bm = new uchar[n];
memset(bm, 0, n);
for(uint i = 0; b; i += n)
{
for(uchar m = 0x80; b && m; b = b->next, m >>= 1)
{
b->i = i; b->m = m;
doGen(b->go, b->on, bm-lb, m);
}
for(uint j = 0; j < n; ++j)
{
if(j%8 == 0)
{
o << "\n\t"; ++oline;
}
o << std::setw(3) << (uint) bm[j] << ", ";
}
}
o << "\n\t};\n";
oline += 2;
}
}
void BitMap::stats()
{
uint n = 0;
for(BitMap *b = first; b; b = b->next)
{
prt(std::cerr, b->go, b->on); std::cerr << std::endl;
++n;
}
std::cerr << n << " bitmaps\n";
first = NULL;
}
void genGoTo(std::ostream &o, State *from, State *to, bool & readCh)
{
if (readCh && from->label + 1 != to->label)
{
o << "\tyych = *YYCURSOR;\n";
readCh = false;
}
o << "\tgoto yy" << to->label << ";\n";
++oline;
}
void genIf(std::ostream &o, char *cmp, uint v, bool &readCh)
{
if (readCh)
{
o << "\tif((yych = *YYCURSOR) ";
readCh = false;
}
else
{
o << "\tif(yych ";
}
o << cmp << " '";
prtCh(o, v);
o << "')";
}
void indent(std::ostream &o, uint i)
{
while(i-- > 0)
{
o << "\t";
}
}
static void need(std::ostream &o, uint n, bool & readCh)
{
if(n == 1)
{
o << "\tif(YYLIMIT == YYCURSOR) YYFILL(1);\n";
++oline;
}
else
{
o << "\tif((YYLIMIT - YYCURSOR) < " << n << ") YYFILL(" << n << ");\n";
++oline;
}
o << "\tyych = *YYCURSOR;\n";
readCh = false;
++oline;
}
void Match::emit(std::ostream &o, bool &readCh)
{
if (state->link)
{
o << "\t++YYCURSOR;\n";
}
else if (!readAhead())
{
/* do not read next char if match */
o << "\t++YYCURSOR;\n";
readCh = true;
}
else
{
o << "\tyych = *++YYCURSOR;\n";
readCh = false;
}
++oline;
if(state->link)
{
++oline;
need(o, state->depth, readCh);
}
}
void Enter::emit(std::ostream &o, bool &readCh)
{
if(state->link){
o << "\t++YYCURSOR;\n";
o << "yy" << label << ":\n";
oline += 2;
need(o, state->depth, readCh);
} else {
/* we shouldn't need 'rule-following' protection here */
o << "\tyych = *++YYCURSOR;\n";
o << "yy" << label << ":\n";
oline += 2;
readCh = false;
}
}
void Save::emit(std::ostream &o, bool &readCh)
{
o << "\tyyaccept = " << selector << ";\n";
++oline;
if(state->link){
o << "\tYYMARKER = ++YYCURSOR;\n";
++oline;
need(o, state->depth, readCh);
} else {
o << "\tyych = *(YYMARKER = ++YYCURSOR);\n";
++oline;
readCh = false;
}
}
Move::Move(State *s) : Action(s) {
;
}
void Move::emit(std::ostream &o, bool &readCh){
;
}
Accept::Accept(State *x, uint n, uint *s, State **r)
: Action(x), nRules(n), saves(s), rules(r){
;
}
void Accept::emit(std::ostream &o, bool &readCh)
{
bool first = true;
for(uint i = 0; i < nRules; ++i)
if(saves[i] != ~0u)
{
if(first)
{
first = false;
o << "\tYYCURSOR = YYMARKER;\n";
o << "\tswitch(yyaccept){\n";
oline += 2;
}
o << "\tcase " << saves[i] << ":";
genGoTo(o, state, rules[i], readCh);
}
if(!first)
{
o << "\t}\n";
++oline;
}
}
Rule::Rule(State *s, RuleOp *r) : Action(s), rule(r) {
;
}
void Rule::emit(std::ostream &o, bool &readCh)
{
uint back = rule->ctx->fixedLength();
if(back != ~0u && back > 0u) {
o << "\tYYCURSOR -= " << back << ";";
}
o << "\n";
++oline;
line_source(rule->code->line, o);
o << rule->code->text;
// not sure if we need this or not. oline += std::count(rule->code->text, rule->code->text + ::strlen(rule->code->text), '\n');
o << "\n";
++oline;
o << "#line " << ++oline << " \"" << outputFileName << "\"\n";
// o << "\n#line " << rule->code->line
// << "\n\t" << rule->code->text << "\n";
}
void doLinear(std::ostream &o, uint i, Span *s, uint n, State *from, State *next, bool &readCh)
{
for(;;)
{
State *bg = s[0].to;
while(n >= 3 && s[2].to == bg && (s[1].ub - s[0].ub) == 1)
{
if(s[1].to == next && n == 3)
{
indent(o, i); genIf(o, "!=", s[0].ub, readCh); genGoTo(o, from, bg, readCh);
indent(o, i); genGoTo(o, from, next, readCh);
return;
}
else
{
indent(o, i); genIf(o, "==", s[0].ub, readCh); genGoTo(o, from, s[1].to, readCh);
}
n -= 2; s += 2;
}
if(n == 1)
{
// if(bg != next){
indent(o, i); genGoTo(o, from, s[0].to, readCh);
// }
return;
}
else if(n == 2 && bg == next)
{
indent(o, i); genIf(o, ">=", s[0].ub, readCh); genGoTo(o, from, s[1].to, readCh);
indent(o, i); genGoTo(o, from, next, readCh);
return;
}
else
{
indent(o, i); genIf(o, "<=", s[0].ub - 1, readCh); genGoTo(o, from, bg, readCh);
n -= 1; s += 1;
}
}
indent(o, i); genGoTo(o, from, next, readCh);
}
void Go::genLinear(std::ostream &o, State *from, State *next, bool &readCh)
{
doLinear(o, 0, span, nSpans, from, next, readCh);
}
void genCases(std::ostream &o, uint lb, Span *s)
{
if(lb < s->ub)
{
for(;;)
{
o << "\tcase '"; prtCh(o, lb); o << "':";
if(++lb == s->ub)
{
break;
}
o << "\n";
++oline;
}
}
}
void Go::genSwitch(std::ostream &o, State *from, State *next, bool &readCh)
{
if(nSpans <= 2){
genLinear(o, from, next, readCh);
}
else
{
State *def = span[nSpans-1].to;
Span **sP = new Span*[nSpans-1], **r, **s, **t;
t = &sP[0];
for(uint i = 0; i < nSpans; ++i)
{
if(span[i].to != def)
{
*(t++) = &span[i];
}
}
if (readCh)
{
o << "\tswitch((yych = *YYCURSOR)) {\n";
readCh =false;
}
else
{
o << "\tswitch(yych){\n";
}
++oline;
while(t != &sP[0])
{
r = s = &sP[0];
if(*s == &span[0])
genCases(o, 0, *s);
else
genCases(o, (*s)[-1].ub, *s);
State *to = (*s)->to;
while(++s < t)
{
if((*s)->to == to)
{
genCases(o, (*s)[-1].ub, *s);
}
else
{
*(r++) = *s;
}
}
genGoTo(o, from, to, readCh);
t = r;
}
o << "\tdefault:";
genGoTo(o, from, def, readCh);
o << "\t}\n";
++oline;
delete [] sP;
}
}
void doBinary(std::ostream &o, uint i, Span *s, uint n, State *from, State *next, bool &readCh)
{
if(n <= 4)
{
doLinear(o, i, s, n, from, next, readCh);
}
else
{
uint h = n/2;
indent(o, i); genIf(o, "<=", s[h-1].ub - 1, readCh); o << "{\n";
++oline;
doBinary(o, i+1, &s[0], h, from, next, readCh);
indent(o, i); o << "\t} else {\n";
++oline;
doBinary(o, i+1, &s[h], n - h, from, next, readCh);
indent(o, i); o << "\t}\n";
++oline;
}
}
void Go::genBinary(std::ostream &o, State *from, State *next, bool &readCh)
{
doBinary(o, 0, span, nSpans, from, next, readCh);
}
void Go::genBase(std::ostream &o, State *from, State *next, bool &readCh)
{
if(nSpans == 0)
{
return;
}
if(!sFlag)
{
genSwitch(o, from, next, readCh);
return;
}
if(nSpans > 8)
{
Span *bot = &span[0], *top = &span[nSpans-1];
uint util;
if(bot[0].to == top[0].to)
{
util = (top[-1].ub - bot[0].ub)/(nSpans - 2);
}
else
{
if(bot[0].ub > (top[0].ub - top[-1].ub))
{
util = (top[0].ub - bot[0].ub)/(nSpans - 1);
}
else
{
util = top[-1].ub/(nSpans - 1);
}
}
if(util <= 2)
{
genSwitch(o, from, next, readCh);
return;
}
}
if(nSpans > 5)
{
genBinary(o, from, next, readCh);
}
else
{
genLinear(o, from, next, readCh);
}
}
void Go::genGoto(std::ostream &o, State *from, State *next, bool &readCh)
{
if(bFlag)
{
for(uint i = 0; i < nSpans; ++i)
{
State *to = span[i].to;
if(to && to->isBase)
{
BitMap *b = BitMap::find(to);
if(b && matches(b->go, b->on, this, to))
{
Go go;
go.span = new Span[nSpans];
go.unmap(this, to);
o << "\tif(yybm[" << b->i << "+";
if (readCh)
{
o << "(yych = *YYCURSOR)";
}
else
{
o << "yych";
}
o << "] & " << (uint) b->m << ")";
genGoTo(o, from, to, readCh);
go.genBase(o, from, next, readCh);
delete [] go.span;
return;
}
}
}
}
genBase(o, from, next, readCh);
}
void State::emit(std::ostream &o, bool &readCh){
o << "yy" << label << ":";
/* o << "\nfprintf(stderr, \"<" << label << ">\");\n";*/
action->emit(o, readCh);
}
uint merge(Span *x0, State *fg, State *bg)
{
Span *x = x0, *f = fg->go.span, *b = bg->go.span;
uint nf = fg->go.nSpans, nb = bg->go.nSpans;
State *prev = NULL, *to;
// NB: we assume both spans are for same range
for(;;)
{
if(f->ub == b->ub)
{
to = f->to == b->to? bg : f->to;
if(to == prev){
--x;
}
else
{
x->to = prev = to;
}
x->ub = f->ub;
++x; ++f; --nf; ++b; --nb;
if(nf == 0 && nb == 0)
{
return x - x0;
}
}
while(f->ub < b->ub)
{
to = f->to == b->to? bg : f->to;
if(to == prev)
{
--x;
}
else
{
x->to = prev = to;
}
x->ub = f->ub;
++x; ++f; --nf;
}
while(b->ub < f->ub)
{
to = b->to == f->to? bg : f->to;
if(to == prev)
{
--x;
}
else
{
x->to = prev = to;
}
x->ub = b->ub;
++x; ++b; --nb;
}
}
}
const uint cInfinity = ~0;
class SCC {
public:
State **top, **stk;
public:
SCC(uint);
~SCC();
void traverse(State*);
};
SCC::SCC(uint size){
top = stk = new State*[size];
}
SCC::~SCC(){
delete [] stk;
}
void SCC::traverse(State *x)
{
*top = x;
uint k = ++top - stk;
x->depth = k;
for(uint i = 0; i < x->go.nSpans; ++i)
{
State *y = x->go.span[i].to;
if(y)
{
if(y->depth == 0)
{
traverse(y);
}
if(y->depth < x->depth)
{
x->depth = y->depth;
}
}
}
if(x->depth == k)
{
do
{
(*--top)->depth = cInfinity;
(*top)->link = x;
} while(*top != x);
}
}
uint maxDist(State *s)
{
uint mm = 0;
for(uint i = 0; i < s->go.nSpans; ++i)
{
State *t = s->go.span[i].to;
if(t)
{
uint m = 1;
if(!t->link)
{
m += maxDist(t);
}
if(m > mm)
{
mm = m;
}
}
}
return mm;
}
void calcDepth(State *head)
{
State *t;
for(State *s = head; s; s = s->next)
{
if(s->link == s){
for(uint i = 0; i < s->go.nSpans; ++i)
{
t = s->go.span[i].to;
if(t && t->link == s)
{
goto inSCC;
}
}
s->link = NULL;
}else
{
inSCC:
s->depth = maxDist(s);
}
}
}
void DFA::findSCCs()
{
SCC scc(nStates);
State *s;
for(s = head; s; s = s->next)
{
s->depth = 0;
s->link = NULL;
}
for(s = head; s; s = s->next)
{
if(!s->depth)
{
scc.traverse(s);
}
}
calcDepth(head);
}
void DFA::split(State *s)
{
State *move = new State;
(void) new Move(move);
addState(&s->next, move);
move->link = s->link;
move->rule = s->rule;
move->go = s->go;
s->rule = NULL;
s->go.nSpans = 1;
s->go.span = new Span[1];
s->go.span[0].ub = ubChar;
s->go.span[0].to = move;
}
void DFA::emit(std::ostream &o)
{
static uint label = 0;
State *s;
uint i;
findSCCs();
head->link = head;
head->depth = maxDist(head);
uint nRules = 0;
for(s = head; s; s = s->next)
{
if(s->rule && s->rule->accept >= nRules)
{
nRules = s->rule->accept + 1;
}
}
uint nSaves = 0;
uint *saves = new uint[nRules];
memset(saves, ~0, (nRules)*sizeof(*saves));
// mark backtracking points
for(s = head; s; s = s->next)
{
RuleOp *ignore = NULL;
if(s->rule)
{
for(i = 0; i < s->go.nSpans; ++i)
{
if(s->go.span[i].to && !s->go.span[i].to->rule){
delete s->action;
if(saves[s->rule->accept] == ~0u)
{
saves[s->rule->accept] = nSaves++;
}
(void) new Save(s, saves[s->rule->accept]);
continue;
}
}
ignore = s->rule;
}
}
// insert actions
State **rules = new State*[nRules];
memset(rules, 0, (nRules)*sizeof(*rules));
State *accept = NULL;
for(s = head; s; s = s->next)
{
State *ow;
if(!s->rule)
{
ow = accept;
}
else
{
if(!rules[s->rule->accept])
{
State *n = new State;
(void) new Rule(n, s->rule);
rules[s->rule->accept] = n;
addState(&s->next, n);
}
ow = rules[s->rule->accept];
}
for(i = 0; i < s->go.nSpans; ++i)
if(!s->go.span[i].to)
{
if(!ow)
{
ow = accept = new State;
(void) new Accept(accept, nRules, saves, rules);
addState(&s->next, accept);
}
s->go.span[i].to = ow;
}
}
// split ``base'' states into two parts
for(s = head; s; s = s->next)
{
s->isBase = false;
if(s->link)
{
for(i = 0; i < s->go.nSpans; ++i)
{
if(s->go.span[i].to == s){
s->isBase = true;
split(s);
if(bFlag)
{
BitMap::find(&s->next->go, s);
}
s = s->next;
break;
}
}
}
}
// find ``base'' state, if possible
Span *span = new Span[ubChar - lbChar];
for(s = head; s; s = s->next)
{
if(!s->link)
{
for(i = 0; i < s->go.nSpans; ++i)
{
State *to = s->go.span[i].to;
if(to && to->isBase)
{
to = to->go.span[0].to;
uint nSpans = merge(span, s, to);
if(nSpans < s->go.nSpans)
{
delete [] s->go.span;
s->go.nSpans = nSpans;
s->go.span = new Span[nSpans];
memcpy(s->go.span, span, nSpans*sizeof(Span));
}
break;
}
}
}
}
delete [] span;
delete head->action;
++oline;
o << "\n#line " << ++oline << " \"" << outputFileName << "\"\n";
o << "{\n\tYYCTYPE yych;\n\tunsigned int yyaccept;\n";
oline += 3;
if(bFlag)
{
BitMap::gen(o, lbChar, ubChar);
}
o << "\tgoto yy" << label << ";\n";
++oline;
(void) new Enter(head, label++);
for(s = head; s; s = s->next)
{
s->label = label++;
}
for(s = head; s; s = s->next)
{
bool readCh = false;
s->emit(o, readCh);
s->go.genGoto(o, s, s->next, readCh);
}
o << "}\n";
++oline;
BitMap::first = NULL;
delete [] saves;
delete [] rules;
}

118
tools/re2c/config.h.in Normal file
View file

@ -0,0 +1,118 @@
/* config.h.in. Generated from configure.in by autoheader. */
/* Define to 1 if you have the `getpagesize' function. */
#undef HAVE_GETPAGESIZE
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
to 0 otherwise. */
#undef HAVE_MALLOC
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
/* Define to 1 if you have the `memset' function. */
#undef HAVE_MEMSET
/* Define to 1 if you have a working `mmap' system call. */
#undef HAVE_MMAP
/* Define to 1 if you have the `munmap' function. */
#undef HAVE_MUNMAP
/* Define to 1 if stdbool.h conforms to C99. */
#undef HAVE_STDBOOL_H
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdlib.h> header file. */
#undef HAVE_STDLIB_H
/* Define to 1 if you have the `strdup' function. */
#undef HAVE_STRDUP
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
/* Define to 1 if you have the <string.h> header file. */
#undef HAVE_STRING_H
/* Define to 1 if you have the <sys/stat.h> header file. */
#undef HAVE_SYS_STAT_H
/* Define to 1 if you have the <sys/types.h> header file. */
#undef HAVE_SYS_TYPES_H
/* Define to 1 if you have uchar type definition */
#undef HAVE_UCHAR
/* Define to 1 if you have uint type definition */
#undef HAVE_UINT
/* Define to 1 if you have ulong type definition */
#undef HAVE_ULONG
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to 1 if you have ushort type definition */
#undef HAVE_USHORT
/* Define to 1 if the system has the type `_Bool'. */
#undef HAVE__BOOL
/* Name of package */
#undef PACKAGE
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#undef PACKAGE_NAME
/* Define to the full name and version of this package. */
#undef PACKAGE_STRING
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the version of this package. */
#undef PACKAGE_VERSION
/* The size of a `char', as computed by sizeof. */
#undef SIZEOF_CHAR
/* The size of a `int', as computed by sizeof. */
#undef SIZEOF_INT
/* The size of a `long', as computed by sizeof. */
#undef SIZEOF_LONG
/* The size of a `short', as computed by sizeof. */
#undef SIZEOF_SHORT
/* Define to 1 if you have the ANSI C header files. */
#undef STDC_HEADERS
/* Version number of package */
#undef VERSION
/* Define to empty if `const' does not conform to ANSI C. */
#undef const
/* Define as `__inline' if that's what the C compiler calls it, or to nothing
if it is not supported. */
#undef inline
/* Define to rpl_malloc if the replacement function should be used. */
#undef malloc
/* Define to `unsigned' if <sys/types.h> does not define. */
#undef size_t
/* Define to empty if the keyword `volatile' does not work. Warning: valid
code using `volatile' can become incorrect without. Disable with care. */
#undef volatile

7415
tools/re2c/configure vendored Normal file
View file

File diff suppressed because it is too large Load diff

59
tools/re2c/configure.in Normal file
View file

@ -0,0 +1,59 @@
AC_PREREQ([2.57])
AC_INIT(Makefile.am, 0.9.3, re2c-general@lists.sourceforge.net)
AM_INIT_AUTOMAKE(re2c, $PACKAGE_VERSION)
AC_CONFIG_SRCDIR(actions.cc)
AM_CONFIG_HEADER(config.h)
AC_SUBST(PACKAGE_VERSION)
AC_SUBST(PACKAGE_NAME)
AC_SUBST(PACKAGE_TARNAME)
# Checks for programs.
AC_PROG_YACC
AC_PROG_CXX
AC_PROG_CC
AC_PROG_INSTALL
# Checks for libraries.
# Checks for header files.
AC_HEADER_STDC
# We only rely on standard C/C++ headers. If we ever have a pressing need
# for something non-standard, we should add a check for it here.
# AC_CHECK_HEADERS([stddef.h stdlib.h string.h])
# Checks for typedefs, structures, and compiler characteristics.
AC_HEADER_STDBOOL
AC_C_CONST
AC_C_INLINE
AC_TYPE_SIZE_T
AC_C_VOLATILE
# Checks for library functions.
AC_FUNC_MALLOC
AC_FUNC_MEMCMP
AC_FUNC_MMAP
AC_CHECK_FUNCS([memset munmap strdup])
AC_CHECK_SIZEOF([char])
AC_CHECK_SIZEOF([short])
AC_CHECK_SIZEOF([int])
AC_CHECK_SIZEOF([long])
AC_CHECK_TYPE([uchar], [
AC_DEFINE([HAVE_UCHAR], [1], [Define to 1 if you have uchar type definition])
])
AC_CHECK_TYPE([ushort], [
AC_DEFINE([HAVE_USHORT], [1], [Define to 1 if you have ushort type definition])
])
AC_CHECK_TYPE([uint], [
AC_DEFINE([HAVE_UINT], [1], [Define to 1 if you have uint type definition])
])
AC_CHECK_TYPE([ulong], [
AC_DEFINE([HAVE_ULONG], [1], [Define to 1 if you have ulong type definition])
])
AC_CONFIG_FILES([Makefile README makerpm re2c.1 re2c.spec])
AC_OUTPUT_COMMANDS([chmod +x makerpm])
AC_OUTPUT

118
tools/re2c/configwin.h Normal file
View file

@ -0,0 +1,118 @@
/* config.h.in. Generated from configure.in by autoheader. */
/* Define to 1 if you have the `getpagesize' function. */
#undef HAVE_GETPAGESIZE
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
to 0 otherwise. */
#undef HAVE_MALLOC
#include <sys/types.h>
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define to 1 if you have the `memset' function. */
#define HAVE_MEMSET 1
/* Define to 1 if you have a working `mmap' system call. */
#undef HAVE_MMAP
/* Define to 1 if you have the `munmap' function. */
#undef HAVE_MUNMAP
/* Define to 1 if stdbool.h conforms to C99. */
#undef HAVE_STDBOOL_H
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the `strdup' function. */
#define HAVE_STRDUP 1
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have uchar type definition */
#undef HAVE_UCHAR
/* Define to 1 if you have uint type definition */
#undef HAVE_UINT
/* Define to 1 if you have ulong type definition */
#undef HAVE_ULONG
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to 1 if you have ushort type definition */
#undef HAVE_USHORT
/* Define to 1 if the system has the type `_Bool'. */
#undef HAVE__BOOL
/* Name of package */
#undef PACKAGE
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#define PACKAGE_NAME "re2c"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "re2c 0.9.3"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "re2c"
/* Define to the version of this package. */
#define PACKAGE_VERSION "0.9.3"
/* The size of a `char', as computed by sizeof. */
#define SIZEOF_CHAR 1
/* The size of a `int', as computed by sizeof. */
#define SIZEOF_INT 4
/* The size of a `long', as computed by sizeof. */
#define SIZEOF_LONG 4
/* The size of a `short', as computed by sizeof. */
#define SIZEOF_SHORT 2
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Version number of package */
#define VERSION "0.9.3"
/* Define to empty if `const' does not conform to ANSI C. */
//#undef const
/* Define as `__inline' if that's what the C compiler calls it, or to nothing
if it is not supported. */
//#undef inline
/* Define to rpl_malloc if the replacement function should be used. */
//#undef malloc
/* Define to `unsigned' if <sys/types.h> does not define. */
//#undef size_t
/* Define to empty if the keyword `volatile' does not work. Warning: valid
code using `volatile' can become incorrect without. Disable with care. */
//#undef volatile

226
tools/re2c/dfa.cc Normal file
View file

@ -0,0 +1,226 @@
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include "globals.h"
#include "substr.h"
#include "dfa.h"
inline char octCh(uint c){
return '0' + c%8;
}
void prtCh(std::ostream &o, uchar c){
uchar oc = talx[c];
switch(oc){
case '\'': o << "\\'"; break;
case '\n': o << "\\n"; break;
case '\t': o << "\\t"; break;
case '\v': o << "\\v"; break;
case '\b': o << "\\b"; break;
case '\r': o << "\\r"; break;
case '\f': o << "\\f"; break;
case '\a': o << "\\a"; break;
case '\\': o << "\\\\"; break;
default:
if(isprint(oc))
o << (char) oc;
else
o << '\\' << octCh(c/64) << octCh(c/8) << octCh(c);
}
}
void printSpan(std::ostream &o, uint lb, uint ub){
if(lb > ub)
o << "*";
o << "[";
if((ub - lb) == 1){
prtCh(o, lb);
} else {
prtCh(o, lb);
o << "-";
prtCh(o, ub-1);
}
o << "]";
}
uint Span::show(std::ostream &o, uint lb){
if(to){
printSpan(o, lb, ub);
o << " " << to->label << "; ";
}
return ub;
}
std::ostream& operator<<(std::ostream &o, const State &s){
o << "state " << s.label;
if(s.rule)
o << " accepts " << s.rule->accept;
o << "\n";
++oline;
uint lb = 0;
for(uint i = 0; i < s.go.nSpans; ++i)
lb = s.go.span[i].show(o, lb);
return o;
}
std::ostream& operator<<(std::ostream &o, const DFA &dfa){
for(State *s = dfa.head; s; s = s->next)
{
o << s << "\n\n";
++oline;
}
return o;
}
State::State() : rule(NULL), link(NULL), kCount(0), kernel(NULL), action(NULL) {
go.nSpans = 0;
go.span = NULL;
}
State::~State(){
delete [] kernel;
delete [] go.span;
}
static Ins **closure(Ins **cP, Ins *i){
while(!isMarked(i)){
mark(i);
*(cP++) = i;
if(i->i.tag == FORK){
cP = closure(cP, i + 1);
i = (Ins*) i->i.link;
} else if(i->i.tag == GOTO){
i = (Ins*) i->i.link;
} else
break;
}
return cP;
}
struct GoTo {
Char ch;
void *to;
};
DFA::DFA(Ins *ins, uint ni, uint lb, uint ub, Char *rep)
: lbChar(lb), ubChar(ub) {
Ins **work = new Ins*[ni+1];
uint nc = ub - lb;
GoTo *goTo = new GoTo[nc];
Span *span = new Span[nc];
memset((char*) goTo, 0, nc*sizeof(GoTo));
tail = &head;
head = NULL;
nStates = 0;
toDo = NULL;
findState(work, closure(work, &ins[0]) - work);
while(toDo){
State *s = toDo;
toDo = s->link;
Ins **cP, **iP, *i;
uint nGoTos = 0;
uint j;
s->rule = NULL;
for(iP = s->kernel; (i = *iP); ++iP){
if(i->i.tag == CHAR){
for(Ins *j = i + 1; j < (Ins*) i->i.link; ++j){
if(!(j->c.link = goTo[j->c.value - lb].to))
goTo[nGoTos++].ch = j->c.value;
goTo[j->c.value - lb].to = j;
}
} else if(i->i.tag == TERM){
if(!s->rule || ((RuleOp*) i->i.link)->accept < s->rule->accept)
s->rule = (RuleOp*) i->i.link;
}
}
for(j = 0; j < nGoTos; ++j){
GoTo *go = &goTo[goTo[j].ch - lb];
i = (Ins*) go->to;
for(cP = work; i; i = (Ins*) i->c.link)
cP = closure(cP, i + i->c.bump);
go->to = findState(work, cP - work);
}
s->go.nSpans = 0;
for(j = 0; j < nc;){
State *to = (State*) goTo[rep[j]].to;
while(++j < nc && goTo[rep[j]].to == to);
span[s->go.nSpans].ub = lb + j;
span[s->go.nSpans].to = to;
s->go.nSpans++;
}
for(j = nGoTos; j-- > 0;)
goTo[goTo[j].ch - lb].to = NULL;
s->go.span = new Span[s->go.nSpans];
memcpy((char*) s->go.span, (char*) span, s->go.nSpans*sizeof(Span));
(void) new Match(s);
}
delete [] work;
delete [] goTo;
delete [] span;
}
DFA::~DFA(){
State *s;
while((s = head)){
head = s->next;
delete s;
}
}
void DFA::addState(State **a, State *s){
s->label = nStates++;
s->next = *a;
*a = s;
if(a == tail)
tail = &s->next;
}
State *DFA::findState(Ins **kernel, uint kCount){
Ins **cP, **iP, *i;
State *s;
kernel[kCount] = NULL;
cP = kernel;
for(iP = kernel; (i = *iP); ++iP){
if(i->i.tag == CHAR || i->i.tag == TERM){
*cP++ = i;
} else {
unmark(i);
}
}
kCount = cP - kernel;
kernel[kCount] = NULL;
for(s = head; s; s = s->next){
if(s->kCount == kCount){
for(iP = s->kernel; (i = *iP); ++iP)
if(!isMarked(i))
goto nextState;
goto unmarkAll;
}
nextState:;
}
s = new State;
addState(tail, s);
s->kCount = kCount;
s->kernel = new Ins*[kCount+1];
memcpy(s->kernel, kernel, (kCount+1)*sizeof(Ins*));
s->link = toDo;
toDo = s;
unmarkAll:
for(iP = kernel; (i = *iP); ++iP)
unmark(i);
return s;
}

174
tools/re2c/dfa.h Normal file
View file

@ -0,0 +1,174 @@
/* $Id: dfa.h,v 1.5 2004/05/13 02:58:17 nuffer Exp $ */
#ifndef _dfa_h
#define _dfa_h
#include <iosfwd>
#include "re.h"
extern void prtCh(std::ostream&, uchar);
extern void printSpan(std::ostream&, uint, uint);
class DFA;
class State;
class Action {
public:
State *state;
public:
Action(State*);
virtual void emit(std::ostream&, bool&) = 0;
virtual bool isRule() const;
virtual bool isMatch() const;
virtual bool readAhead() const;
};
class Match: public Action {
public:
Match(State*);
void emit(std::ostream&, bool&);
bool isMatch() const;
};
class Enter: public Action {
public:
uint label;
public:
Enter(State*, uint);
void emit(std::ostream&, bool&);
};
class Save: public Match {
public:
uint selector;
public:
Save(State*, uint);
void emit(std::ostream&, bool&);
bool isMatch() const;
};
class Move: public Action {
public:
Move(State*);
void emit(std::ostream&, bool&);
};
class Accept: public Action {
public:
uint nRules;
uint *saves;
State **rules;
public:
Accept(State*, uint, uint*, State**);
void emit(std::ostream&, bool&);
};
class Rule: public Action {
public:
RuleOp *rule;
public:
Rule(State*, RuleOp*);
void emit(std::ostream&, bool&);
bool isRule() const;
};
class Span {
public:
uint ub;
State *to;
public:
uint show(std::ostream&, uint);
};
class Go {
public:
uint nSpans;
Span *span;
public:
void genGoto(std::ostream&, State *from, State*, bool &readCh);
void genBase(std::ostream&, State *from, State*, bool &readCh);
void genLinear(std::ostream&, State *from, State*, bool &readCh);
void genBinary(std::ostream&, State *from, State*, bool &readCh);
void genSwitch(std::ostream&, State *from, State*, bool &readCh);
void compact();
void unmap(Go*, State*);
};
class State {
public:
uint label;
RuleOp *rule;
State *next;
State *link;
uint depth; // for finding SCCs
uint kCount;
Ins **kernel;
bool isBase:1;
Go go;
Action *action;
public:
State();
~State();
void emit(std::ostream&, bool&);
friend std::ostream& operator<<(std::ostream&, const State&);
friend std::ostream& operator<<(std::ostream&, const State*);
};
class DFA {
public:
uint lbChar;
uint ubChar;
uint nStates;
State *head, **tail;
State *toDo;
public:
DFA(Ins*, uint, uint, uint, Char*);
~DFA();
void addState(State**, State*);
State *findState(Ins**, uint);
void split(State*);
void findSCCs();
void emit(std::ostream&);
friend std::ostream& operator<<(std::ostream&, const DFA&);
friend std::ostream& operator<<(std::ostream&, const DFA*);
};
inline Action::Action(State *s) : state(s) {
s->action = this;
}
inline bool Action::isRule() const
{ return false; }
inline bool Action::isMatch() const
{ return false; }
inline bool Action::readAhead() const
{ return !isMatch() || (state && state->next && state->next->action && !state->next->action->isRule()); }
inline Match::Match(State *s) : Action(s)
{ }
inline bool Match::isMatch() const
{ return true; }
inline Enter::Enter(State *s, uint l) : Action(s), label(l)
{ }
inline Save::Save(State *s, uint i) : Match(s), selector(i)
{ }
inline bool Save::isMatch() const
{ return false; }
inline bool Rule::isRule() const
{ return true; }
inline std::ostream& operator<<(std::ostream &o, const State *s)
{ return o << *s; }
inline std::ostream& operator<<(std::ostream &o, const DFA *dfa)
{ return o << *dfa; }
#endif

5249
tools/re2c/doc/loplas.ps Normal file
View file

File diff suppressed because it is too large Load diff

48
tools/re2c/doc/sample.bib Normal file
View file

@ -0,0 +1,48 @@
@Article{Bumbulis94,
author = {Peter Bumbulis and Donald D. Cowan},
title = {RE2C -- A More Versatile Scanner Generator},
journal = "ACM Letters on Programming Languages and Systems",
volume = 2,
number = "1--4",
year = 1994,
abstract = {
It is usually claimed that lexical analysis routines are still coded by
hand, despite the widespread availability of scanner generators, for
efficiency reasons. While efficiency is a consideration, there exist
freely available scanner generators such as GLA \cite{Gray88} that can
generate scanners that are faster than most hand-coded ones. However,
most generated scanners are tailored for a particular environment, and
retargetting these scanners to other environments, if possible, is
usually complex enough to make a hand-coded scanner more appealing. In
this paper we describe RE2C, a scanner generator that not only generates
scanners which are faster (and usually smaller) than those produced by
any other scanner generator known to the authors, including GLA, but
also adapt easily to any environment.
}
}
@Article{Gray88,
author = {Robert W. Gray},
title = {{$\gamma$-GLA} - {A} Generator for Lexical Analyzers That
Programmers Can Use},
journal = {USENIX Conference Proceedings},
year = {1988},
month = {June},
pages = {147-160},
abstract = {Writing an efficient lexical analyzer for even a simple
language is not a trivial task, and should not be done by hand. We
describe GLA, a tool that generates very efficient scanners. These
scanners do not use the conventional transition matrix, but instead
use a few 128 element vectors. Scanning time is only slightly
greater than the absolute minimum --- the time it takes to look at
each character in a file. The GLA language allows simple, concise
specification of scanners. Augmenting regular expressions with
auxiliary scanners easily handles nasty problems such as C comments
and C literal constants. We formalize the connection between token
scanning and token processing by associating a processor with
appropriate patterns. A library of canned descriptions simplifies the
specification of commonly used language pieces --- such as,
C\_IDENTIFIERS, C\_STRINGS, PASCAL\_COMMENTS, etc. Finally, carefully
tuned lexical analysis support modules are provided for error
handling, input buffering, storing identifiers in hash tables and
manipulating denotations.}
}

26
tools/re2c/examples/basemmap.c Normal file
View file

@ -0,0 +1,26 @@
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <string.h>
#ifndef MAP_NORESERVE
#define MAP_NORESERVE 0
#endif
volatile char ch;
main(){
struct stat statbuf;
uchar *buf;
fstat(0, &statbuf);
buf = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED|MAP_NORESERVE,
0, 0);
if(buf != (uchar*)(-1)){
uchar *cur, *lim = &buf[statbuf.st_size];
for(cur = buf; buf != lim; ++cur){
ch = *cur;
}
munmap(buf, statbuf.st_size);
}
}

272
tools/re2c/examples/c.re Normal file
View file

@ -0,0 +1,272 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof) {
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
close(in.fd);
}

267
tools/re2c/examples/cmmap.re Normal file
View file

@ -0,0 +1,267 @@
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int unint;
typedef unsigned char uchar;
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
uchar *tok, *ptr, *cur, *pos, *lim, *eof;
unint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
unint cnt = s->lim - s->tok;
uchar *buf = malloc((cnt + 1)*sizeof(uchar));
memcpy(buf, s->tok, cnt);
cursor = &buf[cursor - s->tok];
s->pos = &buf[s->pos - s->tok];
s->ptr = &buf[s->ptr - s->tok];
s->lim = &buf[cnt];
s->eof = s->lim; *(s->eof)++ = '\n';
s->tok = buf;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
#ifndef MAP_NORESERVE
#define MAP_NORESERVE 0
#endif
main(){
Scanner in;
struct stat statbuf;
uchar *buf;
fstat(0, &statbuf);
buf = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED|MAP_NORESERVE,
0, 0);
if(buf != (uchar*)(-1)){
int t;
in.lim = &(in.cur = buf)[statbuf.st_size];
in.pos = NULL;
in.eof = NULL;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
munmap(buf, statbuf.st_size);
}
}

239
tools/re2c/examples/cnokw.re Normal file
View file

@ -0,0 +1,239 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
close(in.fd);
}

258
tools/re2c/examples/cunroll.re Normal file
View file

@ -0,0 +1,258 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
I = L|D;
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
X = any\[*/];
*/
/*!re2c
"/*" { goto comment; }
L { RET(ID); }
L I { RET(ID); }
L I I { RET(ID); }
L I I I { RET(ID); }
L I I I I { RET(ID); }
L I I I I I { RET(ID); }
L I I I I I I { RET(ID); }
L I I I I I I I { RET(ID); }
L I* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\[\n\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\[\n\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \t\v\f]+ { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
X { goto comment; }
X X { goto comment; }
X X X { goto comment; }
X X X X { goto comment; }
X X X X X { goto comment; }
X X X X X X { goto comment; }
X X X X X X X { goto comment; }
X X X X X X X X { goto comment; }
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\t%.*s\n", t, in.cur - in.tok, in.tok);
printf("%d\n", t);
*/
}
close(in.fd);
}

202
tools/re2c/examples/modula.re Normal file
View file

@ -0,0 +1,202 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL {cursor = fill(s, cursor);}
#define RETURN(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\n';
}
s->lim += cnt;
}
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
uint depth;
std:
s->tok = cursor;
/*!re2c
any = [\000-\377];
digit = [0-9];
letter = [a-zA-Z];
*/
/*!re2c
"(*" { depth = 1; goto comment; }
digit + {RETURN(1);}
digit + / ".." {RETURN(1);}
[0-7] + "B" {RETURN(2);}
[0-7] + "C" {RETURN(3);}
digit [0-9A-F] * "H" {RETURN(4);}
digit + "." digit * ("E" ([+-]) ? digit +) ? {RETURN(5);}
['] (any\[\n']) * ['] | ["] (any\[\n"]) * ["] {RETURN(6);}
"#" {RETURN(7);}
"&" {RETURN(8);}
"(" {RETURN(9);}
")" {RETURN(10);}
"*" {RETURN(11);}
"+" {RETURN(12);}
"," {RETURN(13);}
"-" {RETURN(14);}
"." {RETURN(15);}
".." {RETURN(16);}
"/" {RETURN(17);}
":" {RETURN(18);}
":=" {RETURN(19);}
";" {RETURN(20);}
"<" {RETURN(21);}
"<=" {RETURN(22);}
"<>" {RETURN(23);}
"=" {RETURN(24);}
">" {RETURN(25);}
">=" {RETURN(26);}
"[" {RETURN(27);}
"]" {RETURN(28);}
"^" {RETURN(29);}
"{" {RETURN(30);}
"|" {RETURN(31);}
"}" {RETURN(32);}
"~" {RETURN(33);}
"AND" {RETURN(34);}
"ARRAY" {RETURN(35);}
"BEGIN" {RETURN(36);}
"BY" {RETURN(37);}
"CASE" {RETURN(38);}
"CONST" {RETURN(39);}
"DEFINITION" {RETURN(40);}
"DIV" {RETURN(41);}
"DO" {RETURN(42);}
"ELSE" {RETURN(43);}
"ELSIF" {RETURN(44);}
"END" {RETURN(45);}
"EXIT" {RETURN(46);}
"EXPORT" {RETURN(47);}
"FOR" {RETURN(48);}
"FROM" {RETURN(49);}
"IF" {RETURN(50);}
"IMPLEMENTATION" {RETURN(51);}
"IMPORT" {RETURN(52);}
"IN" {RETURN(53);}
"LOOP" {RETURN(54);}
"MOD" {RETURN(55);}
"MODULE" {RETURN(56);}
"NOT" {RETURN(57);}
"OF" {RETURN(58);}
"OR" {RETURN(59);}
"POINTER" {RETURN(60);}
"PROCEDURE" {RETURN(61);}
"QUALIFIED" {RETURN(62);}
"RECORD" {RETURN(63);}
"REPEAT" {RETURN(64);}
"RETURN" {RETURN(65);}
"SET" {RETURN(66);}
"THEN" {RETURN(67);}
"TO" {RETURN(68);}
"TYPE" {RETURN(69);}
"UNTIL" {RETURN(70);}
"VAR" {RETURN(71);}
"WHILE" {RETURN(72);}
"WITH" {RETURN(73);}
letter (letter | digit) * {RETURN(74);}
[ \t]+ { goto std; }
"\n"
{
if(cursor == s->eof) RETURN(0);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*)"
{
if(--depth == 0)
goto std;
else
goto comment;
}
"(*" { ++depth; goto comment; }
"\n"
{
if(cursor == s->eof) RETURN(0);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
/*
void putStr(FILE *o, char *s, uint l){
while(l-- > 0)
putc(*s++, o);
}
*/
main(){
Scanner in;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while(scan(&in)){
/*
putc('<', stdout);
putStr(stdout, (char*) in.tok, in.cur - in.tok);
putc('>', stdout);
putc('\n', stdout);
*/
}
}

1
tools/re2c/examples/rexx/README Normal file
View file

@ -0,0 +1 @@
Replacement modules for an existing REXX interpreter. Not standalone.

319
tools/re2c/examples/rexx/rexx.l Normal file
View file

@ -0,0 +1,319 @@
#include "scanio.h"
#include "scanner.h"
#define CURSOR ch
#define LOADCURSOR ch = *cursor;
#define ADVANCE cursor++;
#define BACK(n) cursor -= (n);
#define CHECK(n) if((ScanCB.lim - cursor) < (n)){cursor = ScanFill(cursor);}
#define MARK(n) ScanCB.ptr = cursor; sel = (n);
#define REVERT cursor = ScanCB.ptr;
#define MARKER sel
#define RETURN(i) {ScanCB.cur = cursor; return i;}
int ScanToken(){
uchar *cursor = ScanCB.cur;
unsigned sel;
uchar ch;
ScanCB.tok = cursor;
ScanCB.eot = NULL;
/*!re2c
all = [\000-\377];
eof = [\000];
any = all\eof;
letter = [a-z]|[A-Z];
digit = [0-9];
symchr = letter|digit|[.!?_];
const = (digit|[.])symchr*([eE][+-]?digit+)?;
simple = (symchr\(digit|[.]))(symchr\[.])*;
stem = simple [.];
symbol = symchr*;
sqstr = ['] ((any\['\n])|(['][']))* ['];
dqstr = ["] ((any\["\n])|(["]["]))* ["];
str = sqstr|dqstr;
ob = [ \t]*;
not = [\\~];
A = [aA];
B = [bB];
C = [cC];
D = [dD];
E = [eE];
F = [fF];
G = [gG];
H = [hH];
I = [iI];
J = [jJ];
K = [kK];
L = [lL];
M = [mM];
N = [nN];
O = [oO];
P = [pP];
Q = [qQ];
R = [rR];
S = [sS];
T = [tT];
U = [uU];
V = [vV];
W = [wW];
X = [xX];
Y = [yY];
Z = [zZ];
*/
scan:
/*!re2c
"\n"
{
++(ScanCB.lineNum);
ScanCB.linePos = ScanCB.pos + (cursor - ScanCB.mrk);
RETURN(SU_EOL);
}
"|" ob "|"
{ RETURN(OP_CONCAT); }
"+"
{ RETURN(OP_PLUS); }
"-"
{ RETURN(OP_MINUS); }
"*"
{ RETURN(OP_MULT); }
"/"
{ RETURN(OP_DIV); }
"%"
{ RETURN(OP_IDIV); }
"/" ob "/"
{ RETURN(OP_REMAIN); }
"*" ob "*"
{ RETURN(OP_POWER); }
"="
{ RETURN(OP_EQUAL); }
not ob "=" | "<" ob ">" | ">" ob "<"
{ RETURN(OP_EQUAL_N); }
">"
{ RETURN(OP_GT); }
"<"
{ RETURN(OP_LT); }
">" ob "=" | not ob "<"
{ RETURN(OP_GE); }
"<" ob "=" | not ob ">"
{ RETURN(OP_LE); }
"=" ob "="
{ RETURN(OP_EQUAL_EQ); }
not ob "=" ob "="
{ RETURN(OP_EQUAL_EQ_N); }
">" ob ">"
{ RETURN(OP_GT_STRICT); }
"<" ob "<"
{ RETURN(OP_LT_STRICT); }
">" ob ">" ob "=" | not ob "<" ob "<"
{ RETURN(OP_GE_STRICT); }
"<" ob "<" ob "=" | not ob ">" ob ">"
{ RETURN(OP_LE_STRICT); }
"&"
{ RETURN(OP_AND); }
"|"
{ RETURN(OP_OR); }
"&" ob "&"
{ RETURN(OP_XOR); }
not
{ RETURN(OP_NOT); }
":"
{ RETURN(SU_COLON); }
","
{ RETURN(SU_COMMA); }
"("
{ RETURN(SU_POPEN); }
")"
{ RETURN(SU_PCLOSE); }
";"
{ RETURN(SU_EOC); }
A D D R E S S
{ RETURN(RX_ADDRESS); }
A R G
{ RETURN(RX_ARG); }
C A L L
{ RETURN(RX_CALL); }
D O
{ RETURN(RX_DO); }
D R O P
{ RETURN(RX_DROP); }
E L S E
{ RETURN(RX_ELSE); }
E N D
{ RETURN(RX_END); }
E X I T
{ RETURN(RX_EXIT); }
I F
{ RETURN(RX_IF); }
I N T E R P R E T
{ RETURN(RX_INTERPRET); }
I T E R A T E
{ RETURN(RX_ITERATE); }
L E A V E
{ RETURN(RX_LEAVE); }
N O P
{ RETURN(RX_NOP); }
N U M E R I C
{ RETURN(RX_NUMERIC); }
O P T I O N S
{ RETURN(RX_OPTIONS); }
O T H E R W I S E
{ RETURN(RX_OTHERWISE); }
P A R S E
{ RETURN(RX_PARSE); }
P R O C E D U R E
{ RETURN(RX_PROCEDURE); }
P U L L
{ RETURN(RX_PULL); }
P U S H
{ RETURN(RX_PUSH); }
Q U E U E
{ RETURN(RX_QUEUE); }
R E T U R N
{ RETURN(RX_RETURN); }
S A Y
{ RETURN(RX_SAY); }
S E L E C T
{ RETURN(RX_SELECT); }
S I G N A L
{ RETURN(RX_SIGNAL); }
T H E N
{ RETURN(RX_THEN); }
T R A C E
{ RETURN(RX_TRACE); }
W H E N
{ RETURN(RX_WHEN); }
O F F
{ RETURN(RXS_OFF); }
O N
{ RETURN(RXS_ON); }
B Y
{ RETURN(RXS_BY); }
D I G I T S
{ RETURN(RXS_DIGITS); }
E N G I N E E R I N G
{ RETURN(RXS_ENGINEERING); }
E R R O R
{ RETURN(RXS_ERROR); }
E X P O S E
{ RETURN(RXS_EXPOSE); }
F A I L U R E
{ RETURN(RXS_FAILURE); }
F O R
{ RETURN(RXS_FOR); }
F O R E V E R
{ RETURN(RXS_FOREVER); }
F O R M
{ RETURN(RXS_FORM); }
F U Z Z
{ RETURN(RXS_FUZZ); }
H A L T
{ RETURN(RXS_HALT); }
L I N E I N
{ RETURN(RXS_LINEIN); }
N A M E
{ RETURN(RXS_NAME); }
N O T R E A D Y
{ RETURN(RXS_NOTREADY); }
N O V A L U E
{ RETURN(RXS_NOVALUE); }
S C I E N T I F I C
{ RETURN(RXS_SCIENTIFIC); }
S O U R C E
{ RETURN(RXS_SOURCE); }
S Y N T A X
{ RETURN(RXS_SYNTAX); }
T O
{ RETURN(RXS_TO); }
U N T I L
{ RETURN(RXS_UNTIL); }
U P P E R
{ RETURN(RXS_UPPER); }
V A L U E
{ RETURN(RXS_VALUE); }
V A R
{ RETURN(RXS_VAR); }
V E R S I O N
{ RETURN(RXS_VERSION); }
W H I L E
{ RETURN(RXS_WHILE); }
W I T H
{ RETURN(RXS_WITH); }
const
{ RETURN(SU_CONST); }
simple
{ RETURN(SU_SYMBOL); }
stem
{ RETURN(SU_SYMBOL_STEM); }
symbol
{ RETURN(SU_SYMBOL_COMPOUND); }
str
{ RETURN(SU_LITERAL); }
str [bB] / (all\symchr)
{ RETURN(SU_LITERAL_BIN); }
str [xX] / (all\symchr)
{ RETURN(SU_LITERAL_HEX); }
eof
{ RETURN(SU_EOF); }
any
{ RETURN(SU_ERROR); }
*/
}
bool StripToken(){
uchar *cursor = ScanCB.cur;
unsigned depth;
uchar ch;
bool blanks = FALSE;
ScanCB.eot = cursor;
strip:
/*!re2c
"/*"
{
depth = 1;
goto comment;
}
"\r"
{ goto strip; }
[ \t]
{
blanks = TRUE;
goto strip;
}
[] / all
{ RETURN(blanks); }
*/
comment:
/*!re2c
"*/"
{
if(--depth == 0)
goto strip;
else
goto comment;
}
"\n"
{
++(ScanCB.lineNum);
ScanCB.linePos = ScanCB.pos + (cursor - ScanCB.mrk);
goto comment;
}
"/*"
{
++depth;
goto comment;
}
eof
{ RETURN(blanks); }
any
{
goto comment;
}
*/
}

41
tools/re2c/examples/rexx/scanio.c Normal file
View file

@ -0,0 +1,41 @@
uchar *ScanFill(uchar *cursor){
unsigned cnt = s->tok - s->bot;
s->pos += cursor - s->mrk;
if(cnt){
if(s->eot){
unsigned len = s->eot - s->tok;
memcpy(s->bot, s->tok, len);
s->eot = &s->bot[len];
if((len = s->lim - cursor) != 0)
memcpy(s->eot, cursor, len);
cursor = s->eot;
s->lim = &cursor[len];
} else {
memcpy(s->bot, s->tok, s->lim - s->tok);
cursor -= cnt;
s->lim -= cnt;
}
s->tok = s->bot;
s->ptr -= cnt;
}
if((s->top - s->lim) < 512){
uchar *buf = (uchar*) malloc(((s->lim - s->bot) + 512)*sizeof(uchar));
memcpy(buf, s->bot, s->lim - s->bot);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
if(s->eot)
s->eot = &buf[s->eot - s->bot];
cursor = &buf[cursor - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[512];
free(s->bot);
s->bot = buf;
}
s->mrk = cursor;
if(ScanCBIO.file){
if((cnt = read(ScanCBIO.u.f.fd, (char*) s->lim, 512)) != 512)
memset(&s->lim[cnt], 0, 512 - cnt);
s->lim += 512;
}
return cursor;
}

7
tools/re2c/examples/sample.re Normal file
View file

@ -0,0 +1,7 @@
/*!re2c
"print" {return PRINT;}
[a-z]+ {return ID;}
[0-9]+ {return DEC;}
"0x" [0-9a-f]+ {return HEX;}
[\000-\377] {return ERR;}
*/

13
tools/re2c/examples/simple.re Normal file
View file

@ -0,0 +1,13 @@
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\000-\377] {return NULL;}
*/
}

18
tools/re2c/globals.h Normal file
View file

@ -0,0 +1,18 @@
/* $Id: globals.h,v 1.4 2004/05/13 03:47:52 nuffer Exp $ */
#ifndef _globals_h
#define _globals_h
#include "basics.h"
extern char *fileName;
extern char *outputFileName;
extern bool sFlag;
extern bool bFlag;
extern unsigned int oline;
extern uchar asc2ebc[256];
extern uchar ebc2asc[256];
extern uchar *xlat, *talx;
#endif

41
tools/re2c/ins.h Normal file
View file

@ -0,0 +1,41 @@
/* $Id: ins.h,v 1.3 2004/05/13 02:58:17 nuffer Exp $ */
#ifndef _ins_h
#define _ins_h
#include "basics.h"
const uint nChars = 256;
typedef uchar Char;
const uint CHAR = 0;
const uint GOTO = 1;
const uint FORK = 2;
const uint TERM = 3;
const uint CTXT = 4;
union Ins {
struct {
byte tag;
byte marked;
void *link;
} i;
struct {
ushort value;
ushort bump;
void *link;
} c;
};
inline bool isMarked(Ins *i){
return i->i.marked != 0;
}
inline void mark(Ins *i){
i->i.marked = true;
}
inline void unmark(Ins *i){
i->i.marked = false;
}
#endif

251
tools/re2c/install-sh Normal file
View file

@ -0,0 +1,251 @@
#!/bin/sh
#
# install - install a program, script, or datafile
# This comes from X11R5 (mit/util/scripts/install.sh).
#
# Copyright 1991 by the Massachusetts Institute of Technology
#
# Permission to use, copy, modify, distribute, and sell this software and its
# documentation for any purpose is hereby granted without fee, provided that
# the above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear in supporting
# documentation, and that the name of M.I.T. not be used in advertising or
# publicity pertaining to distribution of the software without specific,
# written prior permission. M.I.T. makes no representations about the
# suitability of this software for any purpose. It is provided "as is"
# without express or implied warranty.
#
# Calling this script install-sh is preferred over install.sh, to prevent
# `make' implicit rules from creating a file called install from it
# when there is no Makefile.
#
# This script is compatible with the BSD install script, but was written
# from scratch. It can only install one file at a time, a restriction
# shared with many OS's install programs.
# set DOITPROG to echo to test this script
# Don't use :- since 4.3BSD and earlier shells don't like it.
doit="${DOITPROG-}"
# put in absolute paths if you don't have them in your path; or use env. vars.
mvprog="${MVPROG-mv}"
cpprog="${CPPROG-cp}"
chmodprog="${CHMODPROG-chmod}"
chownprog="${CHOWNPROG-chown}"
chgrpprog="${CHGRPPROG-chgrp}"
stripprog="${STRIPPROG-strip}"
rmprog="${RMPROG-rm}"
mkdirprog="${MKDIRPROG-mkdir}"
transformbasename=""
transform_arg=""
instcmd="$mvprog"
chmodcmd="$chmodprog 0755"
chowncmd=""
chgrpcmd=""
stripcmd=""
rmcmd="$rmprog -f"
mvcmd="$mvprog"
src=""
dst=""
dir_arg=""
while [ x"$1" != x ]; do
case $1 in
-c) instcmd="$cpprog"
shift
continue;;
-d) dir_arg=true
shift
continue;;
-m) chmodcmd="$chmodprog $2"
shift
shift
continue;;
-o) chowncmd="$chownprog $2"
shift
shift
continue;;
-g) chgrpcmd="$chgrpprog $2"
shift
shift
continue;;
-s) stripcmd="$stripprog"
shift
continue;;
-t=*) transformarg=`echo $1 | sed 's/-t=//'`
shift
continue;;
-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
shift
continue;;
*) if [ x"$src" = x ]
then
src=$1
else
# this colon is to work around a 386BSD /bin/sh bug
:
dst=$1
fi
shift
continue;;
esac
done
if [ x"$src" = x ]
then
echo "install: no input file specified"
exit 1
else
true
fi
if [ x"$dir_arg" != x ]; then
dst=$src
src=""
if [ -d $dst ]; then
instcmd=:
chmodcmd=""
else
instcmd=mkdir
fi
else
# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
# might cause directories to be created, which would be especially bad
# if $src (and thus $dsttmp) contains '*'.
if [ -f $src -o -d $src ]
then
true
else
echo "install: $src does not exist"
exit 1
fi
if [ x"$dst" = x ]
then
echo "install: no destination specified"
exit 1
else
true
fi
# If destination is a directory, append the input filename; if your system
# does not like double slashes in filenames, you may need to add some logic
if [ -d $dst ]
then
dst="$dst"/`basename $src`
else
true
fi
fi
## this sed command emulates the dirname command
dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
# Make sure that the destination directory exists.
# this part is taken from Noah Friedman's mkinstalldirs script
# Skip lots of stat calls in the usual case.
if [ ! -d "$dstdir" ]; then
defaultIFS='
'
IFS="${IFS-${defaultIFS}}"
oIFS="${IFS}"
# Some sh's can't handle IFS=/ for some reason.
IFS='%'
set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
IFS="${oIFS}"
pathcomp=''
while [ $# -ne 0 ] ; do
pathcomp="${pathcomp}${1}"
shift
if [ ! -d "${pathcomp}" ] ;
then
$mkdirprog "${pathcomp}"
else
true
fi
pathcomp="${pathcomp}/"
done
fi
if [ x"$dir_arg" != x ]
then
$doit $instcmd $dst &&
if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
else
# If we're going to rename the final executable, determine the name now.
if [ x"$transformarg" = x ]
then
dstfile=`basename $dst`
else
dstfile=`basename $dst $transformbasename |
sed $transformarg`$transformbasename
fi
# don't allow the sed command to completely eliminate the filename
if [ x"$dstfile" = x ]
then
dstfile=`basename $dst`
else
true
fi
# Make a temp file name in the proper directory.
dsttmp=$dstdir/#inst.$$#
# Move or copy the file name to the temp name
$doit $instcmd $src $dsttmp &&
trap "rm -f ${dsttmp}" 0 &&
# and set any options; do chmod last to preserve setuid bits
# If any of these fail, we abort the whole thing. If we want to
# ignore errors from any of these, just make sure not to ignore
# errors from the above "$doit $instcmd $src $dsttmp" command.
if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
# Now rename the file to the real destination.
$doit $rmcmd -f $dstdir/$dstfile &&
$doit $mvcmd $dsttmp $dstdir/$dstfile
fi &&
exit 0

149
tools/re2c/main.cc Normal file
View file

@ -0,0 +1,149 @@
/* $Id: main.cc,v 1.10 2004/05/13 03:47:52 nuffer Exp $ */
#ifdef HAVE_CONFIG_H
#include "config.h"
#elif _WIN32
#include "configwin.h"
#endif
#include <fstream>
#include <iostream>
#include <stdlib.h>
#include <string.h>
#include "globals.h"
#include "parser.h"
#include "dfa.h"
#include "mbo_getopt.h"
char *fileName = 0;
char *outputFileName = 0;
bool sFlag = false;
bool bFlag = false;
unsigned int oline = 1;
using namespace std;
static char *opt_arg = NULL;
static int opt_ind = 1;
static const mbo_opt_struct OPTIONS[] = {
{'?', 0, "help"},
{'b', 0, "bit-vectors"},
{'e', 0, "ecb"},
{'h', 0, "help"},
{'s', 0, "nested-ifs"},
{'o', 1, "output"},
{'v', 0, "version"}
};
static void usage()
{
cerr << "usage: re2c [-esbvh] file\n"
"\n"
"-? -h --help Display this info.\n"
"\n"
"-b --bit-vectors Implies -s. Use bit vectors as well in the attempt to\n"
" coax better code out of the compiler. Most useful for\n"
" specifications with more than a few keywords (e.g. for\n"
" most programming languages).\n"
"\n"
"-e --ecb Cross-compile from an ASCII platform to\n"
" an EBCDIC one.\n"
"\n"
"-s --nested-ifs Generate nested ifs for some switches. Many compilers\n"
" need this assist to generate better code.\n"
"\n"
"-o --output=output Specify the output file instead of stdout\n"
"\n"
"-v --version Show version information.\n";
}
int main(int argc, char *argv[])
{
int c;
fileName = NULL;
if (argc == 1) {
usage();
return 2;
}
while ((c = mbo_getopt(argc, argv, OPTIONS, &opt_arg, &opt_ind, 0))!=-1) {
switch (c) {
case 'b':
sFlag = true;
bFlag = true;
break;
case 'e':
xlat = asc2ebc;
talx = ebc2asc;
break;
case 's':
sFlag = true;
break;
case 'o':
outputFileName = opt_arg;
break;
case 'v':
cerr << "re2c " << PACKAGE_VERSION << "\n";
return 2;
case 'h':
case '?':
default:
usage();
return 2;
}
}
if (argc == opt_ind + 1)
{
fileName = argv[opt_ind];
}
else
{
usage();
return 2;
}
// set up the input stream
istream* input = 0;
ifstream inputFile;
if (fileName[0] == '-' && fileName[1] == '\0')
{
fileName = "<stdin>";
input = &cin;
}
else
{
inputFile.open(fileName);
if (!inputFile)
{
cerr << "can't open " << fileName << "\n";
return 1;
}
input = &inputFile;
}
// set up the output stream
ostream* output = 0;
ofstream outputFile;
if (outputFileName == 0 || (fileName[0] == '-' && fileName[1] == '\0'))
{
outputFileName = "<stdout>";
output = &cout;
}
else
{
outputFile.open(outputFileName);
if (!outputFile)
{
cerr << "can't open " << outputFileName << "\n";
return 1;
}
output = &outputFile;
}
parse(*input, *output);
return 0;
}

45
tools/re2c/makerpm.in Normal file
View file

@ -0,0 +1,45 @@
#! /bin/sh
# $Id: makerpm.in,v 1.7 2004/04/19 23:31:34 helly Exp $
# M.Boerger <re2c@somabo.de>
PREFIX="@PACKAGE_NAME@"
VERSION="@PACKAGE_VERSION@"
TARDIR="`basename \`pwd\``"
RELEASE=${1:-1}
echo "Usage:"
echo "$0 <release>"
echo
echo "e.g.:"
echo "$0"
echo -n "Building RPM version $VERSION, release: $RELEASE "
sleep 1 ; echo -n . ; sleep 1 ; echo -n . ; sleep 1 ; echo -n .
echo
TAR=@PACKAGE_TARNAME@-$VERSION.tar.gz
SPEC=re2c-${VERSION}.spec
RPMBASE=/usr/src/redhat
for i in /usr/src/redhat /usr/src/packages /usr/src/RPM; do
if test -d $i; then
RPMBASE=$i
break
fi
done
RPMDIR=${RPMBASE}/RPMS
SPECDIR=${RPMBASE}/SPECS
SRCDIR=${RPMBASE}/SOURCES
make dist
cp $TAR $SRCDIR
cat re2c.spec | sed -e "s/RPM_RELEASE/${RELEASE}/g" > ${SPEC}
echo "CP: (`pwd`) cp -a $SPEC $SPECDIR/${SPEC}"
cp -a $SPEC $SPECDIR/${SPEC}
#cp -a *.patch $SRCDIR
cd $SPECDIR
echo "RPM: rpm -ba ${SPEC}"
rpmbuild -ba ${SPEC}

142
tools/re2c/mbo_getopt.cc Normal file
View file

@ -0,0 +1,142 @@
/*
Author: Marcus Boerger <helly@users.sourceforge.net>
*/
/* $Id: mbo_getopt.cc,v 1.1 2004/04/19 21:14:46 helly Exp $ */
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include "mbo_getopt.h"
#define OPTERRCOLON (1)
#define OPTERRNF (2)
#define OPTERRARG (3)
static int mbo_opt_error(int argc, char * const *argv, int oint, int optchr, int err, int show_err)
{
if (show_err)
{
fprintf(stderr, "Error in argument %d, char %d: ", oint, optchr+1);
switch(err)
{
case OPTERRCOLON:
fprintf(stderr, ": in flags\n");
break;
case OPTERRNF:
fprintf(stderr, "option not found %c\n", argv[oint][optchr]);
break;
case OPTERRARG:
fprintf(stderr, "no argument for option %c\n", argv[oint][optchr]);
break;
default:
fprintf(stderr, "unknown\n");
break;
}
}
return('?');
}
int mbo_getopt(int argc, char* const *argv, const mbo_opt_struct opts[], char **optarg, int *optind, int show_err)
{
static int optchr = 0;
static int dash = 0; /* have already seen the - */
int arg_start = 2;
int opts_idx = -1;
if (*optind >= argc) {
return(EOF);
}
if (!dash) {
if ((argv[*optind][0] != '-')) {
return(EOF);
} else {
if (!argv[*optind][1])
{
/*
* use to specify stdin. Need to let pgm process this and
* the following args
*/
return(EOF);
}
}
}
if ((argv[*optind][0] == '-') && (argv[*optind][1] == '-')) {
/* '--' indicates end of args if not followed by a known long option name */
while (1) {
opts_idx++;
if (opts[opts_idx].opt_char == '-') {
(*optind)++;
return(EOF);
} else if (opts[opts_idx].opt_name && !strcmp(&argv[*optind][2], opts[opts_idx].opt_name)) {
break;
}
}
optchr = 0;
dash = 1;
arg_start = 2 + strlen(opts[opts_idx].opt_name);
}
if (!dash) {
dash = 1;
optchr = 1;
}
/* Check if the guy tries to do a -: kind of flag */
if (argv[*optind][optchr] == ':') {
dash = 0;
(*optind)++;
return (mbo_opt_error(argc, argv, *optind-1, optchr, OPTERRCOLON, show_err));
}
if (opts_idx < 0) {
while (1) {
opts_idx++;
if (opts[opts_idx].opt_char == '-') {
int errind = *optind;
int errchr = optchr;
if (!argv[*optind][optchr+1]) {
dash = 0;
(*optind)++;
} else {
optchr++;
}
return(mbo_opt_error(argc, argv, errind, errchr, OPTERRNF, show_err));
} else if (argv[*optind][optchr] == opts[opts_idx].opt_char) {
break;
}
}
}
if (opts[opts_idx].need_param) {
/* Check for cases where the value of the argument
is in the form -<arg> <val> or in the form -<arg><val> */
dash = 0;
if(!argv[*optind][arg_start]) {
(*optind)++;
if (*optind == argc) {
return(mbo_opt_error(argc, argv, *optind-1, optchr, OPTERRARG, show_err));
}
*optarg = argv[(*optind)++];
} else {
*optarg = &argv[*optind][arg_start];
(*optind)++;
}
return opts[opts_idx].opt_char;
} else {
if (arg_start == 2) {
if (!argv[*optind][optchr+1])
{
dash = 0;
(*optind)++;
} else {
optchr++;
}
} else {
(*optind)++;
}
return opts[opts_idx].opt_char;
}
assert(0);
return(0); /* never reached */
}

16
tools/re2c/mbo_getopt.h Normal file
View file

@ -0,0 +1,16 @@
/*
Author: Marcus Boerger <helly@users.sourceforge.net>
*/
/* $Id: mbo_getopt.h,v 1.1 2004/04/19 21:14:46 helly Exp $ */
/* Define structure for one recognized option (both single char and long name).
* If short_open is '-' this is the last option.
*/
typedef struct _mbo_opt_struct {
const char opt_char;
const int need_param;
const char * opt_name;
} mbo_opt_struct;
int mbo_getopt(int argc, char* const *argv, const mbo_opt_struct opts[], char **optarg, int *optind, int show_err);

198
tools/re2c/missing Normal file
View file

@ -0,0 +1,198 @@
#! /bin/sh
# Common stub for a few missing GNU programs while installing.
# Copyright (C) 1996, 1997, 2001, 2002 Free Software Foundation, Inc.
# Franc,ois Pinard <pinard@iro.umontreal.ca>, 1996.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
if test $# -eq 0; then
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
fi
# In the cases where this matters, `missing' is being run in the
# srcdir already.
if test -f configure.in; then
configure_ac=configure.ac
else
configure_ac=configure.in
fi
case "$1" in
-h|--h|--he|--hel|--help)
echo "\
$0 [OPTION]... PROGRAM [ARGUMENT]...
Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
error status if there is no known handling for PROGRAM.
Options:
-h, --help display this help and exit
-v, --version output version information and exit
Supported PROGRAM values:
aclocal touch file \`aclocal.m4'
autoconf touch file \`configure'
autoheader touch file \`config.h.in'
automake touch all \`Makefile.in' files
bison create \`y.tab.[ch]', if possible, from existing .[ch]
flex create \`lex.yy.c', if possible, from existing .c
lex create \`lex.yy.c', if possible, from existing .c
makeinfo touch the output file
yacc create \`y.tab.[ch]', if possible, from existing .[ch]"
;;
-v|--v|--ve|--ver|--vers|--versi|--versio|--version)
echo "missing - GNU libit 0.0"
;;
-*)
echo 1>&2 "$0: Unknown \`$1' option"
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
;;
aclocal*)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`acinclude.m4' or \`$configure_ac'. You might want
to install the \`Automake' and \`Perl' packages. Grab them from
any GNU archive site."
touch aclocal.m4
;;
autoconf)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`$configure_ac'. You might want to install the
\`Autoconf' and \`GNU m4' packages. Grab them from any GNU
archive site."
touch configure
;;
autoheader)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`acconfig.h' or \`$configure_ac'. You might want
to install the \`Autoconf' and \`GNU m4' packages. Grab them
from any GNU archive site."
files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' $configure_ac`
test -z "$files" && files="config.h"
touch_files=
for f in $files; do
case "$f" in
*:*) touch_files="$touch_files "`echo "$f" |
sed -e 's/^[^:]*://' -e 's/:.*//'`;;
*) touch_files="$touch_files $f.in";;
esac
done
touch $touch_files
;;
automake*)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified \`Makefile.am', \`acinclude.m4' or \`$configure_ac'.
You might want to install the \`Automake' and \`Perl' packages.
Grab them from any GNU archive site."
find . -type f -name Makefile.am -print |
sed 's/\.am$/.in/' |
while read f; do touch "$f"; done
;;
bison|yacc)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified a \`.y' file. You may need the \`Bison' package
in order for those modifications to take effect. You can get
\`Bison' from any GNU archive site."
rm -f y.tab.c y.tab.h
if [ $# -ne 1 ]; then
eval LASTARG="\${$#}"
case "$LASTARG" in
*.y)
SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
if [ -f "$SRCFILE" ]; then
cp "$SRCFILE" y.tab.c
fi
SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
if [ -f "$SRCFILE" ]; then
cp "$SRCFILE" y.tab.h
fi
;;
esac
fi
if [ ! -f y.tab.h ]; then
echo >y.tab.h
fi
if [ ! -f y.tab.c ]; then
echo 'main() { return 0; }' >y.tab.c
fi
;;
lex|flex)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified a \`.l' file. You may need the \`Flex' package
in order for those modifications to take effect. You can get
\`Flex' from any GNU archive site."
rm -f lex.yy.c
if [ $# -ne 1 ]; then
eval LASTARG="\${$#}"
case "$LASTARG" in
*.l)
SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
if [ -f "$SRCFILE" ]; then
cp "$SRCFILE" lex.yy.c
fi
;;
esac
fi
if [ ! -f lex.yy.c ]; then
echo 'main() { return 0; }' >lex.yy.c
fi
;;
makeinfo)
echo 1>&2 "\
WARNING: \`$1' is missing on your system. You should only need it if
you modified a \`.texi' or \`.texinfo' file, or any other file
indirectly affecting the aspect of the manual. The spurious
call might also be the consequence of using a buggy \`make' (AIX,
DU, IRIX). You might want to install the \`Texinfo' package or
the \`GNU make' package. Grab either from any GNU archive site."
file=`echo "$*" | sed -n 's/.*-o \([^ ]*\).*/\1/p'`
if test -z "$file"; then
file=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
file=`sed -n '/^@setfilename/ { s/.* \([^ ]*\) *$/\1/; p; q; }' $file`
fi
touch $file
;;
*)
echo 1>&2 "\
WARNING: \`$1' is needed, and you do not seem to have it handy on your
system. You might have modified some files without having the
proper tools for further handling them. Check the \`README' file,
it often tells you about the needed prerequirements for installing
this package. You may also peek at any GNU archive site, in case
some other package would contain this missing \`$1' program."
exit 1
;;
esac
exit 0

40
tools/re2c/mkinstalldirs Normal file
View file

@ -0,0 +1,40 @@
#! /bin/sh
# mkinstalldirs --- make directory hierarchy
# Author: Noah Friedman <friedman@prep.ai.mit.edu>
# Created: 1993-05-16
# Public domain
# $Id: mkinstalldirs,v 1.13 1999/01/05 03:18:55 bje Exp $
errstatus=0
for file
do
set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
shift
pathcomp=
for d
do
pathcomp="$pathcomp$d"
case "$pathcomp" in
-* ) pathcomp=./$pathcomp ;;
esac
if test ! -d "$pathcomp"; then
echo "mkdir $pathcomp"
mkdir "$pathcomp" || lasterr=$?
if test ! -d "$pathcomp"; then
errstatus=$lasterr
fi
fi
pathcomp="$pathcomp/"
done
done
exit $errstatus
# mkinstalldirs ends here

1433
tools/re2c/parser.cc Normal file
View file

File diff suppressed because it is too large Load diff

23
tools/re2c/parser.h Normal file
View file

@ -0,0 +1,23 @@
/* $Id: parser.h,v 1.4 2004/05/13 02:58:17 nuffer Exp $ */
#ifndef _parser_h
#define _parser_h
#include "scanner.h"
#include "re.h"
#include <iosfwd>
class Symbol {
public:
static Symbol *first;
Symbol *next;
Str name;
RegExp *re;
public:
Symbol(const SubStr&);
static Symbol *find(const SubStr&);
};
void line_source(unsigned int, std::ostream&);
void parse(std::istream&, std::ostream&);
#endif

211
tools/re2c/parser.y Normal file
View file

@ -0,0 +1,211 @@
%{
/* $Id: parser.y,v 1.9 2004/05/13 14:26:01 nuffer Exp $ */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <time.h>
#include <string.h>
#include <stdlib.h>
#include <iostream>
#include "globals.h"
#include "parser.h"
extern "C"
{
int yyparse();
int yylex();
void yyerror(char*);
}
static uint accept;
static RegExp *spec;
static Scanner *in;
/* Bison version 1.875 emits a definition that is not working
* with several g++ version. Hence we disable it here.
*/
#if defined(__GNUC__)
#define __attribute__(x)
#endif
/* strdup() isn't standard C, so if we don't have it, we'll create our
* own version
*/
#if !defined(HAVE_STRDUP)
static char* strdup(const char* s)
{
char* rv = (char*)malloc(strlen(s) + 1);
if (rv == NULL)
return NULL;
strcpy(rv, s);
return rv;
}
#endif
%}
%start spec
%union {
Symbol *symbol;
RegExp *regexp;
Token *token;
char op;
ExtOp extop;
};
%token CLOSESIZE CLOSE ID CODE RANGE STRING
%type <op> CLOSE
%type <op> close
%type <extop> CLOSESIZE
%type <symbol> ID
%type <token> CODE
%type <regexp> RANGE STRING
%type <regexp> rule look expr diff term factor primary
%%
spec :
{ accept = 0;
spec = NULL; }
| spec rule
{ spec = spec? mkAlt(spec, $2) : $2; }
| spec decl
;
decl : ID '=' expr ';'
{ if($1->re)
in->fatal("sym already defined");
$1->re = $3; }
;
rule : expr look CODE
{ $$ = new RuleOp($1, $2, $3, accept++); }
;
look :
{ $$ = new NullOp; }
| '/' expr
{ $$ = $2; }
;
expr : diff
{ $$ = $1; }
| expr '|' diff
{ $$ = mkAlt($1, $3); }
;
diff : term
{ $$ = $1; }
| diff '\\' term
{ $$ = mkDiff($1, $3);
if(!$$)
in->fatal("can only difference char sets");
}
;
term : factor
{ $$ = $1; }
| term factor
{ $$ = new CatOp($1, $2); }
;
factor : primary
{ $$ = $1; }
| primary close
{
switch($2){
case '*':
$$ = mkAlt(new CloseOp($1), new NullOp());
break;
case '+':
$$ = new CloseOp($1);
break;
case '?':
$$ = mkAlt($1, new NullOp());
break;
}
}
| primary CLOSESIZE
{
$$ = new CloseVOp($1, $2.minsize, $2.maxsize);
}
;
close : CLOSE
{ $$ = $1; }
| close CLOSE
{ $$ = ($1 == $2) ? $1 : '*'; }
;
primary : ID
{ if(!$1->re)
in->fatal("can't find symbol");
$$ = $1->re; }
| RANGE
{ $$ = $1; }
| STRING
{ $$ = $1; }
| '(' expr ')'
{ $$ = $2; }
;
%%
extern "C" {
void yyerror(char* s){
in->fatal(s);
}
int yylex(){
return in->scan();
}
} // end extern "C"
void line_source(unsigned int line, std::ostream& o)
{
char * fnamebuf;
char * token;
o << "#line " << line << " \"";
if( fileName != NULL ) {
fnamebuf = strdup( fileName );
} else {
fnamebuf = strdup( "<stdin>" );
}
token = strtok( fnamebuf, "\\" );
for(;;) {
o << token;
token = strtok( NULL, "\\" );
if( token == NULL ) break;
o << "\\\\";
}
o << "\"\n";
++oline;
free( fnamebuf );
}
void parse(std::istream& i, std::ostream &o){
o << "/* Generated by re2c " PACKAGE_VERSION " on ";
time_t now = time(&now);
o.write(ctime(&now), 24);
o << " */\n";
oline += 2;
in = new Scanner(i);
line_source(in->line(), o);
while(in->echo(o)){
yyparse();
if(spec)
genCode(o, spec);
line_source(in->line(), o);
}
}

206
tools/re2c/re.h Normal file
View file

@ -0,0 +1,206 @@
/* $Id: re.h,v 1.5 2004/05/13 02:58:18 nuffer Exp $ */
#ifndef _re_h
#define _re_h
#include <iostream>
#include "token.h"
#include "ins.h"
typedef struct extop {
char op;
int minsize;
int maxsize;
} ExtOp;
struct CharPtn {
uint card;
CharPtn *fix;
CharPtn *nxt;
};
struct CharSet {
CharPtn *fix;
CharPtn *freeHead, **freeTail;
CharPtn *rep[nChars];
CharPtn ptn[nChars];
};
class Range {
public:
Range *next;
uint lb, ub; // [lb,ub)
public:
Range(uint l, uint u) : next(NULL), lb(l), ub(u)
{ }
Range(Range &r) : next(NULL), lb(r.lb), ub(r.ub)
{ }
friend std::ostream& operator<<(std::ostream&, const Range&);
friend std::ostream& operator<<(std::ostream&, const Range*);
};
inline std::ostream& operator<<(std::ostream &o, const Range *r){
return r? o << *r : o;
}
class RegExp {
public:
uint size;
public:
virtual char *typeOf() = 0;
RegExp *isA(char *t)
{ return typeOf() == t? this : NULL; }
virtual void split(CharSet&) = 0;
virtual void calcSize(Char*) = 0;
virtual uint fixedLength();
virtual void compile(Char*, Ins*) = 0;
virtual void display(std::ostream&) const = 0;
friend std::ostream& operator<<(std::ostream&, const RegExp&);
friend std::ostream& operator<<(std::ostream&, const RegExp*);
};
inline std::ostream& operator<<(std::ostream &o, const RegExp &re){
re.display(o);
return o;
}
inline std::ostream& operator<<(std::ostream &o, const RegExp *re){
return o << *re;
}
class NullOp: public RegExp {
public:
static char *type;
public:
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << "_";
}
};
class MatchOp: public RegExp {
public:
static char *type;
Range *match;
public:
MatchOp(Range *m) : match(m)
{ }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream&) const;
};
class RuleOp: public RegExp {
private:
RegExp *exp;
public:
RegExp *ctx;
static char *type;
Ins *ins;
uint accept;
Token *code;
uint line;
public:
RuleOp(RegExp*, RegExp*, Token*, uint);
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp << "/" << ctx << ";";
}
};
class AltOp: public RegExp {
private:
RegExp *exp1, *exp2;
public:
static char *type;
public:
AltOp(RegExp *e1, RegExp *e2)
{ exp1 = e1; exp2 = e2; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp1 << "|" << exp2;
}
friend RegExp *mkAlt(RegExp*, RegExp*);
};
class CatOp: public RegExp {
private:
RegExp *exp1, *exp2;
public:
static char *type;
public:
CatOp(RegExp *e1, RegExp *e2)
{ exp1 = e1; exp2 = e2; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
uint fixedLength();
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp1 << exp2;
}
};
class CloseOp: public RegExp {
private:
RegExp *exp;
public:
static char *type;
public:
CloseOp(RegExp *e)
{ exp = e; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp << "+";
}
};
class CloseVOp: public RegExp {
private:
RegExp *exp;
int min;
int max;
public:
static char *type;
public:
CloseVOp(RegExp *e, int lb, int ub)
{ exp = e; min = lb; max = ub; }
char *typeOf()
{ return type; }
void split(CharSet&);
void calcSize(Char*);
void compile(Char*, Ins*);
void display(std::ostream &o) const {
o << exp << "+";
}
};
extern void genCode(std::ostream&, RegExp*);
extern RegExp *mkDiff(RegExp*, RegExp*);
extern RegExp *strToRE(SubStr);
extern RegExp *ranToRE(SubStr);
extern RegExp *strToCaseInsensitiveRE(SubStr s);
#endif

601
tools/re2c/re2c.1 Normal file
View file

@ -0,0 +1,601 @@
./"
./" $Id: re2c.1.in,v 1.9 2004/05/26 13:33:25 nuffer Exp $
./"
.TH RE2C 1 "14 March 1994" "Version 0.9.3"
.ds re \fBre2c\fP
.ds le \fBlex\fP
.ds rx regular expression
.ds lx \fIl\fP-expression
\"$Log: re2c.1.in,v $
\"Revision 1.9 2004/05/26 13:33:25 nuffer
\"Added description of -o option.
\"
\"Revision 1.8 2004/04/19 22:32:48 helly
\"Update
\"
\"Revision 1.7 2004/04/19 02:13:48 helly
\"Featurerequest #869298 (Add case insensitive string literals)
\"
\"Revision 1.6 2004/04/17 15:49:13 helly
\"Fix example, cur must be set to make the uncommented printf's working
\"
\"Revision 1.5 2004/03/30 01:02:45 helly
\"Update docu
\"
\"Revision 1.4 2004/03/14 14:23:40 helly
\"Update
\"
\"Revision 1.3 2004/03/14 12:54:19 helly
\"Next step of autogen patch
\"
\"Revision 1.2 2004/03/13 20:35:12 helly
\"Updated configure stuff
\"
\"Revision 1.1 2004/01/31 15:44:39 nuffer
\"Applied patch from Marcus Boerger
\"
\"Revision 1.2 1994/04/16 15:50:32 peter
\"Fix bug in simple example.
\"
\"Revision 1.1 1994/04/08 15:39:09 peter
\"Initial revision
\"
.SH NAME
re2c \- convert regular expressions to C/C++
.SH SYNOPSIS
\*(re [\fB-esbvh\fP] [\fB-o output\fP] file\fP
.SH DESCRIPTION
\*(re is a preprocessor that generates C-based recognizers from regular
expressions.
The input to \*(re consists of C/C++ source interleaved with
comments of the form \fC/*!re2c\fP ... \fC*/\fP which contain
scanner specifications.
In the output these comments are replaced with code that, when
executed, will find the next input token and then execute
some user-supplied token-specific code.
For example, given the following code
.in +3
.nf
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\\000-\\377] {return NULL;}
*/
}
.fi
.in -3
\*(re will generate
.in +3
.nf
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
.fi
.in -3
.SH OPTIONS
\*(re provides the following options:
.TP
\fB-e\fP
Cross-compile from an ASCII platform to an EBCDIC one.
.TP
\fB-s\fP
Generate nested \fCif\fPs for some \fCswitch\fPes. Many compilers need this
assist to generate better code.
.TP
\fB-b\fP
Implies \fB-s\fP. Use bit vectors as well in the attempt to coax better
code out of the compiler. Most useful for specifications with more than a
few keywords (e.g. for most programming languages).
.TP
\fB-h\fP
\fB-?\fP
Invoke a short help.
.TP
\fB-v\fP
Show version information.
.TP
\fB-o output\fP
Specify the output file.
.SH "INTERFACE CODE"
Unlike other scanner generators, \*(re does not generate complete scanners:
the user must supply some interface code.
In particular, the user must define the following macros:
.TP
\fCYYCHAR\fP
Type used to hold an input symbol.
Usually \fCchar\fP or \fCunsigned char\fP.
.TP
\fCYYCURSOR\fP
\*(lx of type \fC*YYCHAR\fP that points to the current input symbol.
The generated code advances \fCYYCURSOR\fP as symbols are matched.
On entry, \fCYYCURSOR\fP is assumed to point to the first character of the
current token. On exit, \fCYYCURSOR\fP will point to the first character of
the following token.
.TP
\fCYLIMIT\fP
Expression of type \fC*YYCHAR\fP that marks the end of the buffer
(\fCYLIMIT[-1]\fP is the last character in the buffer).
The generated code repeatedly compares \fCYYCURSOR\fP to \fCYLIMIT\fP
to determine when the buffer needs (re)filling.
.TP
\fCYYMARKER\fP
\*(lx of type \fC*YYCHAR\fP.
The generated code saves backtracking information in \fCYYMARKER\fP.
.TP
\fCYYFILL(\fP\fIn\fP\fC)\fP
The generated code "calls" \fCYYFILL\fP when the buffer needs
(re)filling: at least \fIn\fP additional characters should
be provided. \fCYYFILL\fP should adjust \fCYYCURSOR\fP, \fCYYLIMIT\fP and
\fCYYMARKER\fP as needed. Note that for typical programming languages
\fIn\fP will be the length of the longest keyword plus one.
.SH "SCANNER SPECIFICATIONS"
Each scanner specification consists of a set of \fIrules\fP and name
definitions.
Rules consist of a regular expression along with a block of C/C++ code that
is to be executed when the associated regular expression is matched.
Name definitions are of the form
``\fIname\fP \fC=\fP \fIregular expression\fP\fC;\fP''.
.SH "SUMMARY OF RE2C REGULAR EXPRESSIONS"
.TP
\fC"foo"\fP
the literal string \fCfoo\fP.
ANSI-C escape sequences can be used.
.TP
\fC'foo'\fP
the literal string \fCfoo\fP (characters [a-zA-Z] treated case-insensitive).
ANSI-C escape sequences can be used.
.TP
\fC[xyz]\fP
a "character class"; in this case,
the \*(rx matches either an '\fCx\fP', a '\fCy\fP', or a '\fCz\fP'.
.TP
\fC[abj-oZ]\fP
a "character class" with a range in it;
matches an '\fCa\fP', a '\fCb\fP', any letter from '\fCj\fP' through '\fCo\fP',
or a '\fCZ\fP'.
.TP
\fIr\fP\fC\e\fP\fIs\fP
match any \fIr\fP which isn't an \fIs\fP. \fIr\fP and \fIs\fP must be regular expressions
which can be expressed as character classes.
.TP
\fIr\fP\fC*\fP
zero or more \fIr\fP's, where \fIr\fP is any regular expression
.TP
\fC\fIr\fP\fC+\fP
one or more \fIr\fP's
.TP
\fC\fIr\fP\fC?\fP
zero or one \fIr\fP's (that is, "an optional \fIr\fP")
.TP
name
the expansion of the "name" definition (see above)
.TP
\fC(\fP\fIr\fP\fC)\fP
an \fIr\fP; parentheses are used to override precedence
(see below)
.TP
\fIrs\fP
an \fIr\fP followed by an \fIs\fP ("concatenation")
.TP
\fIr\fP\fC|\fP\fIs\fP
either an \fIr\fP or an \fIs\fP
.TP
\fIr\fP\fC/\fP\fIs\fP
an \fIr\fP but only if it is followed by an \fIs\fP. The s is not part of
the matched text. This type of \*(rx is called "trailing context".
.TP
\fIr\fP\fC{\fP\fIn\fP\fC}\fP
matches \fIr\fP exactly \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,}\fP
matches \fIr\fP at least \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,\fP\fIm\fP\fC}\fP
matches \fIr\fP at least \fIn\fP but not more than \fIm\fP times.
.LP
The regular expressions listed above are grouped according to
precedence, from highest precedence at the top to lowest at the bottom.
Those grouped together have equal precedence.
.SH "A LARGER EXAMPLE"
.LP
.in +3
.nf
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*)
malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\\n';
}
s->lim += cnt;
}
s->cur = cursor;
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\\000-\\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\\\] ([abfnrtv?'"\\\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\\[\\n\\\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\\[\\n\\\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \\t\\v\\f]+ { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\\t%.*s\\n", t, in.cur - in.tok, in.tok);
printf("%d\\n", t);
*/
}
close(in.fd);
}
.fi
.in -3
.SH FEATURES
.LP
\*(re does not provide a default action:
the generated code assumes that the input
will consist of a sequence of tokens.
Typically this can be dealt with by adding a rule such as the one for
unexpected characters in the example above.
.LP
The user must arrange for a sentinel token to appear at the end of input
(and provide a rule for matching it):
\*(re does not provide an \fC<<EOF>>\fP expression.
If the source is from a null-byte terminated string, a
rule matching a null character will suffice. If the source is from a
file then the approach taken in the example can be used: pad the input with
a newline (or some other character that can't appear within another token);
upon recognizing such a character check to see if it is the sentinel
and act accordingly.
.LP
\*(re does not provide start conditions: use a separate scanner
specification for each start condition (as illustrated in the above example).
.LP
No [^x]. Use difference instead.
.SH BUGS
.LP
Only fixed length trailing context can be handled.
.LP
The maximum value appearing as a parameter \fIn\fP to \fCYYFILL\fP is not
provided to the generated code (this value is needed for constructing
the interface code).
Note that this value is usually relatively small: for
typical programming languages \fIn\fP will be the length of the longest
keyword plus one.
.LP
Difference only works for character sets.
.LP
The \*(re internal algorithms need documentation.
.SH "SEE ALSO"
.LP
flex(1), lex(1).
.P
More information on \fBre2c\fP can be found here:
.PD 0
.P
.B http://sourceforge.net/projects/re2c/
.PD 1
.SH AUTHORS
.PD 0
.P
Peter Bumbulis <peter@csg.uwaterloo.ca>
.P
Brian Young <bayoung@acm.org>
.P
Dan Nuffer <nuffer@users.sourceforge.net>
.P
Marcus Boerger <helly@users.sourceforge.net>
.P
Hartmut Kaiser <hkaiser@users.sourceforge.net>
.P
.PD 1
.SH VERSION INFORMATION
This manpage describes \fBre2c\fP, version 0.9.3.
.fi

601
tools/re2c/re2c.1.in Normal file
View file

@ -0,0 +1,601 @@
./"
./" $Id: re2c.1.in,v 1.9 2004/05/26 13:33:25 nuffer Exp $
./"
.TH RE2C 1 "14 March 1994" "Version @PACKAGE_VERSION@"
.ds re \fBre2c\fP
.ds le \fBlex\fP
.ds rx regular expression
.ds lx \fIl\fP-expression
\"$Log: re2c.1.in,v $
\"Revision 1.9 2004/05/26 13:33:25 nuffer
\"Added description of -o option.
\"
\"Revision 1.8 2004/04/19 22:32:48 helly
\"Update
\"
\"Revision 1.7 2004/04/19 02:13:48 helly
\"Featurerequest #869298 (Add case insensitive string literals)
\"
\"Revision 1.6 2004/04/17 15:49:13 helly
\"Fix example, cur must be set to make the uncommented printf's working
\"
\"Revision 1.5 2004/03/30 01:02:45 helly
\"Update docu
\"
\"Revision 1.4 2004/03/14 14:23:40 helly
\"Update
\"
\"Revision 1.3 2004/03/14 12:54:19 helly
\"Next step of autogen patch
\"
\"Revision 1.2 2004/03/13 20:35:12 helly
\"Updated configure stuff
\"
\"Revision 1.1 2004/01/31 15:44:39 nuffer
\"Applied patch from Marcus Boerger
\"
\"Revision 1.2 1994/04/16 15:50:32 peter
\"Fix bug in simple example.
\"
\"Revision 1.1 1994/04/08 15:39:09 peter
\"Initial revision
\"
.SH NAME
re2c \- convert regular expressions to C/C++
.SH SYNOPSIS
\*(re [\fB-esbvh\fP] [\fB-o output\fP] file\fP
.SH DESCRIPTION
\*(re is a preprocessor that generates C-based recognizers from regular
expressions.
The input to \*(re consists of C/C++ source interleaved with
comments of the form \fC/*!re2c\fP ... \fC*/\fP which contain
scanner specifications.
In the output these comments are replaced with code that, when
executed, will find the next input token and then execute
some user-supplied token-specific code.
For example, given the following code
.in +3
.nf
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
/*!re2c
[0-9]+ {return YYCURSOR;}
[\\000-\\377] {return NULL;}
*/
}
.fi
.in -3
\*(re will generate
.in +3
.nf
/* Generated by re2c on Sat Apr 16 11:40:58 1994 */
#line 1 "simple.re"
#define NULL ((char*) 0)
char *scan(char *p){
char *q;
#define YYCTYPE char
#define YYCURSOR p
#define YYLIMIT p
#define YYMARKER q
#define YYFILL(n)
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/') goto yy4;
if(yych >= ':') goto yy4;
yy2: yych = *++YYCURSOR;
goto yy7;
yy3:
#line 10
{return YYCURSOR;}
yy4: yych = *++YYCURSOR;
yy5:
#line 11
{return NULL;}
yy6: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
yy7: if(yych <= '/') goto yy3;
if(yych <= '9') goto yy6;
goto yy3;
}
#line 12
}
.fi
.in -3
.SH OPTIONS
\*(re provides the following options:
.TP
\fB-e\fP
Cross-compile from an ASCII platform to an EBCDIC one.
.TP
\fB-s\fP
Generate nested \fCif\fPs for some \fCswitch\fPes. Many compilers need this
assist to generate better code.
.TP
\fB-b\fP
Implies \fB-s\fP. Use bit vectors as well in the attempt to coax better
code out of the compiler. Most useful for specifications with more than a
few keywords (e.g. for most programming languages).
.TP
\fB-h\fP
\fB-?\fP
Invoke a short help.
.TP
\fB-v\fP
Show version information.
.TP
\fB-o output\fP
Specify the output file.
.SH "INTERFACE CODE"
Unlike other scanner generators, \*(re does not generate complete scanners:
the user must supply some interface code.
In particular, the user must define the following macros:
.TP
\fCYYCHAR\fP
Type used to hold an input symbol.
Usually \fCchar\fP or \fCunsigned char\fP.
.TP
\fCYYCURSOR\fP
\*(lx of type \fC*YYCHAR\fP that points to the current input symbol.
The generated code advances \fCYYCURSOR\fP as symbols are matched.
On entry, \fCYYCURSOR\fP is assumed to point to the first character of the
current token. On exit, \fCYYCURSOR\fP will point to the first character of
the following token.
.TP
\fCYLIMIT\fP
Expression of type \fC*YYCHAR\fP that marks the end of the buffer
(\fCYLIMIT[-1]\fP is the last character in the buffer).
The generated code repeatedly compares \fCYYCURSOR\fP to \fCYLIMIT\fP
to determine when the buffer needs (re)filling.
.TP
\fCYYMARKER\fP
\*(lx of type \fC*YYCHAR\fP.
The generated code saves backtracking information in \fCYYMARKER\fP.
.TP
\fCYYFILL(\fP\fIn\fP\fC)\fP
The generated code "calls" \fCYYFILL\fP when the buffer needs
(re)filling: at least \fIn\fP additional characters should
be provided. \fCYYFILL\fP should adjust \fCYYCURSOR\fP, \fCYYLIMIT\fP and
\fCYYMARKER\fP as needed. Note that for typical programming languages
\fIn\fP will be the length of the longest keyword plus one.
.SH "SCANNER SPECIFICATIONS"
Each scanner specification consists of a set of \fIrules\fP and name
definitions.
Rules consist of a regular expression along with a block of C/C++ code that
is to be executed when the associated regular expression is matched.
Name definitions are of the form
``\fIname\fP \fC=\fP \fIregular expression\fP\fC;\fP''.
.SH "SUMMARY OF RE2C REGULAR EXPRESSIONS"
.TP
\fC"foo"\fP
the literal string \fCfoo\fP.
ANSI-C escape sequences can be used.
.TP
\fC'foo'\fP
the literal string \fCfoo\fP (characters [a-zA-Z] treated case-insensitive).
ANSI-C escape sequences can be used.
.TP
\fC[xyz]\fP
a "character class"; in this case,
the \*(rx matches either an '\fCx\fP', a '\fCy\fP', or a '\fCz\fP'.
.TP
\fC[abj-oZ]\fP
a "character class" with a range in it;
matches an '\fCa\fP', a '\fCb\fP', any letter from '\fCj\fP' through '\fCo\fP',
or a '\fCZ\fP'.
.TP
\fIr\fP\fC\e\fP\fIs\fP
match any \fIr\fP which isn't an \fIs\fP. \fIr\fP and \fIs\fP must be regular expressions
which can be expressed as character classes.
.TP
\fIr\fP\fC*\fP
zero or more \fIr\fP's, where \fIr\fP is any regular expression
.TP
\fC\fIr\fP\fC+\fP
one or more \fIr\fP's
.TP
\fC\fIr\fP\fC?\fP
zero or one \fIr\fP's (that is, "an optional \fIr\fP")
.TP
name
the expansion of the "name" definition (see above)
.TP
\fC(\fP\fIr\fP\fC)\fP
an \fIr\fP; parentheses are used to override precedence
(see below)
.TP
\fIrs\fP
an \fIr\fP followed by an \fIs\fP ("concatenation")
.TP
\fIr\fP\fC|\fP\fIs\fP
either an \fIr\fP or an \fIs\fP
.TP
\fIr\fP\fC/\fP\fIs\fP
an \fIr\fP but only if it is followed by an \fIs\fP. The s is not part of
the matched text. This type of \*(rx is called "trailing context".
.TP
\fIr\fP\fC{\fP\fIn\fP\fC}\fP
matches \fIr\fP exactly \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,}\fP
matches \fIr\fP at least \fIn\fP times.
.TP
\fIr\fP\fC{\fP\fIn\fP\fC,\fP\fIm\fP\fC}\fP
matches \fIr\fP at least \fIn\fP but not more than \fIm\fP times.
.LP
The regular expressions listed above are grouped according to
precedence, from highest precedence at the top to lowest at the bottom.
Those grouped together have equal precedence.
.SH "A LARGER EXAMPLE"
.LP
.in +3
.nf
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#define ADDEQ 257
#define ANDAND 258
#define ANDEQ 259
#define ARRAY 260
#define ASM 261
#define AUTO 262
#define BREAK 263
#define CASE 264
#define CHAR 265
#define CONST 266
#define CONTINUE 267
#define DECR 268
#define DEFAULT 269
#define DEREF 270
#define DIVEQ 271
#define DO 272
#define DOUBLE 273
#define ELLIPSIS 274
#define ELSE 275
#define ENUM 276
#define EQL 277
#define EXTERN 278
#define FCON 279
#define FLOAT 280
#define FOR 281
#define FUNCTION 282
#define GEQ 283
#define GOTO 284
#define ICON 285
#define ID 286
#define IF 287
#define INCR 288
#define INT 289
#define LEQ 290
#define LONG 291
#define LSHIFT 292
#define LSHIFTEQ 293
#define MODEQ 294
#define MULEQ 295
#define NEQ 296
#define OREQ 297
#define OROR 298
#define POINTER 299
#define REGISTER 300
#define RETURN 301
#define RSHIFT 302
#define RSHIFTEQ 303
#define SCON 304
#define SHORT 305
#define SIGNED 306
#define SIZEOF 307
#define STATIC 308
#define STRUCT 309
#define SUBEQ 310
#define SWITCH 311
#define TYPEDEF 312
#define UNION 313
#define UNSIGNED 314
#define VOID 315
#define VOLATILE 316
#define WHILE 317
#define XOREQ 318
#define EOI 319
typedef unsigned int uint;
typedef unsigned char uchar;
#define BSIZE 8192
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT s->lim
#define YYMARKER s->ptr
#define YYFILL(n) {cursor = fill(s, cursor);}
#define RET(i) {s->cur = cursor; return i;}
typedef struct Scanner {
int fd;
uchar *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint line;
} Scanner;
uchar *fill(Scanner *s, uchar *cursor){
if(!s->eof){
uint cnt = s->tok - s->bot;
if(cnt){
memcpy(s->bot, s->tok, s->lim - s->tok);
s->tok = s->bot;
s->ptr -= cnt;
cursor -= cnt;
s->pos -= cnt;
s->lim -= cnt;
}
if((s->top - s->lim) < BSIZE){
uchar *buf = (uchar*)
malloc(((s->lim - s->bot) + BSIZE)*sizeof(uchar));
memcpy(buf, s->tok, s->lim - s->tok);
s->tok = buf;
s->ptr = &buf[s->ptr - s->bot];
cursor = &buf[cursor - s->bot];
s->pos = &buf[s->pos - s->bot];
s->lim = &buf[s->lim - s->bot];
s->top = &s->lim[BSIZE];
free(s->bot);
s->bot = buf;
}
if((cnt = read(s->fd, (char*) s->lim, BSIZE)) != BSIZE){
s->eof = &s->lim[cnt]; *(s->eof)++ = '\\n';
}
s->lim += cnt;
}
s->cur = cursor;
return cursor;
}
int scan(Scanner *s){
uchar *cursor = s->cur;
std:
s->tok = cursor;
/*!re2c
any = [\\000-\\377];
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
FS = [fFlL];
IS = [uUlL]*;
ESC = [\\\\] ([abfnrtv?'"\\\\] | "x" H+ | O+);
*/
/*!re2c
"/*" { goto comment; }
"auto" { RET(AUTO); }
"break" { RET(BREAK); }
"case" { RET(CASE); }
"char" { RET(CHAR); }
"const" { RET(CONST); }
"continue" { RET(CONTINUE); }
"default" { RET(DEFAULT); }
"do" { RET(DO); }
"double" { RET(DOUBLE); }
"else" { RET(ELSE); }
"enum" { RET(ENUM); }
"extern" { RET(EXTERN); }
"float" { RET(FLOAT); }
"for" { RET(FOR); }
"goto" { RET(GOTO); }
"if" { RET(IF); }
"int" { RET(INT); }
"long" { RET(LONG); }
"register" { RET(REGISTER); }
"return" { RET(RETURN); }
"short" { RET(SHORT); }
"signed" { RET(SIGNED); }
"sizeof" { RET(SIZEOF); }
"static" { RET(STATIC); }
"struct" { RET(STRUCT); }
"switch" { RET(SWITCH); }
"typedef" { RET(TYPEDEF); }
"union" { RET(UNION); }
"unsigned" { RET(UNSIGNED); }
"void" { RET(VOID); }
"volatile" { RET(VOLATILE); }
"while" { RET(WHILE); }
L (L|D)* { RET(ID); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
(['] (ESC|any\\[\\n\\\\'])* ['])
{ RET(ICON); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(FCON); }
(["] (ESC|any\\[\\n\\\\"])* ["])
{ RET(SCON); }
"..." { RET(ELLIPSIS); }
">>=" { RET(RSHIFTEQ); }
"<<=" { RET(LSHIFTEQ); }
"+=" { RET(ADDEQ); }
"-=" { RET(SUBEQ); }
"*=" { RET(MULEQ); }
"/=" { RET(DIVEQ); }
"%=" { RET(MODEQ); }
"&=" { RET(ANDEQ); }
"^=" { RET(XOREQ); }
"|=" { RET(OREQ); }
">>" { RET(RSHIFT); }
"<<" { RET(LSHIFT); }
"++" { RET(INCR); }
"--" { RET(DECR); }
"->" { RET(DEREF); }
"&&" { RET(ANDAND); }
"||" { RET(OROR); }
"<=" { RET(LEQ); }
">=" { RET(GEQ); }
"==" { RET(EQL); }
"!=" { RET(NEQ); }
";" { RET(';'); }
"{" { RET('{'); }
"}" { RET('}'); }
"," { RET(','); }
":" { RET(':'); }
"=" { RET('='); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
"?" { RET('?'); }
[ \\t\\v\\f]+ { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->pos = cursor; s->line++;
goto std;
}
any
{
printf("unexpected character: %c\\n", *s->tok);
goto std;
}
*/
comment:
/*!re2c
"*/" { goto std; }
"\\n"
{
if(cursor == s->eof) RET(EOI);
s->tok = s->pos = cursor; s->line++;
goto comment;
}
any { goto comment; }
*/
}
main(){
Scanner in;
int t;
memset((char*) &in, 0, sizeof(in));
in.fd = 0;
while((t = scan(&in)) != EOI){
/*
printf("%d\\t%.*s\\n", t, in.cur - in.tok, in.tok);
printf("%d\\n", t);
*/
}
close(in.fd);
}
.fi
.in -3
.SH FEATURES
.LP
\*(re does not provide a default action:
the generated code assumes that the input
will consist of a sequence of tokens.
Typically this can be dealt with by adding a rule such as the one for
unexpected characters in the example above.
.LP
The user must arrange for a sentinel token to appear at the end of input
(and provide a rule for matching it):
\*(re does not provide an \fC<<EOF>>\fP expression.
If the source is from a null-byte terminated string, a
rule matching a null character will suffice. If the source is from a
file then the approach taken in the example can be used: pad the input with
a newline (or some other character that can't appear within another token);
upon recognizing such a character check to see if it is the sentinel
and act accordingly.
.LP
\*(re does not provide start conditions: use a separate scanner
specification for each start condition (as illustrated in the above example).
.LP
No [^x]. Use difference instead.
.SH BUGS
.LP
Only fixed length trailing context can be handled.
.LP
The maximum value appearing as a parameter \fIn\fP to \fCYYFILL\fP is not
provided to the generated code (this value is needed for constructing
the interface code).
Note that this value is usually relatively small: for
typical programming languages \fIn\fP will be the length of the longest
keyword plus one.
.LP
Difference only works for character sets.
.LP
The \*(re internal algorithms need documentation.
.SH "SEE ALSO"
.LP
flex(1), lex(1).
.P
More information on \fBre2c\fP can be found here:
.PD 0
.P
.B http://sourceforge.net/projects/re2c/
.PD 1
.SH AUTHORS
.PD 0
.P
Peter Bumbulis <peter@csg.uwaterloo.ca>
.P
Brian Young <bayoung@acm.org>
.P
Dan Nuffer <nuffer@users.sourceforge.net>
.P
Marcus Boerger <helly@users.sourceforge.net>
.P
Hartmut Kaiser <hkaiser@users.sourceforge.net>
.P
.PD 1
.SH VERSION INFORMATION
This manpage describes \fBre2c\fP, version @PACKAGE_VERSION@.
.fi

51
tools/re2c/re2c.spec.in Normal file
View file

@ -0,0 +1,51 @@
Summary: re2c - A tool for generating C-based recognizers from regular expressions
Name: @PACKAGE_NAME@
Version: @PACKAGE_VERSION@
Release: RPM_RELEASE
Group: Development
License: public domain
URL: http://sourceforge.net/projects/re2c/
Source: %{name}-%{version}.tar.gz
BuildRoot: %{_tmppath}/%{name}-%{version}-root
%description
re2c is a great tool for writing fast and flexible lexers. It has
served many people well for many years and it deserves to be
maintained more actively. re2c is on the order of 2-3 times faster
than a flex based scanner, and its input model is much more
flexible.
%prep
%setup -q -n @PACKAGE_NAME@-@PACKAGE_VERSION@
%build
./configure \
--prefix=%{_prefix}
make re2c
#regenerate file scanner.cc
rm -f scanner.cc
./re2c scanner.re > scanner.cc
rm -f re2c scanner.o
make
%install
rm -rf $RPM_BUILD_ROOT
mkdir -p $RPM_BUILD_ROOT%{_bindir}
install -m 0755 re2c $RPM_BUILD_ROOT%{_bindir}
mkdir -p $RPM_BUILD_ROOT%{_mandir}/man1
install -m 0755 re2c.1 $RPM_BUILD_ROOT%{_mandir}/man1
%clean
rm -rf $RPM_BUILD_ROOT
%changelog
* Sun Jan 04 2003 Marcus Boerger <re2c@somabo.de>
- Initial version.
%files
%defattr(-,root,root)
%{_bindir}/re2c
%{_mandir}/man1/re2c.1*
%doc README examples doc/*

199
tools/re2c/re2c.vcproj Normal file
View file

@ -0,0 +1,199 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="7.10"
Name="re2c"
ProjectGUID="{667D2EE7-C357-49E2-9BAB-0A4A45F0F76E}"
Keyword="Win32Proj">
<Platforms>
<Platform
Name="Win32"/>
</Platforms>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="."
IntermediateDirectory="Build"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="1"
OptimizeForProcessor="3"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
StringPooling="TRUE"
ExceptionHandling="TRUE"
RuntimeLibrary="2"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="FALSE"
DebugInformationFormat="0"
CallingConvention="0"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/re2c.exe"
LinkIncremental="1"
GenerateDebugInformation="FALSE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
OptimizeForWindows98="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="."
IntermediateDirectory="Build"
ConfigurationType="1"
CharacterSet="2">
<Tool
Name="VCCLCompilerTool"
Optimization="1"
OptimizeForProcessor="3"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
StringPooling="TRUE"
ExceptionHandling="TRUE"
RuntimeLibrary="2"
UsePrecompiledHeader="0"
WarningLevel="3"
Detect64BitPortabilityProblems="FALSE"
DebugInformationFormat="0"
CallingConvention="0"/>
<Tool
Name="VCCustomBuildTool"/>
<Tool
Name="VCLinkerTool"
OutputFile="$(OutDir)/re2c.exe"
LinkIncremental="1"
GenerateDebugInformation="FALSE"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
OptimizeForWindows98="1"
TargetMachine="1"/>
<Tool
Name="VCMIDLTool"/>
<Tool
Name="VCPostBuildEventTool"/>
<Tool
Name="VCPreBuildEventTool"/>
<Tool
Name="VCPreLinkEventTool"/>
<Tool
Name="VCResourceCompilerTool"/>
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
Name="VCXMLDataGeneratorTool"/>
<Tool
Name="VCWebDeploymentTool"/>
<Tool
Name="VCManagedWrapperGeneratorTool"/>
<Tool
Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
<File
RelativePath=".\actions.cc">
</File>
<File
RelativePath=".\code.cc">
</File>
<File
RelativePath=".\dfa.cc">
</File>
<File
RelativePath=".\main.cc">
</File>
<File
RelativePath=".\mbo_getopt.cc">
</File>
<File
RelativePath=".\parser.cc">
</File>
<File
RelativePath=".\scanner.cc">
</File>
<File
RelativePath=".\substr.cc">
</File>
<File
RelativePath=".\translate.cc">
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
<File
RelativePath=".\basics.h">
</File>
<File
RelativePath=".\dfa.h">
</File>
<File
RelativePath=".\globals.h">
</File>
<File
RelativePath=".\ins.h">
</File>
<File
RelativePath=".\mbo_getopt.h">
</File>
<File
RelativePath=".\parser.h">
</File>
<File
RelativePath=".\re.h">
</File>
<File
RelativePath=".\scanner.h">
</File>
<File
RelativePath=".\substr.h">
</File>
<File
RelativePath=".\token.h">
</File>
<File
RelativePath=".\y.tab.h">
</File>
</Filter>
<Filter
Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

639
tools/re2c/scanner.cc Normal file
View file

@ -0,0 +1,639 @@
/* Generated by re2c 0.9.3 on Wed May 26 17:58:01 2004 */
#line 1 "scanner.re"
/* $Id: scanner.re,v 1.10 2004/05/13 02:58:18 nuffer Exp $ */
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include "scanner.h"
#include "parser.h"
#include "y.tab.h"
extern YYSTYPE yylval;
#ifndef MAX
#define MAX(a,b) (((a)>(b))?(a):(b))
#endif
#define BSIZE 8192
#define YYCTYPE char
#define YYCURSOR cursor
#define YYLIMIT lim
#define YYMARKER ptr
#define YYFILL(n) {cursor = fill(cursor);}
#define RETURN(i) {cur = cursor; return i;}
Scanner::Scanner(std::istream& i) : in(i),
bot(NULL), tok(NULL), ptr(NULL), cur(NULL), pos(NULL), lim(NULL),
top(NULL), eof(NULL), tchar(0), tline(0), cline(1) {
;
}
char *Scanner::fill(char *cursor){
if(!eof){
uint cnt = tok - bot;
if(cnt){
memcpy(bot, tok, lim - tok);
tok = bot;
ptr -= cnt;
cursor -= cnt;
pos -= cnt;
lim -= cnt;
}
if((top - lim) < BSIZE){
char *buf = new char[(lim - bot) + BSIZE];
memcpy(buf, tok, lim - tok);
tok = buf;
ptr = &buf[ptr - bot];
cursor = &buf[cursor - bot];
pos = &buf[pos - bot];
lim = &buf[lim - bot];
top = &lim[BSIZE];
delete [] bot;
bot = buf;
}
if((cnt = in.rdbuf()->sgetn((char*) lim, BSIZE)) != BSIZE){
eof = &lim[cnt]; *eof++ = '\n';
}
lim += cnt;
}
return cursor;
}
#line 72 "scanner.re"
int Scanner::echo(std::ostream &out){
char *cursor = cur;
// Catch EOF
if (eof && cursor == eof)
return 0;
tok = cursor;
echo:
#line 7 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 7) YYFILL(7);
yych = *YYCURSOR;
if(yych == '\n') goto yy4;
if(yych != '/') goto yy6;
goto yy2;
yy2: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych == '*') goto yy7;
goto yy3;
yy3:
#line 91 "scanner.re"
{ goto echo; }
#line 26 "scanner.cc"
yy4: ++YYCURSOR;
goto yy5;
yy5:
#line 87 "scanner.re"
{ if(cursor == eof) RETURN(0);
out.write((const char*)(tok), (const char*)(cursor) - (const char*)(tok));
tok = pos = cursor; cline++;
goto echo; }
#line 35 "scanner.cc"
yy6: yych = *++YYCURSOR;
goto yy3;
yy7: yych = *++YYCURSOR;
if(yych == '!') goto yy9;
goto yy8;
yy8: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy3;
}
yy9: yych = *++YYCURSOR;
if(yych != 'r') goto yy8;
goto yy10;
yy10: yych = *++YYCURSOR;
if(yych != 'e') goto yy8;
goto yy11;
yy11: yych = *++YYCURSOR;
if(yych != '2') goto yy8;
goto yy12;
yy12: yych = *++YYCURSOR;
if(yych != 'c') goto yy8;
goto yy13;
yy13: ++YYCURSOR;
goto yy14;
yy14:
#line 84 "scanner.re"
{ out.write((const char*)(tok), (const char*)(&cursor[-7]) - (const char*)(tok));
tok = cursor;
RETURN(1); }
#line 64 "scanner.cc"
}
#line 92 "scanner.re"
}
int Scanner::scan(){
char *cursor = cur;
uint depth;
scan:
tchar = cursor - pos;
tline = cline;
tok = cursor;
#line 68 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy15;
yy16: ++YYCURSOR;
yy15:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '/'){
if(yych <= '"'){
if(yych <= '\n'){
if(yych <= '\b') goto yy37;
if(yych <= '\t') goto yy33;
goto yy35;
} else {
if(yych == ' ') goto yy33;
if(yych <= '!') goto yy37;
goto yy23;
}
} else {
if(yych <= ')'){
if(yych <= '&') goto yy37;
if(yych <= '\'') goto yy25;
goto yy29;
} else {
if(yych <= '*') goto yy21;
if(yych <= '+') goto yy30;
if(yych <= '.') goto yy37;
goto yy19;
}
}
} else {
if(yych <= '@'){
if(yych <= '<'){
if(yych == ';') goto yy29;
goto yy37;
} else {
if(yych <= '=') goto yy29;
if(yych == '?') goto yy30;
goto yy37;
}
} else {
if(yych <= '`'){
if(yych <= 'Z') goto yy31;
if(yych <= '[') goto yy27;
if(yych <= '\\') goto yy29;
goto yy37;
} else {
if(yych <= 'z') goto yy31;
if(yych <= '{') goto yy17;
if(yych <= '|') goto yy29;
goto yy37;
}
}
}
yy17: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych <= '/') goto yy18;
if(yych <= '9') goto yy63;
goto yy18;
yy18:
#line 105 "scanner.re"
{ depth = 1;
goto code;
}
#line 134 "scanner.cc"
yy19: ++YYCURSOR;
if((yych = *YYCURSOR) == '*') goto yy61;
goto yy20;
yy20:
#line 131 "scanner.re"
{ RETURN(*tok); }
#line 141 "scanner.cc"
yy21: ++YYCURSOR;
if((yych = *YYCURSOR) == '/') goto yy59;
goto yy22;
yy22:
#line 133 "scanner.re"
{ yylval.op = *tok;
RETURN(CLOSE); }
#line 149 "scanner.cc"
yy23: yyaccept = 1;
yych = *(YYMARKER = ++YYCURSOR);
if(yych != '\n') goto yy55;
goto yy24;
yy24:
#line 122 "scanner.re"
{ fatal("unterminated string constant (missing \")"); }
#line 157 "scanner.cc"
yy25: yyaccept = 2;
yych = *(YYMARKER = ++YYCURSOR);
if(yych != '\n') goto yy50;
goto yy26;
yy26:
#line 123 "scanner.re"
{ fatal("unterminated string constant (missing ')"); }
#line 165 "scanner.cc"
yy27: yyaccept = 3;
yych = *(YYMARKER = ++YYCURSOR);
if(yych != '\n') goto yy44;
goto yy28;
yy28:
#line 129 "scanner.re"
{ fatal("unterminated range (missing ])"); }
#line 173 "scanner.cc"
yy29: yych = *++YYCURSOR;
goto yy20;
yy30: yych = *++YYCURSOR;
goto yy22;
yy31: ++YYCURSOR;
yych = *YYCURSOR;
goto yy42;
yy32:
#line 148 "scanner.re"
{ cur = cursor;
yylval.symbol = Symbol::find(token());
return ID; }
#line 185 "scanner.cc"
yy33: ++YYCURSOR;
yych = *YYCURSOR;
goto yy40;
yy34:
#line 152 "scanner.re"
{ goto scan; }
#line 191 "scanner.cc"
yy35: ++YYCURSOR;
goto yy36;
yy36:
#line 154 "scanner.re"
{ if(cursor == eof) RETURN(0);
pos = cursor; cline++;
goto scan;
}
#line 200 "scanner.cc"
yy37: ++YYCURSOR;
goto yy38;
yy38:
#line 159 "scanner.re"
{ std::cerr << "unexpected character: " << *tok << std::endl;
goto scan;
}
#line 208 "scanner.cc"
yy39: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy40;
yy40: if(yych == '\t') goto yy39;
if(yych == ' ') goto yy39;
goto yy34;
yy41: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy42;
yy42: if(yych <= '@'){
if(yych <= '/') goto yy32;
if(yych <= '9') goto yy41;
goto yy32;
} else {
if(yych <= 'Z') goto yy41;
if(yych <= '`') goto yy32;
if(yych <= 'z') goto yy41;
goto yy32;
}
yy43: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy44;
yy44: if(yych <= '['){
if(yych != '\n') goto yy43;
goto yy45;
} else {
if(yych <= '\\') goto yy46;
if(yych <= ']') goto yy47;
goto yy43;
}
yy45: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy18;
case 1: goto yy24;
case 2: goto yy26;
case 3: goto yy28;
}
yy46: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy45;
goto yy43;
yy47: ++YYCURSOR;
goto yy48;
yy48:
#line 125 "scanner.re"
{ cur = cursor;
yylval.regexp = ranToRE(token());
return RANGE; }
#line 265 "scanner.cc"
yy49: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy50;
yy50: if(yych <= '&'){
if(yych == '\n') goto yy45;
goto yy49;
} else {
if(yych <= '\'') goto yy52;
if(yych != '\\') goto yy49;
goto yy51;
}
yy51: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy45;
goto yy49;
yy52: ++YYCURSOR;
goto yy53;
yy53:
#line 118 "scanner.re"
{ cur = cursor;
yylval.regexp = strToCaseInsensitiveRE(token());
return STRING; }
#line 292 "scanner.cc"
yy54: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy55;
yy55: if(yych <= '!'){
if(yych == '\n') goto yy45;
goto yy54;
} else {
if(yych <= '"') goto yy57;
if(yych != '\\') goto yy54;
goto yy56;
}
yy56: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy45;
goto yy54;
yy57: ++YYCURSOR;
goto yy58;
yy58:
#line 114 "scanner.re"
{ cur = cursor;
yylval.regexp = strToRE(token());
return STRING; }
#line 319 "scanner.cc"
yy59: ++YYCURSOR;
goto yy60;
yy60:
#line 111 "scanner.re"
{ tok = cursor;
RETURN(0); }
#line 326 "scanner.cc"
yy61: ++YYCURSOR;
goto yy62;
yy62:
#line 108 "scanner.re"
{ depth = 1;
goto comment; }
#line 333 "scanner.cc"
yy63: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy64;
yy64: if(yych <= '/'){
if(yych == ',') goto yy67;
goto yy45;
} else {
if(yych <= '9') goto yy63;
if(yych != '}') goto yy45;
goto yy65;
}
yy65: ++YYCURSOR;
goto yy66;
yy66:
#line 136 "scanner.re"
{ yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = atoi((char *)tok+1);
RETURN(CLOSESIZE); }
#line 354 "scanner.cc"
yy67: yych = *++YYCURSOR;
if(yych != '}') goto yy71;
goto yy68;
yy68: ++YYCURSOR;
goto yy69;
yy69:
#line 144 "scanner.re"
{ yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = -1;
RETURN(CLOSESIZE); }
#line 365 "scanner.cc"
yy70: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy71;
yy71: if(yych <= '/') goto yy45;
if(yych <= '9') goto yy70;
if(yych != '}') goto yy45;
goto yy72;
yy72: ++YYCURSOR;
goto yy73;
yy73:
#line 140 "scanner.re"
{ yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = MAX(yylval.extop.minsize,atoi(strchr((char *)tok, ',')+1));
RETURN(CLOSESIZE); }
#line 382 "scanner.cc"
}
#line 162 "scanner.re"
code:
#line 386 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy74;
yy75: ++YYCURSOR;
yy74:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= '&'){
if(yych <= '\n'){
if(yych <= '\t') goto yy82;
goto yy80;
} else {
if(yych == '"') goto yy84;
goto yy82;
}
} else {
if(yych <= '{'){
if(yych <= '\'') goto yy85;
if(yych <= 'z') goto yy82;
goto yy78;
} else {
if(yych != '}') goto yy82;
goto yy76;
}
}
yy76: ++YYCURSOR;
goto yy77;
yy77:
#line 166 "scanner.re"
{ if(--depth == 0){
cur = cursor;
yylval.token = new Token(token(), tline);
return CODE;
}
goto code; }
#line 423 "scanner.cc"
yy78: ++YYCURSOR;
goto yy79;
yy79:
#line 172 "scanner.re"
{ ++depth;
goto code; }
#line 430 "scanner.cc"
yy80: ++YYCURSOR;
goto yy81;
yy81:
#line 174 "scanner.re"
{ if(cursor == eof) fatal("missing '}'");
pos = cursor; cline++;
goto code;
}
#line 439 "scanner.cc"
yy82: ++YYCURSOR;
goto yy83;
yy83:
#line 178 "scanner.re"
{ goto code; }
#line 445 "scanner.cc"
yy84: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych == '\n') goto yy83;
goto yy91;
yy85: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
if(yych == '\n') goto yy83;
goto yy87;
yy86: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy87;
yy87: if(yych <= '&'){
if(yych != '\n') goto yy86;
goto yy88;
} else {
if(yych <= '\'') goto yy82;
if(yych == '\\') goto yy89;
goto yy86;
}
yy88: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy83;
}
yy89: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy88;
goto yy86;
yy90: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy91;
yy91: if(yych <= '!'){
if(yych == '\n') goto yy88;
goto yy90;
} else {
if(yych <= '"') goto yy82;
if(yych != '\\') goto yy90;
goto yy92;
}
yy92: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
if(yych == '\n') goto yy88;
goto yy90;
}
#line 179 "scanner.re"
comment:
#line 499 "scanner.cc"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy93;
yy94: ++YYCURSOR;
yy93:
if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
yych = *YYCURSOR;
if(yych <= ')'){
if(yych == '\n') goto yy98;
goto yy100;
} else {
if(yych <= '*') goto yy95;
if(yych == '/') goto yy97;
goto yy100;
}
yy95: ++YYCURSOR;
if((yych = *YYCURSOR) == '/') goto yy103;
goto yy96;
yy96:
#line 193 "scanner.re"
{ goto comment; }
#line 522 "scanner.cc"
yy97: yych = *++YYCURSOR;
if(yych == '*') goto yy101;
goto yy96;
yy98: ++YYCURSOR;
goto yy99;
yy99:
#line 189 "scanner.re"
{ if(cursor == eof) RETURN(0);
tok = pos = cursor; cline++;
goto comment;
}
#line 534 "scanner.cc"
yy100: yych = *++YYCURSOR;
goto yy96;
yy101: ++YYCURSOR;
goto yy102;
yy102:
#line 187 "scanner.re"
{ ++depth;
goto comment; }
#line 543 "scanner.cc"
yy103: ++YYCURSOR;
goto yy104;
yy104:
#line 183 "scanner.re"
{ if(--depth == 0)
goto scan;
else
goto comment; }
#line 552 "scanner.cc"
}
#line 194 "scanner.re"
}
void Scanner::fatal(char *msg){
std::cerr << "line " << tline << ", column " << (tchar + 1) << ": "
<< msg << std::endl;
exit(1);
}

34
tools/re2c/scanner.h Normal file
View file

@ -0,0 +1,34 @@
/* $Id: scanner.h,v 1.5 2004/05/13 02:58:18 nuffer Exp $ */
#ifndef _scanner_h
#define _scanner_h
#include <iosfwd>
#include "token.h"
class Scanner {
private:
std::istream& in;
char *bot, *tok, *ptr, *cur, *pos, *lim, *top, *eof;
uint tchar, tline, cline;
private:
char *fill(char*);
Scanner(const Scanner&); //unimplemented
Scanner& operator=(const Scanner&); //unimplemented
public:
Scanner(std::istream&);
int echo(std::ostream&);
int scan();
void fatal(char*);
SubStr token();
uint line();
};
inline SubStr Scanner::token(){
return SubStr(tok, cur - tok);
}
inline uint Scanner::line(){
return cline;
}
#endif

201
tools/re2c/scanner.re Normal file
View file

@ -0,0 +1,201 @@
/* $Id: scanner.re,v 1.10 2004/05/13 02:58:18 nuffer Exp $ */
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include "scanner.h"
#include "parser.h"
#include "y.tab.h"
extern YYSTYPE yylval;
#ifndef MAX
#define MAX(a,b) (((a)>(b))?(a):(b))
#endif
#define BSIZE 8192
#define YYCTYPE char
#define YYCURSOR cursor
#define YYLIMIT lim
#define YYMARKER ptr
#define YYFILL(n) {cursor = fill(cursor);}
#define RETURN(i) {cur = cursor; return i;}
Scanner::Scanner(std::istream& i) : in(i),
bot(NULL), tok(NULL), ptr(NULL), cur(NULL), pos(NULL), lim(NULL),
top(NULL), eof(NULL), tchar(0), tline(0), cline(1) {
;
}
char *Scanner::fill(char *cursor){
if(!eof){
uint cnt = tok - bot;
if(cnt){
memcpy(bot, tok, lim - tok);
tok = bot;
ptr -= cnt;
cursor -= cnt;
pos -= cnt;
lim -= cnt;
}
if((top - lim) < BSIZE){
char *buf = new char[(lim - bot) + BSIZE];
memcpy(buf, tok, lim - tok);
tok = buf;
ptr = &buf[ptr - bot];
cursor = &buf[cursor - bot];
pos = &buf[pos - bot];
lim = &buf[lim - bot];
top = &lim[BSIZE];
delete [] bot;
bot = buf;
}
if((cnt = in.rdbuf()->sgetn((char*) lim, BSIZE)) != BSIZE){
eof = &lim[cnt]; *eof++ = '\n';
}
lim += cnt;
}
return cursor;
}
/*!re2c
any = [\000-\377];
dot = any \ [\n];
esc = dot \ [\\];
cstring = "[" ((esc \ [\]]) | "\\" dot)* "]" ;
dstring = "\"" ((esc \ ["] ) | "\\" dot)* "\"";
sstring = "'" ((esc \ ['] ) | "\\" dot)* "'" ;
letter = [a-zA-Z];
digit = [0-9];
*/
int Scanner::echo(std::ostream &out){
char *cursor = cur;
// Catch EOF
if (eof && cursor == eof)
return 0;
tok = cursor;
echo:
/*!re2c
"/*!re2c" { out.write((const char*)(tok), (const char*)(&cursor[-7]) - (const char*)(tok));
tok = cursor;
RETURN(1); }
"\n" { if(cursor == eof) RETURN(0);
out.write((const char*)(tok), (const char*)(cursor) - (const char*)(tok));
tok = pos = cursor; cline++;
goto echo; }
any { goto echo; }
*/
}
int Scanner::scan(){
char *cursor = cur;
uint depth;
scan:
tchar = cursor - pos;
tline = cline;
tok = cursor;
/*!re2c
"{" { depth = 1;
goto code;
}
"/*" { depth = 1;
goto comment; }
"*/" { tok = cursor;
RETURN(0); }
dstring { cur = cursor;
yylval.regexp = strToRE(token());
return STRING; }
sstring { cur = cursor;
yylval.regexp = strToCaseInsensitiveRE(token());
return STRING; }
"\"" { fatal("unterminated string constant (missing \")"); }
"'" { fatal("unterminated string constant (missing ')"); }
cstring { cur = cursor;
yylval.regexp = ranToRE(token());
return RANGE; }
"[" { fatal("unterminated range (missing ])"); }
[()|=;/\\] { RETURN(*tok); }
[*+?] { yylval.op = *tok;
RETURN(CLOSE); }
"{" [0-9]+ "}" { yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = atoi((char *)tok+1);
RETURN(CLOSESIZE); }
"{" [0-9]+ "," [0-9]+ "}" { yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = MAX(yylval.extop.minsize,atoi(strchr((char *)tok, ',')+1));
RETURN(CLOSESIZE); }
"{" [0-9]+ ",}" { yylval.extop.minsize = atoi((char *)tok+1);
yylval.extop.maxsize = -1;
RETURN(CLOSESIZE); }
letter (letter|digit)* { cur = cursor;
yylval.symbol = Symbol::find(token());
return ID; }
[ \t]+ { goto scan; }
"\n" { if(cursor == eof) RETURN(0);
pos = cursor; cline++;
goto scan;
}
any { std::cerr << "unexpected character: " << *tok << std::endl;
goto scan;
}
*/
code:
/*!re2c
"}" { if(--depth == 0){
cur = cursor;
yylval.token = new Token(token(), tline);
return CODE;
}
goto code; }
"{" { ++depth;
goto code; }
"\n" { if(cursor == eof) fatal("missing '}'");
pos = cursor; cline++;
goto code;
}
dstring | sstring | any { goto code; }
*/
comment:
/*!re2c
"*/" { if(--depth == 0)
goto scan;
else
goto comment; }
"/*" { ++depth;
goto comment; }
"\n" { if(cursor == eof) RETURN(0);
tok = pos = cursor; cline++;
goto comment;
}
any { goto comment; }
*/
}
void Scanner::fatal(char *msg){
std::cerr << "line " << tline << ", column " << (tchar + 1) << ": "
<< msg << std::endl;
exit(1);
}

0
tools/re2c/stamp-h.in Normal file
View file

37
tools/re2c/substr.cc Normal file
View file

@ -0,0 +1,37 @@
/* $Id: substr.cc,v 1.4 2004/05/13 02:58:18 nuffer Exp $ */
#include <string.h>
#include "substr.h"
#include "globals.h"
void SubStr::out(std::ostream& o) const {
o.write(str, len);
for (size_t i = 0; i < (size_t)len; ++i)
{
if (str[i] == '\n')
++oline;
}
}
bool operator==(const SubStr &s1, const SubStr &s2){
return (bool) (s1.len == s2.len && memcmp(s1.str, s2.str, s1.len) == 0);
}
Str::Str(const SubStr& s) : SubStr(new char[s.len], s.len) {
memcpy(str, s.str, s.len);
}
Str::Str(Str& s) : SubStr(s.str, s.len) {
s.str = NULL;
s.len = 0;
}
Str::Str() : SubStr((char*) NULL, 0) {
;
}
Str::~Str() {
delete str;
str = (char*)-1;
len = (uint)-1;
}

46
tools/re2c/substr.h Normal file
View file

@ -0,0 +1,46 @@
/* $Id: substr.h,v 1.3 2004/05/13 02:58:18 nuffer Exp $ */
#ifndef _substr_h
#define _substr_h
#include <iostream>
#include "basics.h"
class SubStr {
public:
char *str;
uint len;
public:
friend bool operator==(const SubStr &, const SubStr &);
SubStr(uchar*, uint);
SubStr(char*, uint);
SubStr(const SubStr&);
void out(std::ostream&) const;
};
class Str: public SubStr {
public:
Str(const SubStr&);
Str(Str&);
Str();
~Str();
};
inline std::ostream& operator<<(std::ostream& o, const SubStr &s){
s.out(o);
return o;
}
inline std::ostream& operator<<(std::ostream& o, const SubStr* s){
return o << *s;
}
inline SubStr::SubStr(uchar *s, uint l)
: str((char*) s), len(l) { }
inline SubStr::SubStr(char *s, uint l)
: str(s), len(l) { }
inline SubStr::SubStr(const SubStr &s)
: str(s.str), len(s.len) { }
#endif

162
tools/re2c/test.c Normal file
View file

@ -0,0 +1,162 @@
/* Generated by re2c 0.9.3 on Mon Feb 14 13:46:36 2005 */
#line 1 "test.re"
#line 6 "test.c"
{
YYCTYPE yych;
unsigned int yyaccept;
goto yy0;
yy1: ++YYCURSOR;
yy0:
if((YYLIMIT - YYCURSOR) < 4) YYFILL(4);
yych = *YYCURSOR;
switch(yych){
case '.': goto yy4;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy2;
default: goto yy6;
}
yy2: yyaccept = 0;
yych = *(YYMARKER = ++YYCURSOR);
goto yy16;
yy3:
#line 4 "test.re"
{ return true; }
#line 35 "test.c"
yy4: ++YYCURSOR;
switch((yych = *YYCURSOR)) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy7;
default: goto yy5;
}
yy5:
#line 5 "test.re"
{ return false; }
#line 53 "test.c"
yy6: yych = *++YYCURSOR;
goto yy5;
yy7: yyaccept = 0;
YYMARKER = ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy8;
yy8: switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy7;
case 'D':
case 'E': case 'd':
case 'e': goto yy9;
default: goto yy3;
}
yy9: yych = *++YYCURSOR;
switch(yych){
case '+': case '-': goto yy11;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy12;
default: goto yy10;
}
yy10: YYCURSOR = YYMARKER;
switch(yyaccept){
case 0: goto yy3;
}
yy11: yych = *++YYCURSOR;
switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy12;
default: goto yy10;
}
yy12: ++YYCURSOR;
if(YYLIMIT == YYCURSOR) YYFILL(1);
yych = *YYCURSOR;
goto yy13;
yy13: switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy12;
default: goto yy3;
}
yy14: yych = *++YYCURSOR;
switch(yych){
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy7;
default: goto yy10;
}
yy15: yyaccept = 0;
YYMARKER = ++YYCURSOR;
if((YYLIMIT - YYCURSOR) < 3) YYFILL(3);
yych = *YYCURSOR;
goto yy16;
yy16: switch(yych){
case '.': goto yy14;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': goto yy15;
case 'D':
case 'E': case 'd':
case 'e': goto yy9;
default: goto yy3;
}
}
#line 6 "test.re"

6
tools/re2c/test.re Normal file
View file

@ -0,0 +1,6 @@
/*!re2c
digits = [0-9];
(digits+ | digits* "." digits+) ([dDeE] [+-]? digits+)? { return true; }
[\000-\377] { return false; }
*/

19
tools/re2c/token.h Normal file
View file

@ -0,0 +1,19 @@
/* $Id: token.h,v 1.2 2004/01/31 15:44:39 nuffer Exp $ */
#ifndef _token_h
#define _token_h
#include "substr.h"
class Token {
public:
Str text;
uint line;
public:
Token(SubStr, uint);
};
inline Token::Token(SubStr t, uint l) : text(t), line(l) {
;
}
#endif

62
tools/re2c/translate.cc Normal file
View file

@ -0,0 +1,62 @@
/* $Id: translate.cc,v 1.2 2004/01/31 15:44:39 nuffer Exp $ */
#include "globals.h"
uchar asc2asc[256] = {
0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,
0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,
0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,
0x40,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,
0x50,0x51,0x52,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,
0x60,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,
0x80,0x81,0x82,0x83,0x84,0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,
0xa0,0xa1,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,
0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,
0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,
0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,
0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,
0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff
};
uchar *xlat = asc2asc;
uchar *talx = asc2asc;
uchar asc2ebc[256] = { /* Based on ISO 8859/1 and Code Page 37 */
0x00,0x01,0x02,0x03,0x37,0x2d,0x2e,0x2f,0x16,0x05,0x25,0x0b,0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13,0x3c,0x3d,0x32,0x26,0x18,0x19,0x3f,0x27,0x1c,0x1d,0x1e,0x1f,
0x40,0x5a,0x7f,0x7b,0x5b,0x6c,0x50,0x7d,0x4d,0x5d,0x5c,0x4e,0x6b,0x60,0x4b,0x61,
0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0x7a,0x5e,0x4c,0x7e,0x6e,0x6f,
0x7c,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,
0xd7,0xd8,0xd9,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xba,0xe0,0xbb,0xb0,0x6d,
0x79,0x81,0x82,0x83,0x84,0x85,0x86,0x87,0x88,0x89,0x91,0x92,0x93,0x94,0x95,0x96,
0x97,0x98,0x99,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xc0,0x4f,0xd0,0xa1,0x07,
0x20,0x21,0x22,0x23,0x24,0x15,0x06,0x17,0x28,0x29,0x2a,0x2b,0x2c,0x09,0x0a,0x1b,
0x30,0x31,0x1a,0x33,0x34,0x35,0x36,0x08,0x38,0x39,0x3a,0x3b,0x04,0x14,0x3e,0xff,
0x41,0xaa,0x4a,0xb1,0x9f,0xb2,0x6a,0xb5,0xbd,0xb4,0x9a,0x8a,0x5f,0xca,0xaf,0xbc,
0x90,0x8f,0xea,0xfa,0xbe,0xa0,0xb6,0xb3,0x9d,0xda,0x9b,0x8b,0xb7,0xb8,0xb9,0xab,
0x64,0x65,0x62,0x66,0x63,0x67,0x9e,0x68,0x74,0x71,0x72,0x73,0x78,0x75,0x76,0x77,
0xac,0x69,0xed,0xee,0xeb,0xef,0xec,0xbf,0x80,0xfd,0xfe,0xfb,0xfc,0xad,0x8e,0x59,
0x44,0x45,0x42,0x46,0x43,0x47,0x9c,0x48,0x54,0x51,0x52,0x53,0x58,0x55,0x56,0x57,
0x8c,0x49,0xcd,0xce,0xcb,0xcf,0xcc,0xe1,0x70,0xdd,0xde,0xdb,0xdc,0x8d,0xae,0xdf
};
uchar ebc2asc[256] = { /* Based on ISO 8859/1 and Code Page 37 */
0x00,0x01,0x02,0x03,0x9c,0x09,0x86,0x7f,0x97,0x8d,0x8e,0x0b,0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13,0x9d,0x85,0x08,0x87,0x18,0x19,0x92,0x8f,0x1c,0x1d,0x1e,0x1f,
0x80,0x81,0x82,0x83,0x84,0x0a,0x17,0x1b,0x88,0x89,0x8a,0x8b,0x8c,0x05,0x06,0x07,
0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9a,0x9b,0x14,0x15,0x9e,0x1a,
0x20,0xa0,0xe2,0xe4,0xe0,0xe1,0xe3,0xe5,0xe7,0xf1,0xa2,0x2e,0x3c,0x28,0x2b,0x7c,
0x26,0xe9,0xea,0xeb,0xe8,0xed,0xee,0xef,0xec,0xdf,0x21,0x24,0x2a,0x29,0x3b,0xac,
0x2d,0x2f,0xc2,0xc4,0xc0,0xc1,0xc3,0xc5,0xc7,0xd1,0xa6,0x2c,0x25,0x5f,0x3e,0x3f,
0xf8,0xc9,0xca,0xcb,0xc8,0xcd,0xce,0xcf,0xcc,0x60,0x3a,0x23,0x40,0x27,0x3d,0x22,
0xd8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xab,0xbb,0xf0,0xfd,0xde,0xb1,
0xb0,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0xaa,0xba,0xe6,0xb8,0xc6,0xa4,
0xb5,0x7e,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0xa1,0xbf,0xd0,0xdd,0xfe,0xae,
0x5e,0xa3,0xa5,0xb7,0xa9,0xa7,0xb6,0xbc,0xbd,0xbe,0x5b,0x5d,0xaf,0xa8,0xb4,0xd7,
0x7b,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xad,0xf4,0xf6,0xf2,0xf3,0xf5,
0x7d,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0xb9,0xfb,0xfc,0xf9,0xfa,0xff,
0x5c,0xf7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5a,0xb2,0xd4,0xd6,0xd2,0xd3,0xd5,
0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xb3,0xdb,0xdc,0xd9,0xda,0x9f
};

69
tools/re2c/y.tab.h Normal file
View file

@ -0,0 +1,69 @@
/* A Bison parser, made by GNU Bison 1.875. */
/* Skeleton parser for Yacc-like parsing with Bison,
Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002 Free Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA. */
/* As a special exception, when this file is copied by Bison into a
Bison output file, you may use that output file without restriction.
This special exception was added by the Free Software Foundation
in version 1.24 of Bison. */
/* Tokens. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
/* Put the tokens into the symbol table, so that GDB and other debuggers
know about them. */
enum yytokentype {
CLOSESIZE = 258,
CLOSE = 259,
ID = 260,
CODE = 261,
RANGE = 262,
STRING = 263
};
#endif
#define CLOSESIZE 258
#define CLOSE 259
#define ID 260
#define CODE 261
#define RANGE 262
#define STRING 263
#if ! defined (YYSTYPE) && ! defined (YYSTYPE_IS_DECLARED)
#line 53 "parser.y"
typedef union YYSTYPE {
Symbol *symbol;
RegExp *regexp;
Token *token;
char op;
ExtOp extop;
} YYSTYPE;
/* Line 1240 of yacc.c. */
#line 60 "y.tab.h"
# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
# define YYSTYPE_IS_TRIVIAL 1
#endif
extern YYSTYPE yylval;