diff options
Diffstat (limited to 'usr.bin/lex')
| -rw-r--r-- | usr.bin/lex/COPYING | 38 | ||||
| -rw-r--r-- | usr.bin/lex/FlexLexer.h | 186 | ||||
| -rw-r--r-- | usr.bin/lex/Makefile | 50 | ||||
| -rw-r--r-- | usr.bin/lex/NEWS | 1235 | ||||
| -rw-r--r-- | usr.bin/lex/README | 60 | ||||
| -rw-r--r-- | usr.bin/lex/ccl.c | 151 | ||||
| -rw-r--r-- | usr.bin/lex/config.h | 26 | ||||
| -rw-r--r-- | usr.bin/lex/dfa.c | 1097 | ||||
| -rw-r--r-- | usr.bin/lex/ecs.c | 227 | ||||
| -rw-r--r-- | usr.bin/lex/flex.skl | 1574 | ||||
| -rw-r--r-- | usr.bin/lex/flexdef.h | 1049 | ||||
| -rw-r--r-- | usr.bin/lex/gen.c | 1629 | ||||
| -rw-r--r-- | usr.bin/lex/initscan.c | 3728 | ||||
| -rw-r--r-- | usr.bin/lex/lex.1 | 4068 | ||||
| -rw-r--r-- | usr.bin/lex/lib/Makefile | 22 | ||||
| -rw-r--r-- | usr.bin/lex/lib/libmain.c | 16 | ||||
| -rw-r--r-- | usr.bin/lex/lib/libyywrap.c | 9 | ||||
| -rw-r--r-- | usr.bin/lex/main.c | 1179 | ||||
| -rw-r--r-- | usr.bin/lex/misc.c | 888 | ||||
| -rwxr-xr-x | usr.bin/lex/mkskel.sh | 16 | ||||
| -rw-r--r-- | usr.bin/lex/nfa.c | 711 | ||||
| -rw-r--r-- | usr.bin/lex/parse.y | 914 | ||||
| -rw-r--r-- | usr.bin/lex/scan.l | 711 | ||||
| -rw-r--r-- | usr.bin/lex/sym.c | 264 | ||||
| -rw-r--r-- | usr.bin/lex/tblcmp.c | 889 | ||||
| -rw-r--r-- | usr.bin/lex/version.h | 3 | ||||
| -rw-r--r-- | usr.bin/lex/yylex.c | 218 |
27 files changed, 20958 insertions, 0 deletions
diff --git a/usr.bin/lex/COPYING b/usr.bin/lex/COPYING new file mode 100644 index 0000000..dcb775e --- /dev/null +++ b/usr.bin/lex/COPYING @@ -0,0 +1,38 @@ +Flex carries the copyright used for BSD software, slightly modified +because it originated at the Lawrence Berkeley (not Livermore!) Laboratory, +which operates under a contract with the Department of Energy: + + Copyright (c) 1990 The Regents of the University of California. + All rights reserved. + + This code is derived from software contributed to Berkeley by + Vern Paxson. + + The United States Government has rights in this work pursuant + to contract no. DE-AC03-76SF00098 between the United States + Department of Energy and the University of California. + + Redistribution and use in source and binary forms are permitted + provided that: (1) source distributions retain this entire + copyright notice and comment, and (2) distributions including + binaries display the following acknowledgement: ``This product + includes software developed by the University of California, + Berkeley and its contributors'' in the documentation or other + materials provided with the distribution and in all advertising + materials mentioning features or use of this software. Neither the + name of the University nor the names of its contributors may be + used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. + +This basically says "do whatever you please with this software except +remove this notice or take advantage of the University's (or the flex +authors') name". + +Note that the "flex.skl" scanner skeleton carries no copyright notice. +You are free to do whatever you please with scanners generated using flex; +for them, you are not even bound by the above copyright. diff --git a/usr.bin/lex/FlexLexer.h b/usr.bin/lex/FlexLexer.h new file mode 100644 index 0000000..12e8ec3 --- /dev/null +++ b/usr.bin/lex/FlexLexer.h @@ -0,0 +1,186 @@ +// $Header: /home/daffy/u0/vern/flex/RCS/FlexLexer.h,v 1.19 96/05/25 20:43:02 vern Exp $ +// $FreeBSD$ + +// FlexLexer.h -- define interfaces for lexical analyzer classes generated +// by flex + +// Copyright (c) 1993 The Regents of the University of California. +// All rights reserved. +// +// This code is derived from software contributed to Berkeley by +// Kent Williams and Tom Epperly. +// +// Redistribution and use in source and binary forms are permitted provided +// that: (1) source distributions retain this entire copyright notice and +// comment, and (2) distributions including binaries display the following +// acknowledgement: ``This product includes software developed by the +// University of California, Berkeley and its contributors'' in the +// documentation or other materials provided with the distribution and in +// all advertising materials mentioning features or use of this software. +// Neither the name of the University nor the names of its contributors may +// be used to endorse or promote products derived from this software without +// specific prior written permission. +// THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +// This file defines FlexLexer, an abstract class which specifies the +// external interface provided to flex C++ lexer objects, and yyFlexLexer, +// which defines a particular lexer class. +// +// If you want to create multiple lexer classes, you use the -P flag +// to rename each yyFlexLexer to some other xxFlexLexer. You then +// include <FlexLexer.h> in your other sources once per lexer class: +// +// #undef yyFlexLexer +// #define yyFlexLexer xxFlexLexer +// #include <FlexLexer.h> +// +// #undef yyFlexLexer +// #define yyFlexLexer zzFlexLexer +// #include <FlexLexer.h> +// ... + +#ifndef __FLEX_LEXER_H +// Never included before - need to define base class. +#define __FLEX_LEXER_H +#include <iostream> + +extern "C++" { + +struct yy_buffer_state; +typedef int yy_state_type; + +class FlexLexer { +public: + virtual ~FlexLexer() { } + + const char* YYText() { return yytext; } + int YYLeng() { return yyleng; } + + virtual void + yy_switch_to_buffer( struct yy_buffer_state* new_buffer ) = 0; + virtual struct yy_buffer_state* + yy_create_buffer( std::istream* s, int size ) = 0; + virtual void yy_delete_buffer( struct yy_buffer_state* b ) = 0; + virtual void yyrestart( std::istream* s ) = 0; + + virtual int yylex() = 0; + + // Call yylex with new input/output sources. + int yylex( std::istream* new_in, std::ostream* new_out = 0 ) + { + switch_streams( new_in, new_out ); + return yylex(); + } + + // Switch to new input/output streams. A nil stream pointer + // indicates "keep the current one". + virtual void switch_streams( std::istream* new_in = 0, + std::ostream* new_out = 0 ) = 0; + + int lineno() const { return yylineno; } + + int debug() const { return yy_flex_debug; } + void set_debug( int flag ) { yy_flex_debug = flag; } + +protected: + char* yytext; + int yyleng; + int yylineno; // only maintained if you use %option yylineno + int yy_flex_debug; // only has effect with -d or "%option debug" +}; + +} +#endif + +#if defined(yyFlexLexer) || ! defined(yyFlexLexerOnce) +// Either this is the first time through (yyFlexLexerOnce not defined), +// or this is a repeated include to define a different flavor of +// yyFlexLexer, as discussed in the flex man page. +#define yyFlexLexerOnce + +class yyFlexLexer : public FlexLexer { +public: + // arg_yyin and arg_yyout default to the cin and cout, but we + // only make that assignment when initializing in yylex(). + yyFlexLexer( std::istream* arg_yyin = 0, std::ostream* arg_yyout = 0 ); + + virtual ~yyFlexLexer(); + + void yy_switch_to_buffer( struct yy_buffer_state* new_buffer ); + struct yy_buffer_state* yy_create_buffer( std::istream* s, int size ); + void yy_delete_buffer( struct yy_buffer_state* b ); + void yyrestart( std::istream* s ); + + virtual int yylex(); + virtual void switch_streams( std::istream* new_in, std::ostream* new_out ); + +protected: + virtual int LexerInput( char* buf, int max_size ); + virtual void LexerOutput( const char* buf, int size ); + virtual void LexerError( const char* msg ); + + void yyunput( int c, char* buf_ptr ); + int yyinput(); + + void yy_load_buffer_state(); + void yy_init_buffer( struct yy_buffer_state* b, std::istream* s ); + void yy_flush_buffer( struct yy_buffer_state* b ); + + int yy_start_stack_ptr; + int yy_start_stack_depth; + int* yy_start_stack; + + void yy_push_state( int new_state ); + void yy_pop_state(); + int yy_top_state(); + + yy_state_type yy_get_previous_state(); + yy_state_type yy_try_NUL_trans( yy_state_type current_state ); + int yy_get_next_buffer(); + + std::istream* yyin; // input source for default LexerInput + std::ostream* yyout; // output sink for default LexerOutput + + struct yy_buffer_state* yy_current_buffer; + + // yy_hold_char holds the character lost when yytext is formed. + char yy_hold_char; + + // Number of characters read into yy_ch_buf. + int yy_n_chars; + + // Points to current character in buffer. + char* yy_c_buf_p; + + int yy_init; // whether we need to initialize + int yy_start; // start state number + + // Flag which is used to allow yywrap()'s to do buffer switches + // instead of setting up a fresh yyin. A bit of a hack ... + int yy_did_buffer_switch_on_eof; + + // The following are not always needed, but may be depending + // on use of certain flex features (like REJECT or yymore()). + + yy_state_type yy_last_accepting_state; + char* yy_last_accepting_cpos; + + yy_state_type* yy_state_buf; + yy_state_type* yy_state_ptr; + + char* yy_full_match; + int* yy_full_state; + int yy_full_lp; + + int yy_lp; + int yy_looking_for_trail_begin; + + int yy_more_flag; + int yy_more_len; + int yy_more_offset; + int yy_prev_more_offset; +}; + +#endif diff --git a/usr.bin/lex/Makefile b/usr.bin/lex/Makefile new file mode 100644 index 0000000..947eba1 --- /dev/null +++ b/usr.bin/lex/Makefile @@ -0,0 +1,50 @@ +# $FreeBSD$ +# +# By default, flex will be configured to generate 8-bit scanners only if the +# -8 flag is given. If you want it to always generate 8-bit scanners, add +# "-DDEFAULT_CSIZE=256" to CFLAGS. Note that doing so will double the size +# of all uncompressed scanners. +# +# Bootstrapping of lex is handled automatically. +# Also note that flex.skel no longer gets installed. +# + +PROG= lex +LINKS+= ${BINDIR}/lex ${BINDIR}/lex++ +LINKS+= ${BINDIR}/lex ${BINDIR}/flex +LINKS+= ${BINDIR}/lex ${BINDIR}/flex++ + +SRCS= scan.c ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.y \ + skel.c sym.c tblcmp.c yylex.c +LFLAGS+= -is +CFLAGS+= -I. -I${.CURDIR} +INCS= FlexLexer.h +INCSDIR= ${INCLUDEDIR} +MLINKS+= lex.1 flex.1 +MLINKS+= lex.1 flex++.1 +MLINKS+= lex.1 lex++.1 + +WARNS?= 2 + +CLEANFILES= scan.c skel.c + +SUBDIR= lib + +skel.c: mkskel.sh flex.skl + sh ${.CURDIR}/mkskel.sh ${.CURDIR}/flex.skl > skel.c + +bootstrap: initscan.c + @cmp -s ${.CURDIR}/initscan.c scan.c || { \ + echo "Bootstrapping flex" ; \ + rm -f scan.c ; \ + cp -f ${.CURDIR}/initscan.c scan.c ; \ + } + +test: check +check: $(PROG) + ./$(PROG) $(LFLAGS) -t $(COMPRESSION) $(.CURDIR)/scan.l \ + | sed s,\"$(.CURDIR)/scan.l",\"scan.l", \ + | diff -I '\$$FreeBS[D]:.*\$$' $(.CURDIR)/initscan.c - + @echo "Check successful" + +.include <bsd.prog.mk> diff --git a/usr.bin/lex/NEWS b/usr.bin/lex/NEWS new file mode 100644 index 0000000..e632885 --- /dev/null +++ b/usr.bin/lex/NEWS @@ -0,0 +1,1235 @@ +$FreeBSD$ + +Changes between release 2.5.4 (11Sep96) and release 2.5.3: + + - Fixed a bug introduced in 2.5.3 that blew it when a call + to input() occurred at the end of an input file. + + - Fixed scanner skeleton so the example in the man page of + scanning strings using exclusive start conditions works. + + - Minor Makefile tweaks. + + +Changes between release 2.5.3 (29May96) and release 2.5.2: + + - Some serious bugs in yymore() have been fixed. In particular, + when using AT&T-lex-compatibility or %array, you can intermix + calls to input(), unput(), and yymore(). (This still doesn't + work for %pointer, and isn't likely to in the future.) + + - A bug in handling NUL's in the input stream of scanners using + REJECT has been fixed. + + - The default main() in libfl.a now repeatedly calls yylex() until + it returns 0, rather than just calling it once. + + - Minor tweak for Windows NT Makefile, MISC/NT/Makefile. + + +Changes between release 2.5.2 (25Apr95) and release 2.5.1: + + - The --prefix configuration option now works. + + - A bug that completely broke the "-Cf" table compression + option has been fixed. + + - A major headache involving "const" declarators and Solaris + systems has been fixed. + + - An octal escape sequence in a flex regular expression must + now contain only the digits 0-7. + + - You can now use "--" on the flex command line to mark the + end of flex options. + + - You can now specify the filename '-' as a synonym for stdin. + + - By default, the scanners generated by flex no longer + statically initialize yyin and yyout to stdin and stdout. + This change is necessary because in some ANSI environments, + stdin and stdout are not compile-time constant. You can + force the initialization using "%option stdinit" in the first + section of your flex input. + + - "%option nounput" now correctly omits the unput() routine + from the output. + + - "make clean" now removes config.log, config.cache, and the + flex binary. The fact that it removes the flex binary means + you should take care if making changes to scan.l, to make + sure you don't wind up in a bootstrap problem. + + - In general, the Makefile has been reworked somewhat (thanks + to Francois Pinard) for added flexibility - more changes will + follow in subsequent releases. + + - The .texi and .info files in MISC/texinfo/ have been updated, + thanks also to Francois Pinard. + + - The FlexLexer::yylex(istream* new_in, ostream* new_out) method + now does not have a default for the first argument, to disambiguate + it from FlexLexer::yylex(). + + - A bug in destructing a FlexLexer object before doing any scanning + with it has been fixed. + + - A problem with including FlexLexer.h multiple times has been fixed. + + - The alloca() chud necessary to accommodate bison has grown + even uglier, but hopefully more correct. + + - A portability tweak has been added to accommodate compilers that + use char* generic pointers. + + - EBCDIC contact information in the file MISC/EBCDIC has been updated. + + - An OS/2 Makefile and config.h for flex 2.5 is now available in + MISC/OS2/, contributed by Kai Uwe Rommel. + + - The descrip.mms file for building flex under VMS has been updated, + thanks to Pat Rankin. + + - The notes on building flex for the Amiga have been updated for + flex 2.5, contributed by Andreas Scherer. + + +Changes between release 2.5.1 (28Mar95) and release 2.4.7: + + - A new concept of "start condition" scope has been introduced. + A start condition scope is begun with: + + <SCs>{ + + where SCs is a list of one or more start conditions. Inside + the start condition scope, every rule automatically has the + prefix <SCs> applied to it, until a '}' which matches the + initial '{'. So, for example: + + <ESC>{ + "\\n" return '\n'; + "\\r" return '\r'; + "\\f" return '\f'; + "\\0" return '\0'; + } + + is equivalent to: + + <ESC>"\\n" return '\n'; + <ESC>"\\r" return '\r'; + <ESC>"\\f" return '\f'; + <ESC>"\\0" return '\0'; + + As indicated in this example, rules inside start condition scopes + (and any rule, actually, other than the first) can be indented, + to better show the extent of the scope. + + Start condition scopes may be nested. + + - The new %option directive can be used in the first section of + a flex scanner to control scanner-generation options. Most + options are given simply as names, optionally preceded by the + word "no" (with no intervening whitespace) to negate their + meaning. Some are equivalent to flex flags, so putting them + in your scanner source is equivalent to always specifying + the flag (%option's take precedence over flags): + + 7bit -7 option + 8bit -8 option + align -Ca option + backup -b option + batch -B option + c++ -+ option + caseful opposite of -i option (caseful is the default); + case-sensitive same as above + caseless -i option; + case-insensitive same as above + debug -d option + default opposite of -s option + ecs -Ce option + fast -F option + full -f option + interactive -I option + lex-compat -l option + meta-ecs -Cm option + perf-report -p option + read -Cr option + stdout -t option + verbose -v option + warn opposite of -w option (so use "%option nowarn" for -w) + + array equivalent to "%array" + pointer equivalent to "%pointer" (default) + + Some provide new features: + + always-interactive generate a scanner which always + considers its input "interactive" (no call to isatty() + will be made when the scanner runs) + main supply a main program for the scanner, which + simply calls yylex(). Implies %option noyywrap. + never-interactive generate a scanner which never + considers its input "interactive" (no call to isatty() + will be made when the scanner runs) + stack if set, enable start condition stacks (see below) + stdinit if unset ("%option nostdinit"), initialize yyin + and yyout statically to nil FILE* pointers, instead + of stdin and stdout + yylineno if set, keep track of the current line + number in global yylineno (this option is expensive + in terms of performance). The line number is available + to C++ scanning objects via the new member function + lineno(). + yywrap if unset ("%option noyywrap"), scanner does not + call yywrap() upon EOF but simply assumes there + are no more files to scan + + Flex scans your rule actions to determine whether you use the + REJECT or yymore features (this is not new). Two %options can be + used to override its decision, either by setting them to indicate + the feature is indeed used, or unsetting them to indicate it + actually is not used: + + reject + yymore + + Three %option's take string-delimited values, offset with '=': + + outfile="<name>" equivalent to -o<name> + prefix="<name>" equivalent to -P<name> + yyclass="<name>" set the name of the C++ scanning class + (see below) + + A number of %option's are available for lint purists who + want to suppress the appearance of unneeded routines in + the generated scanner. Each of the following, if unset, + results in the corresponding routine not appearing in the + generated scanner: + + input, unput + yy_push_state, yy_pop_state, yy_top_state + yy_scan_buffer, yy_scan_bytes, yy_scan_string + + You can specify multiple options with a single %option directive, + and multiple directives in the first section of your flex input file. + + - The new function: + + YY_BUFFER_STATE yy_scan_string( const char *str ) + + returns a YY_BUFFER_STATE (which also becomes the current input + buffer) for scanning the given string, which occurs starting + with the next call to yylex(). The string must be NUL-terminated. + A related function: + + YY_BUFFER_STATE yy_scan_bytes( const char *bytes, int len ) + + creates a buffer for scanning "len" bytes (including possibly NUL's) + starting at location "bytes". + + Note that both of these functions create and scan a *copy* of + the string/bytes. (This may be desirable, since yylex() modifies + the contents of the buffer it is scanning.) You can avoid the + copy by using: + + YY_BUFFER_STATE yy_scan_buffer( char *base, yy_size_t size ) + + which scans in place the buffer starting at "base", consisting + of "size" bytes, the last two bytes of which *must* be + YY_END_OF_BUFFER_CHAR (these bytes are not scanned; thus, scanning + consists of base[0] through base[size-2], inclusive). If you + fail to set up "base" in this manner, yy_scan_buffer returns a + nil pointer instead of creating a new input buffer. + + The type yy_size_t is an integral type to which you can cast + an integer expression reflecting the size of the buffer. + + - Three new routines are available for manipulating stacks of + start conditions: + + void yy_push_state( int new_state ) + + pushes the current start condition onto the top of the stack + and BEGIN's "new_state" (recall that start condition names are + also integers). + + void yy_pop_state() + + pops the top of the stack and BEGIN's to it, and + + int yy_top_state() + + returns the top of the stack without altering the stack's + contents. + + The start condition stack grows dynamically and so has no built-in + size limitation. If memory is exhausted, program execution + is aborted. + + To use start condition stacks, your scanner must include + a "%option stack" directive. + + - flex now supports POSIX character class expressions. These + are expressions enclosed inside "[:" and ":]" delimiters (which + themselves must appear between the '[' and ']' of a character + class; other elements may occur inside the character class, too). + The expressions flex recognizes are: + + [:alnum:] [:alpha:] [:blank:] [:cntrl:] [:digit:] [:graph:] + [:lower:] [:print:] [:punct:] [:space:] [:upper:] [:xdigit:] + + These expressions all designate a set of characters equivalent to + the corresponding isXXX function (for example, [:alnum:] designates + those characters for which isalnum() returns true - i.e., any + alphabetic or numeric). Some systems don't provide isblank(), + so flex defines [:blank:] as a blank or a tab. + + For example, the following character classes are all equivalent: + + [[:alnum:]] + [[:alpha:][:digit:] + [[:alpha:]0-9] + [a-zA-Z0-9] + + If your scanner is case-insensitive (-i flag), then [:upper:] + and [:lower:] are equivalent to [:alpha:]. + + - The promised rewrite of the C++ FlexLexer class has not yet + been done. Support for FlexLexer is limited at the moment to + fixing show-stopper bugs, so, for example, the new functions + yy_scan_string() & friends are not available to FlexLexer + objects. + + - The new macro + + yy_set_interactive(is_interactive) + + can be used to control whether the current buffer is considered + "interactive". An interactive buffer is processed more slowly, + but must be used when the scanner's input source is indeed + interactive to avoid problems due to waiting to fill buffers + (see the discussion of the -I flag in flex.1). A non-zero value + in the macro invocation marks the buffer as interactive, a zero + value as non-interactive. Note that use of this macro overrides + "%option always-interactive" or "%option never-interactive". + + yy_set_interactive() must be invoked prior to beginning to + scan the buffer. + + - The new macro + + yy_set_bol(at_bol) + + can be used to control whether the current buffer's scanning + context for the next token match is done as though at the + beginning of a line (non-zero macro argument; makes '^' anchored + rules active) or not at the beginning of a line (zero argument, + '^' rules inactive). + + - Related to this change, the mechanism for determining when a scan is + starting at the beginning of a line has changed. It used to be + that '^' was active iff the character prior to that at which the + scan started was a newline. The mechanism now is that '^' is + active iff the last token ended in a newline (or the last call to + input() returned a newline). For most users, the difference in + mechanisms is negligible. Where it will make a difference, + however, is if unput() or yyless() is used to alter the input + stream. When in doubt, use yy_set_bol(). + + - The new beginning-of-line mechanism involved changing some fairly + twisted code, so it may have introduced bugs - beware ... + + - The macro YY_AT_BOL() returns true if the next token scanned from + the current buffer will have '^' rules active, false otherwise. + + - The new function + + void yy_flush_buffer( struct yy_buffer_state* b ) + + flushes the contents of the current buffer (i.e., next time + the scanner attempts to match a token using b as the current + buffer, it will begin by invoking YY_INPUT to fill the buffer). + This routine is also available to C++ scanners (unlike some + of the other new routines). + + The related macro + + YY_FLUSH_BUFFER + + flushes the contents of the current buffer. + + - A new "-ooutput" option writes the generated scanner to "output". + If used with -t, the scanner is still written to stdout, but + its internal #line directives (see previous item) use "output". + + - Flex now generates #line directives relating the code it + produces to the output file; this means that error messages + in the flex-generated code should be correctly pinpointed. + + - When generating #line directives, filenames with embedded '\'s + have those characters escaped (i.e., turned into '\\'). This + feature helps with reporting filenames for some MS-DOS and OS/2 + systems. + + - The FlexLexer class includes two new public member functions: + + virtual void switch_streams( istream* new_in = 0, + ostream* new_out = 0 ) + + reassigns yyin to new_in (if non-nil) and yyout to new_out + (ditto), deleting the previous input buffer if yyin is + reassigned. It is used by: + + int yylex( istream* new_in = 0, ostream* new_out = 0 ) + + which first calls switch_streams() and then returns the value + of calling yylex(). + + - C++ scanners now have yy_flex_debug as a member variable of + FlexLexer rather than a global, and member functions for testing + and setting it. + + - When generating a C++ scanning class, you can now use + + %option yyclass="foo" + + to inform flex that you have derived "foo" as a subclass of + yyFlexLexer, so flex will place your actions in the member + function foo::yylex() instead of yyFlexLexer::yylex(). It also + generates a yyFlexLexer::yylex() member function that generates a + run-time error if called (by invoking yyFlexLexer::LexerError()). + This feature is necessary if your subclass "foo" introduces some + additional member functions or variables that you need to access + from yylex(). + + - Current texinfo files in MISC/texinfo, contributed by Francois + Pinard. + + - You can now change the name "flex" to something else (e.g., "lex") + by redefining $(FLEX) in the Makefile. + + - Two bugs (one serious) that could cause "bigcheck" to fail have + been fixed. + + - A number of portability/configuration changes have been made + for easier portability. + + - You can use "YYSTATE" in your scanner as an alias for YY_START + (for AT&T lex compatibility). + + - input() now maintains yylineno. + + - input() no longer trashes yytext. + + - interactive scanners now read characters in YY_INPUT up to a + newline, a large performance gain. + + - C++ scanner objects now work with the -P option. You include + <FlexLexer.h> once per scanner - see comments in <FlexLexer.h> + (or flex.1) for details. + + - C++ FlexLexer objects now use the "cerr" stream to report -d output + instead of stdio. + + - The -c flag now has its full glorious POSIX interpretation (do + nothing), rather than being interpreted as an old-style -C flag. + + - Scanners generated by flex now include two #define's giving + the major and minor version numbers (YY_FLEX_MAJOR_VERSION, + YY_FLEX_MINOR_VERSION). These can then be tested to see + whether certain flex features are available. + + - Scanners generated using -l lex compatibility now have the symbol + YY_FLEX_LEX_COMPAT #define'd. + + - When initializing (i.e., yy_init is non-zero on entry to yylex()), + generated scanners now set yy_init to zero before executing + YY_USER_INIT. This means that you can set yy_init back to a + non-zero value in YY_USER_INIT if you need the scanner to be + reinitialized on the next call. + + - You can now use "#line" directives in the first section of your + scanner specification. + + - When generating full-table scanners (-Cf), flex now puts braces + around each row of the 2-d array initialization, to silence warnings + on over-zealous compilers. + + - Improved support for MS-DOS. The flex sources have been successfully + built, unmodified, for Borland 4.02 (all that's required is a + Borland Makefile and config.h file, which are supplied in + MISC/Borland - contributed by Terrence O Kane). + + - Improved support for Macintosh using Think C - the sources should + build for this platform "out of the box". Contributed by Scott + Hofmann. + + - Improved support for VMS, in MISC/VMS/, contributed by Pat Rankin. + + - Support for the Amiga, in MISC/Amiga/, contributed by Andreas + Scherer. Note that the contributed files were developed for + flex 2.4 and have not been tested with flex 2.5. + + - Some notes on support for the NeXT, in MISC/NeXT, contributed + by Raf Schietekat. + + - The MISC/ directory now includes a preformatted version of flex.1 + in flex.man, and pre-yacc'd versions of parse.y in parse.{c,h}. + + - The flex.1 and flexdoc.1 manual pages have been merged. There + is now just one document, flex.1, which includes an overview + at the beginning to help you find the section you need. + + - Documentation now clarifies that start conditions persist across + switches to new input files or different input buffers. If you + want to e.g., return to INITIAL, you must explicitly do so. + + - The "Performance Considerations" section of the manual has been + updated. + + - Documented the "yy_act" variable, which when YY_USER_ACTION is + invoked holds the number of the matched rule, and added an + example of using yy_act to profile how often each rule is matched. + + - Added YY_NUM_RULES, a definition that gives the total number + of rules in the file, including the default rule (even if you + use -s). + + - Documentation now clarifies that you can pass a nil FILE* pointer + to yy_create_buffer() or yyrestart() if you've arrange YY_INPUT + to not need yyin. + + - Documentation now clarifies that YY_BUFFER_STATE is a pointer to + an opaque "struct yy_buffer_state". + + - Documentation now stresses that you gain the benefits of removing + backing-up states only if you remove *all* of them. + + - Documentation now points out that traditional lex allows you + to put the action on a separate line from the rule pattern if + the pattern has trailing whitespace (ugh!), but flex doesn't + support this. + + - A broken example in documentation of the difference between + inclusive and exclusive start conditions is now fixed. + + - Usage (-h) report now goes to stdout. + + - Version (-V) info now goes to stdout. + + - More #ifdef chud has been added to the parser in attempt to + deal with bison's use of alloca(). + + - "make clean" no longer deletes emacs backup files (*~). + + - Some memory leaks have been fixed. + + - A bug was fixed in which dynamically-expanded buffers were + reallocated a couple of bytes too small. + + - A bug was fixed which could cause flex to read and write beyond + the end of the input buffer. + + - -S will not be going away. + + +Changes between release 2.4.7 (03Aug94) and release 2.4.6: + + - Fixed serious bug in reading multiple files. + + - Fixed bug in scanning NUL's. + + - Fixed bug in input() returning 8-bit characters. + + - Fixed bug in matching text with embedded NUL's when + using %array or lex compatibility. + + - Fixed multiple invocations of YY_USER_ACTION when using '|' + continuation action. + + - Minor prototyping fixes. + +Changes between release 2.4.6 (04Jan94) and release 2.4.5: + + - Linking with -lfl no longer required if your program includes + its own yywrap() and main() functions. (This change will cause + problems if you have a non-ANSI compiler on a system for which + sizeof(int) != sizeof(void*) or sizeof(int) != sizeof(size_t).) + + - The use of 'extern "C++"' in FlexLexer.h has been modified to + get around an incompatibility with g++'s header files. + +Changes between release 2.4.5 (11Dec93) and release 2.4.4: + + - Fixed bug breaking C++ scanners that use REJECT or variable + trailing context. + + - Fixed serious input problem for interactive scanners on + systems for which char is unsigned. + + - Fixed bug in incorrectly treating '$' operator as variable + trailing context. + + - Fixed bug in -CF table representation that could lead to + corrupt tables. + + - Fixed fairly benign memory leak. + + - Added `extern "C++"' wrapper to FlexLexer.h header. This + should overcome the g++ 2.5.X problems mentioned in the + NEWS for release 2.4.3. + + - Changed #include of FlexLexer.h to use <> instead of "". + + - Added feature to control whether the scanner attempts to + refill the input buffer once it's exhausted. This feature + will be documented in the 2.5 release. + + +Changes between release 2.4.4 (07Dec93) and release 2.4.3: + + - Fixed two serious bugs in scanning 8-bit characters. + + - Fixed bug in YY_USER_ACTION that caused it to be executed + inappropriately (on the scanner's own internal actions, and + with incorrect yytext/yyleng values). + + - Fixed bug in pointing yyin at a new file and resuming scanning. + + - Portability fix regarding min/max/abs macros conflicting with + function definitions in standard header files. + + - Added a virtual LexerError() method to the C++ yyFlexLexer class + for reporting error messages instead of always using cerr. + + - Added warning in flexdoc that the C++ scanning class is presently + experimental and subject to considerable change between major + releases. + + +Changes between release 2.4.3 (03Dec93) and release 2.4.2: + + - Fixed bug causing fatal scanner messages to fail to print. + + - Fixed things so FlexLexer.h can be included in other C++ + sources. One side-effect of this change is that -+ and -CF + are now incompatible. + + - libfl.a now supplies private versions of the the <string.h>/ + <strings.h> string routines needed by flex and the scanners + it generates, to enhance portability to some BSD systems. + + - More robust solution to 2.4.2's flexfatal() bug fix. + + - Added ranlib of installed libfl.a. + + - Some lint tweaks. + + - NOTE: problems have been encountered attempting to build flex + C++ scanners using g++ version 2.5.X. The problem is due to an + unfortunate heuristic in g++ 2.5.X that attempts to discern between + C and C++ headers. Because FlexLexer.h is installed (by default) + in /usr/local/include and not /usr/local/lib/g++-include, g++ 2.5.X + decides that it's a C header :-(. So if you have problems, install + the header in /usr/local/lib/g++-include instead. + + +Changes between release 2.4.2 (01Dec93) and release 2.4.1: + + - Fixed bug in libfl.a referring to non-existent "flexfatal" function. + + - Modified to produce both compress'd and gzip'd tar files for + distributions (you probably don't care about this change!). + + +Changes between release 2.4.1 (30Nov93) and release 2.3.8: + + - The new '-+' flag instructs flex to generate a C++ scanner class + (thanks to Kent Williams). flex writes an implementation of the + class defined in FlexLexer.h to lex.yy.cc. You may include + multiple scanner classes in your program using the -P flag. Note + that the scanner class also provides a mechanism for creating + reentrant scanners. The scanner class uses C++ streams for I/O + instead of FILE*'s (thanks to Tom Epperly). If the flex executable's + name ends in '+' then the '-+' flag is automatically on, so creating + a symlink or copy of "flex" to "flex++" results in a version of + flex that can be used exclusively for C++ scanners. + + Note that without the '-+' flag, flex-generated scanners can still + be compiled using C++ compilers, though they use FILE*'s for I/O + instead of streams. + + See the "GENERATING C++ SCANNERS" section of flexdoc for details. + + - The new '-l' flag turns on maximum AT&T lex compatibility. In + particular, -l includes support for "yylineno" and makes yytext + be an array instead of a pointer. It does not, however, do away + with all incompatibilities. See the "INCOMPATIBILITIES WITH LEX + AND POSIX" section of flexdoc for details. + + - The new '-P' option specifies a prefix to use other than "yy" + for the scanner's globally-visible variables, and for the + "lex.yy.c" filename. Using -P you can link together multiple + flex scanners in the same executable. + + - The distribution includes a "texinfo" version of flexdoc.1, + contributed by Roland Pesch (thanks also to Marq Kole, who + contributed another version). It has not been brought up to + date, but reflects version 2.3. See MISC/flex.texinfo. + + The flex distribution will soon include G.T. Nicol's flex + manual; he is presently bringing it up-to-date for version 2.4. + + - yywrap() is now a function, and you now *must* link flex scanners + with libfl.a. + + - Site-configuration is now done via an autoconf-generated + "configure" script contributed by Francois Pinard. + + - Scanners now use fread() (or getc(), if interactive) and not + read() for input. A new "table compression" option, -Cr, + overrides this change and causes the scanner to use read() + (because read() is a bit faster than fread()). -f and -F + are now equivalent to -Cfr and -CFr; i.e., they imply the + -Cr option. + + - In the blessed name of POSIX compliance, flex supports "%array" + and "%pointer" directives in the definitions (first) section of + the scanner specification. The former specifies that yytext + should be an array (of size YYLMAX), the latter, that it should + be a pointer. The array version of yytext is universally slower + than the pointer version, but has the advantage that its contents + remain unmodified across calls to input() and unput() (the pointer + version of yytext is, still, trashed by such calls). + + "%array" cannot be used with the '-+' C++ scanner class option. + + - The new '-Ca' option directs flex to trade off memory for + natural alignment when generating a scanner's tables. In + particular, table entries that would otherwise be "short" + become "long". + + - The new '-h' option produces a summary of the flex flags. + + - The new '-V' option reports the flex version number and exits. + + - The new scanner macro YY_START returns an integer value + corresponding to the current start condition. You can return + to that start condition by passing the value to a subsequent + "BEGIN" action. You also can implement "start condition stacks" + by storing the values in an integer stack. + + - You can now redefine macros such as YY_INPUT by just #define'ing + them to some other value in the first section of the flex input; + no need to first #undef them. + + - flex now generates warnings for rules that can't be matched. + These warnings can be turned off using the new '-w' flag. If + your scanner uses REJECT then you will not get these warnings. + + - If you specify the '-s' flag but the default rule can be matched, + flex now generates a warning. + + - "yyleng" is now a global, and may be modified by the user (though + doing so and then using yymore() will yield weird results). + + - Name definitions in the first section of a scanner specification + can now include a leading '^' or trailing '$' operator. In this + case, the definition is *not* pushed back inside of parentheses. + + - Scanners with compressed tables are now "interactive" (-I option) + by default. You can suppress this attribute (which makes them + run slightly slower) using the new '-B' flag. + + - Flex now generates 8-bit scanners by default, unless you use the + -Cf or -CF compression options (-Cfe and -CFe result in 8-bit + scanners). You can force it to generate a 7-bit scanner using + the new '-7' flag. You can build flex to generate 8-bit scanners + for -Cf and -CF, too, by adding -DDEFAULT_CSIZE=256 to CFLAGS + in the Makefile. + + - You no longer need to call the scanner routine yyrestart() to + inform the scanner that you have switched to a new file after + having seen an EOF on the current input file. Instead, just + point yyin at the new file and continue scanning. + + - You no longer need to invoke YY_NEW_FILE in an <<EOF>> action + to indicate you wish to continue scanning. Simply point yyin + at a new file. + + - A leading '#' no longer introduces a comment in a flex input. + + - flex no longer considers formfeed ('\f') a whitespace character. + + - %t, I'm happy to report, has been nuked. + + - The '-p' option may be given twice ('-pp') to instruct flex to + report minor performance problems as well as major ones. + + - The '-v' verbose output no longer includes start/finish time + information. + + - Newlines in flex inputs can optionally include leading or + trailing carriage-returns ('\r'), in support of several PC/Mac + run-time libraries that automatically include these. + + - A start condition of the form "<*>" makes the following rule + active in every start condition, whether exclusive or inclusive. + + - The following items have been corrected in the flex documentation: + + - '-C' table compression options *are* cumulative. + + - You may modify yytext but not lengthen it by appending + characters to the end. Modifying its final character + will affect '^' anchoring for the next rule matched + if the character is changed to or from a newline. + + - The term "backtracking" has been renamed "backing up", + since it is a one-time repositioning and not a repeated + search. What used to be the "lex.backtrack" file is now + "lex.backup". + + - Unindented "/* ... */" comments are allowed in the first + flex input section, but not in the second. + + - yyless() can only be used in the flex input source, not + externally. + + - You can use "yyrestart(yyin)" to throw away the + current contents of the input buffer. + + - To write high-speed scanners, attempt to match as much + text as possible with each rule. See MISC/fastwc/README + for more information. + + - Using the beginning-of-line operator ('^') is fairly + cheap. Using unput() is expensive. Using yyless() is + cheap. + + - An example of scanning strings with embedded escape + sequences has been added. + + - The example of backing-up in flexdoc was erroneous; it + has been corrected. + + - A flex scanner's internal buffer now dynamically grows if needed + to match large tokens. Note that growing the buffer presently + requires rescanning the (large) token, so consuming a lot of + text this way is a slow process. Also note that presently the + buffer does *not* grow if you unput() more text than can fit + into the buffer. + + - The MISC/ directory has been reorganized; see MISC/README for + details. + + - yyless() can now be used in the third (user action) section + of a scanner specification, thanks to Ceriel Jacobs. yyless() + remains a macro and cannot be used outside of the scanner source. + + - The skeleton file is no longer opened at run-time, but instead + compiled into a large string array (thanks to John Gilmore and + friends at Cygnus). You can still use the -S flag to point flex + at a different skeleton file. + + - flex no longer uses a temporary file to store the scanner's + actions. + + - A number of changes have been made to decrease porting headaches. + In particular, flex no longer uses memset() or ctime(), and + provides a single simple mechanism for dealing with C compilers + that still define malloc() as returning char* instead of void*. + + - Flex now detects if the scanner specification requires the -8 flag + but the flag was not given or on by default. + + - A number of table-expansion fencepost bugs have been fixed, + making flex more robust for generating large scanners. + + - flex more consistently identifies the location of errors in + its input. + + - YY_USER_ACTION is now invoked only for "real" actions, not for + internal actions used by the scanner for things like filling + the buffer or handling EOF. + + - The rule "[^]]" now matches any character other than a ']'; + formerly it matched any character at all followed by a ']'. + This change was made for compatibility with AT&T lex. + + - A large number of miscellaneous bugs have been found and fixed + thanks to Gerhard Wilhelms. + + - The source code has been heavily reformatted, making patches + relative to previous flex releases no longer accurate. + + +Changes between 2.3 Patch #8 (21Feb93) and 2.3 Patch #7: + + - Fixed bugs in dynamic memory allocation leading to grievous + fencepost problems when generating large scanners. + - Fixed bug causing infinite loops on character classes with 8-bit + characters in them. + - Fixed bug in matching repetitions with a lower bound of 0. + - Fixed bug in scanning NUL characters using an "interactive" scanner. + - Fixed bug in using yymore() at the end of a file. + - Fixed bug in misrecognizing rules with variable trailing context. + - Fixed bug compiling flex on Suns using gcc 2. + - Fixed bug in not recognizing that input files with the character + ASCII 128 in them require the -8 flag. + - Fixed bug that could cause an infinite loop writing out + error messages. + - Fixed bug in not recognizing old-style lex % declarations if + followed by a tab instead of a space. + - Fixed potential crash when flex terminated early (usually due + to a bad flag) and the -v flag had been given. + - Added some missing declarations of void functions. + - Changed to only use '\a' for __STDC__ compilers. + - Updated mailing addresses. + + +Changes between 2.3 Patch #7 (28Mar91) and 2.3 Patch #6: + + - Fixed out-of-bounds array access that caused bad tables + to be produced on machines where the bad reference happened + to yield a 1. This caused problems installing or running + flex on some Suns, in particular. + + +Changes between 2.3 Patch #6 (29Aug90) and 2.3 Patch #5: + + - Fixed a serious bug in yymore() which basically made it + completely broken. Thanks goes to Jean Christophe of + the Nethack development team for finding the problem + and passing along the fix. + + +Changes between 2.3 Patch #5 (16Aug90) and 2.3 Patch #4: + + - An up-to-date version of initscan.c so "make test" will + work after applying the previous patches + + +Changes between 2.3 Patch #4 (14Aug90) and 2.3 Patch #3: + + - Fixed bug in hexadecimal escapes which allowed only digits, + not letters, in escapes + - Fixed bug in previous "Changes" file! + + +Changes between 2.3 Patch #3 (03Aug90) and 2.3 Patch #2: + + - Correction to patch #2 for gcc compilation; thanks goes to + Paul Eggert for catching this. + + +Changes between 2.3 Patch #2 (02Aug90) and original 2.3 release: + + - Fixed (hopefully) headaches involving declaring malloc() + and free() for gcc, which defines __STDC__ but (often) doesn't + come with the standard include files such as <stdlib.h>. + Reordered #ifdef maze in the scanner skeleton in the hope of + getting the declarations right for cfront and g++, too. + + - Note that this patch supercedes patch #1 for release 2.3, + which was never announced but was available briefly for + anonymous ftp. + + +Changes between 2.3 (full) release of 28Jun90 and 2.2 (alpha) release: + + User-visible: + + - A lone <<EOF>> rule (that is, one which is not qualified with + a list of start conditions) now specifies the EOF action for + *all* start conditions which haven't already had <<EOF>> actions + given. To specify an end-of-file action for just the initial + state, use <INITIAL><<EOF>>. + + - -d debug output is now contingent on the global yy_flex_debug + being set to a non-zero value, which it is by default. + + - A new macro, YY_USER_INIT, is provided for the user to specify + initialization action to be taken on the first call to the + scanner. This action is done before the scanner does its + own initialization. + + - yy_new_buffer() has been added as an alias for yy_create_buffer() + + - Comments beginning with '#' and extending to the end of the line + now work, but have been deprecated (in anticipation of making + flex recognize #line directives). + + - The funky restrictions on when semi-colons could follow the + YY_NEW_FILE and yyless macros have been removed. They now + behave identically to functions. + + - A bug in the sample redefinition of YY_INPUT in the documentation + has been corrected. + + - A bug in the sample simple tokener in the documentation has + been corrected. + + - The documentation on the incompatibilities between flex and + lex has been reordered so that the discussion of yylineno + and input() come first, as it's anticipated that these will + be the most common source of headaches. + + + Things which didn't used to be documented but now are: + + - flex interprets "^foo|bar" differently from lex. flex interprets + it as "match either a 'foo' or a 'bar', providing it comes at the + beginning of a line", whereas lex interprets it as "match either + a 'foo' at the beginning of a line, or a 'bar' anywhere". + + - flex initializes the global "yyin" on the first call to the + scanner, while lex initializes it at compile-time. + + - yy_switch_to_buffer() can be used in the yywrap() macro/routine. + + - flex scanners do not use stdio for their input, and hence when + writing an interactive scanner one must explicitly call fflush() + after writing out a prompt. + + - flex scanner can be made reentrant (after a fashion) by using + "yyrestart( yyin );". This is useful for interactive scanners + which have interrupt handlers that long-jump out of the scanner. + + - a defense of why yylineno is not supported is included, along + with a suggestion on how to convert scanners which rely on it. + + + Other changes: + + - Prototypes and proper declarations of void routines have + been added to the flex source code, courtesy of Kevin B. Kenny. + + - Routines dealing with memory allocation now use void* pointers + instead of char* - see Makefile for porting implications. + + - Error-checking is now done when flex closes a file. + + - Various lint tweaks were added to reduce the number of gripes. + + - Makefile has been further parameterized to aid in porting. + + - Support for SCO Unix added. + + - Flex now sports the latest & greatest UC copyright notice + (which is only slightly different from the previous one). + + - A note has been added to flexdoc.1 mentioning work in progress + on modifying flex to generate straight C code rather than a + table-driven automaton, with an email address of whom to contact + if you are working along similar lines. + + +Changes between 2.2 Patch #3 (30Mar90) and 2.2 Patch #2: + + - fixed bug which caused -I scanners to bomb + + +Changes between 2.2 Patch #2 (27Mar90) and 2.2 Patch #1: + + - fixed bug writing past end of input buffer in yyunput() + - fixed bug detecting NUL's at the end of a buffer + + +Changes between 2.2 Patch #1 (23Mar90) and 2.2 (alpha) release: + + - Makefile fixes: definition of MAKE variable for systems + which don't have it; installation of flexdoc.1 along with + flex.1; fixed two bugs which could cause "bigtest" to fail. + + - flex.skel fix for compiling with g++. + + - README and flexdoc.1 no longer list an out-of-date BITNET address + for contacting me. + + - minor typos and formatting changes to flex.1 and flexdoc.1. + + +Changes between 2.2 (alpha) release of March '90 and previous release: + + User-visible: + + - Full user documentation now available. + + - Support for 8-bit scanners. + + - Scanners now accept NUL's. + + - A facility has been added for dealing with multiple + input buffers. + + - Two manual entries now. One which fully describes flex + (rather than just its differences from lex), and the + other for quick(er) reference. + + - A number of changes to bring flex closer into compliance + with the latest POSIX lex draft: + + %t support + flex now accepts multiple input files and concatenates + them together to form its input + previous -c (compress) flag renamed -C + do-nothing -c and -n flags added + Any indented code or code within %{}'s in section 2 is + now copied to the output + + - yyleng is now a bona fide global integer. + + - -d debug information now gives the line number of the + matched rule instead of which number rule it was from + the beginning of the file. + + - -v output now includes a summary of the flags used to generate + the scanner. + + - unput() and yyrestart() are now globally callable. + + - yyrestart() no longer closes the previous value of yyin. + + - C++ support; generated scanners can be compiled with C++ compiler. + + - Primitive -lfl library added, containing default main() + which calls yylex(). A number of routines currently living + in the scanner skeleton will probably migrate to here + in the future (in particular, yywrap() will probably cease + to be a macro and instead be a function in the -lfl library). + + - Hexadecimal (\x) escape sequences added. + + - Support for MS-DOS, VMS, and Turbo-C integrated. + + - The %used/%unused operators have been deprecated. They + may go away soon. + + + Other changes: + + - Makefile enhanced for easier testing and installation. + - The parser has been tweaked to detect some erroneous + constructions which previously were missed. + - Scanner input buffer overflow is now detected. + - Bugs with missing "const" declarations fixed. + - Out-of-date Minix/Atari patches provided. + - Scanners no longer require printf() unless FLEX_DEBUG is being used. + - A subtle input() bug has been fixed. + - Line numbers for "continued action" rules (those following + the special '|' action) are now correct. + - unput() bug fixed; had been causing problems porting flex to VMS. + - yymore() handling rewritten to fix bug with interaction + between yymore() and trailing context. + - EOF in actions now generates an error message. + - Bug involving -CFe and generating equivalence classes fixed. + - Bug which made -CF be treated as -Cf fixed. + - Support for SysV tmpnam() added. + - Unused #define's for scanner no longer generated. + - Error messages which are associated with a particular input + line are now all identified with their input line in standard + format. + - % directives which are valid to lex but not to flex are + now ignored instead of generating warnings. + - -DSYS_V flag can now also be specified -DUSG for System V + compilation. + + +Changes between 2.1 beta-test release of June '89 and previous release: + + User-visible: + + - -p flag generates a performance report to stderr. The report + consists of comments regarding features of the scanner rules + which result in slower scanners. + + - -b flag generates backtracking information to lex.backtrack. + This is a list of scanner states which require backtracking + and the characters on which they do so. By adding rules + one can remove backtracking states. If all backtracking states + are eliminated, the generated scanner will run faster. + Backtracking is not yet documented in the manual entry. + + - Variable trailing context now works, i.e., one can have + rules like "(foo)*/[ \t]*bletch". Some trailing context + patterns still cannot be properly matched and generate + error messages. These are patterns where the ending of the + first part of the rule matches the beginning of the second + part, such as "zx*/xy*", where the 'x*' matches the 'x' at + the beginning of the trailing context. Lex won't get these + patterns right either. + + - Faster scanners. + + - End-of-file rules. The special rule "<<EOF>>" indicates + actions which are to be taken when an end-of-file is + encountered and yywrap() returns non-zero (i.e., indicates + no further files to process). See manual entry for example. + + - The -r (reject used) flag is gone. flex now scans the input + for occurrences of the string "REJECT" to determine if the + action is needed. It tries to be intelligent about this but + can be fooled. One can force the presence or absence of + REJECT by adding a line in the first section of the form + "%used REJECT" or "%unused REJECT". + + - yymore() has been implemented. Similarly to REJECT, flex + detects the use of yymore(), which can be overridden using + "%used" or "%unused". + + - Patterns like "x{0,3}" now work (i.e., with lower-limit == 0). + + - Removed '\^x' for ctrl-x misfeature. + + - Added '\a' and '\v' escape sequences. + + - \<digits> now works for octal escape sequences; previously + \0<digits> was required. + + - Better error reporting; line numbers are associated with rules. + + - yyleng is a macro; it cannot be accessed outside of the + scanner source file. + + - yytext and yyleng should not be modified within a flex action. + + - Generated scanners #define the name FLEX_SCANNER. + + - Rules are internally separated by YY_BREAK in lex.yy.c rather + than break, to allow redefinition. + + - The macro YY_USER_ACTION can be redefined to provide an action + which is always executed prior to the matched rule's action. + + - yyrestart() is a new action which can be used to restart + the scanner after it has seen an end-of-file (a "real" one, + that is, one for which yywrap() returned non-zero). It takes + a FILE* argument indicating a new file to scan and sets + things up so that a subsequent call to yylex() will start + scanning that file. + + - Internal scanner names all preceded by "yy_" + + - lex.yy.c is deleted if errors are encountered during processing. + + - Comments may be put in the first section of the input by preceding + them with '#'. + + + + Other changes: + + - Some portability-related bugs fixed, in particular for machines + with unsigned characters or sizeof( int* ) != sizeof( int ). + Also, tweaks for VMS and Microsoft C (MS-DOS), and identifiers all + trimmed to be 31 or fewer characters. Shortened file names + for dinosaur OS's. Checks for allocating > 64K memory + on 16 bit'ers. Amiga tweaks. Compiles using gcc on a Sun-3. + - Compressed and fast scanner skeletons merged. + - Skeleton header files done away with. + - Generated scanner uses prototypes and "const" for __STDC__. + - -DSV flag is now -DSYS_V for System V compilation. + - Removed all references to FTL language. + - Software now covered by BSD Copyright. + - flex will replace lex in subsequent BSD releases. diff --git a/usr.bin/lex/README b/usr.bin/lex/README new file mode 100644 index 0000000..7a4224d --- /dev/null +++ b/usr.bin/lex/README @@ -0,0 +1,60 @@ +This is release 2.5 of flex. See "version.h" for the exact patch-level. + +See the file "NEWS" to find out what is new in this Flex release. + +Read the file "INSTALL" for general installation directives. Peek near +the beginning of the file "Makefile.in" for special DEFS values. On most +systems, you can just run the "configure" script and type "make" to build +flex; then "make check" to test whether it built correctly; and if it did, +then "make install" to install it. + +If you're feeling adventurous, you can also issue "make bigcheck" (be +prepared to wait a while). + +Note that flex is distributed under a copyright very similar to that of +BSD Unix, and not under the GNU General Public License (GPL), except for +the "configure" script, which is covered by the GPL. + +Many thanks to the 2.5 beta-testers for finding bugs and helping test and +increase portability: Stan Adermann, Scott David Daniels, Charles Elliott, +Joe Gayda, Chris Meier, James Nordby, Terrence O'Kane, Karsten Pahnke, +Francois Pinard, Pat Rankin, Andreas Scherer, Marc Wiese, Nathan Zelle. + +Please send bug reports and feedback to: Vern Paxson (vern@ee.lbl.gov). + + +The flex distribution consists of the following files: + + README This message + + NEWS Differences between the various releases + + INSTALL General installation information + + COPYING flex's copyright + + conf.in, configure.in, configure, Makefile.in, install.sh, + mkinstalldirs + elements of the "autoconf" auto-configuration process + + flexdef.h, parse.y, scan.l, ccl.c, dfa.c, ecs.c, gen.c, main.c, + misc.c, nfa.c, sym.c, tblcmp.c, yylex.c + source files + + version.h version of this flex release + + flex.skl flex scanner skeleton + mkskel.sh script for converting flex.skl to C source file skel.c + skel.c pre-converted C version of flex.skl + + libmain.c flex library (-lfl) sources + libyywrap.c + + initscan.c pre-flex'd version of scan.l + + FlexLexer.h header file for C++ lexer class + + flex.1 user documentation + + MISC/ a directory containing miscellaneous contributions. + See MISC/README for details. diff --git a/usr.bin/lex/ccl.c b/usr.bin/lex/ccl.c new file mode 100644 index 0000000..f2607bf --- /dev/null +++ b/usr.bin/lex/ccl.c @@ -0,0 +1,151 @@ +/* ccl - routines for character classes */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/ccl.c,v 2.9 93/09/16 20:32:14 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + +/* ccladd - add a single character to a ccl */ + +void ccladd( cclp, ch ) +int cclp; +int ch; + { + int ind, len, newpos, i; + + check_char( ch ); + + len = ccllen[cclp]; + ind = cclmap[cclp]; + + /* check to see if the character is already in the ccl */ + + for ( i = 0; i < len; ++i ) + if ( ccltbl[ind + i] == ch ) + return; + + newpos = ind + len; + + if ( newpos >= current_max_ccl_tbl_size ) + { + current_max_ccl_tbl_size += MAX_CCL_TBL_SIZE_INCREMENT; + + ++num_reallocs; + + ccltbl = reallocate_Character_array( ccltbl, + current_max_ccl_tbl_size ); + } + + ccllen[cclp] = len + 1; + ccltbl[newpos] = ch; + } + + +/* cclinit - return an empty ccl */ + +int cclinit() + { + if ( ++lastccl >= current_maxccls ) + { + current_maxccls += MAX_CCLS_INCREMENT; + + ++num_reallocs; + + cclmap = reallocate_integer_array( cclmap, current_maxccls ); + ccllen = reallocate_integer_array( ccllen, current_maxccls ); + cclng = reallocate_integer_array( cclng, current_maxccls ); + } + + if ( lastccl == 1 ) + /* we're making the first ccl */ + cclmap[lastccl] = 0; + + else + /* The new pointer is just past the end of the last ccl. + * Since the cclmap points to the \first/ character of a + * ccl, adding the length of the ccl to the cclmap pointer + * will produce a cursor to the first free space. + */ + cclmap[lastccl] = cclmap[lastccl - 1] + ccllen[lastccl - 1]; + + ccllen[lastccl] = 0; + cclng[lastccl] = 0; /* ccl's start out life un-negated */ + + return lastccl; + } + + +/* cclnegate - negate the given ccl */ + +void cclnegate( cclp ) +int cclp; + { + cclng[cclp] = 1; + } + + +/* list_character_set - list the members of a set of characters in CCL form + * + * Writes to the given file a character-class representation of those + * characters present in the given CCL. A character is present if it + * has a non-zero value in the cset array. + */ + +void list_character_set( file, cset ) +FILE *file; +int cset[]; + { + int i; + + putc( '[', file ); + + for ( i = 0; i < csize; ++i ) + { + if ( cset[i] ) + { + int start_char = i; + + putc( ' ', file ); + + fputs( readable_form( i ), file ); + + while ( ++i < csize && cset[i] ) + ; + + if ( i - 1 > start_char ) + /* this was a run */ + fprintf( file, "-%s", readable_form( i - 1 ) ); + + putc( ' ', file ); + } + } + + putc( ']', file ); + } diff --git a/usr.bin/lex/config.h b/usr.bin/lex/config.h new file mode 100644 index 0000000..2e0b5b4 --- /dev/null +++ b/usr.bin/lex/config.h @@ -0,0 +1,26 @@ +/* config.h. Generated automatically by configure. */ +/* $FreeBSD$ */ + +/* Define to empty if the keyword does not work. */ +/* #undef const */ + +/* Define to `unsigned' if <sys/types.h> doesn't define. */ +/* #undef size_t */ + +/* Define if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define if you have the <malloc.h> header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define if you have <alloca.h> and it should be used (not on Ultrix). */ +/* #undef HAVE_ALLOCA_H */ + +/* Define if platform-specific command line handling is necessary. */ +/* #undef NEED_ARGV_FIXUP */ diff --git a/usr.bin/lex/dfa.c b/usr.bin/lex/dfa.c new file mode 100644 index 0000000..5a86af1 --- /dev/null +++ b/usr.bin/lex/dfa.c @@ -0,0 +1,1097 @@ +/* dfa - DFA construction routines */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/dfa.c,v 2.26 95/04/20 13:53:14 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + + +/* declare functions that have forward references */ + +void dump_associated_rules PROTO((FILE*, int)); +void dump_transitions PROTO((FILE*, int[])); +void sympartition PROTO((int[], int, int[], int[])); +int symfollowset PROTO((int[], int, int, int[])); + + +/* check_for_backing_up - check a DFA state for backing up + * + * synopsis + * void check_for_backing_up( int ds, int state[numecs] ); + * + * ds is the number of the state to check and state[] is its out-transitions, + * indexed by equivalence class. + */ + +void check_for_backing_up( ds, state ) +int ds; +int state[]; + { + if ( (reject && ! dfaacc[ds].dfaacc_set) || + (! reject && ! dfaacc[ds].dfaacc_state) ) + { /* state is non-accepting */ + ++num_backing_up; + + if ( backing_up_report ) + { + fprintf( backing_up_file, + _( "State #%d is non-accepting -\n" ), ds ); + + /* identify the state */ + dump_associated_rules( backing_up_file, ds ); + + /* Now identify it further using the out- and + * jam-transitions. + */ + dump_transitions( backing_up_file, state ); + + putc( '\n', backing_up_file ); + } + } + } + + +/* check_trailing_context - check to see if NFA state set constitutes + * "dangerous" trailing context + * + * synopsis + * void check_trailing_context( int nfa_states[num_states+1], int num_states, + * int accset[nacc+1], int nacc ); + * + * NOTES + * Trailing context is "dangerous" if both the head and the trailing + * part are of variable size \and/ there's a DFA state which contains + * both an accepting state for the head part of the rule and NFA states + * which occur after the beginning of the trailing context. + * + * When such a rule is matched, it's impossible to tell if having been + * in the DFA state indicates the beginning of the trailing context or + * further-along scanning of the pattern. In these cases, a warning + * message is issued. + * + * nfa_states[1 .. num_states] is the list of NFA states in the DFA. + * accset[1 .. nacc] is the list of accepting numbers for the DFA state. + */ + +void check_trailing_context( nfa_states, num_states, accset, nacc ) +int *nfa_states, num_states; +int *accset; +int nacc; + { + int i, j; + + for ( i = 1; i <= num_states; ++i ) + { + int ns = nfa_states[i]; + int type = state_type[ns]; + int ar = assoc_rule[ns]; + + if ( type == STATE_NORMAL || rule_type[ar] != RULE_VARIABLE ) + { /* do nothing */ + } + + else if ( type == STATE_TRAILING_CONTEXT ) + { + /* Potential trouble. Scan set of accepting numbers + * for the one marking the end of the "head". We + * assume that this looping will be fairly cheap + * since it's rare that an accepting number set + * is large. + */ + for ( j = 1; j <= nacc; ++j ) + if ( accset[j] & YY_TRAILING_HEAD_MASK ) + { + line_warning( + _( "dangerous trailing context" ), + rule_linenum[ar] ); + return; + } + } + } + } + + +/* dump_associated_rules - list the rules associated with a DFA state + * + * Goes through the set of NFA states associated with the DFA and + * extracts the first MAX_ASSOC_RULES unique rules, sorts them, + * and writes a report to the given file. + */ + +void dump_associated_rules( file, ds ) +FILE *file; +int ds; + { + int i, j; + int num_associated_rules = 0; + int rule_set[MAX_ASSOC_RULES + 1]; + int *dset = dss[ds]; + int size = dfasiz[ds]; + + for ( i = 1; i <= size; ++i ) + { + int rule_num = rule_linenum[assoc_rule[dset[i]]]; + + for ( j = 1; j <= num_associated_rules; ++j ) + if ( rule_num == rule_set[j] ) + break; + + if ( j > num_associated_rules ) + { /* new rule */ + if ( num_associated_rules < MAX_ASSOC_RULES ) + rule_set[++num_associated_rules] = rule_num; + } + } + + bubble( rule_set, num_associated_rules ); + + fprintf( file, _( " associated rule line numbers:" ) ); + + for ( i = 1; i <= num_associated_rules; ++i ) + { + if ( i % 8 == 1 ) + putc( '\n', file ); + + fprintf( file, "\t%d", rule_set[i] ); + } + + putc( '\n', file ); + } + + +/* dump_transitions - list the transitions associated with a DFA state + * + * synopsis + * dump_transitions( FILE *file, int state[numecs] ); + * + * Goes through the set of out-transitions and lists them in human-readable + * form (i.e., not as equivalence classes); also lists jam transitions + * (i.e., all those which are not out-transitions, plus EOF). The dump + * is done to the given file. + */ + +void dump_transitions( file, state ) +FILE *file; +int state[]; + { + int i, ec; + int out_char_set[CSIZE]; + + for ( i = 0; i < csize; ++i ) + { + ec = ABS( ecgroup[i] ); + out_char_set[i] = state[ec]; + } + + fprintf( file, _( " out-transitions: " ) ); + + list_character_set( file, out_char_set ); + + /* now invert the members of the set to get the jam transitions */ + for ( i = 0; i < csize; ++i ) + out_char_set[i] = ! out_char_set[i]; + + fprintf( file, _( "\n jam-transitions: EOF " ) ); + + list_character_set( file, out_char_set ); + + putc( '\n', file ); + } + + +/* epsclosure - construct the epsilon closure of a set of ndfa states + * + * synopsis + * int *epsclosure( int t[num_states], int *numstates_addr, + * int accset[num_rules+1], int *nacc_addr, + * int *hashval_addr ); + * + * NOTES + * The epsilon closure is the set of all states reachable by an arbitrary + * number of epsilon transitions, which themselves do not have epsilon + * transitions going out, unioned with the set of states which have non-null + * accepting numbers. t is an array of size numstates of nfa state numbers. + * Upon return, t holds the epsilon closure and *numstates_addr is updated. + * accset holds a list of the accepting numbers, and the size of accset is + * given by *nacc_addr. t may be subjected to reallocation if it is not + * large enough to hold the epsilon closure. + * + * hashval is the hash value for the dfa corresponding to the state set. + */ + +int *epsclosure( t, ns_addr, accset, nacc_addr, hv_addr ) +int *t, *ns_addr, accset[], *nacc_addr, *hv_addr; + { + int stkpos, ns, tsp; + int numstates = *ns_addr, nacc, hashval, transsym, nfaccnum; + int stkend, nstate; + static int did_stk_init = false, *stk; + +#define MARK_STATE(state) \ +trans1[state] = trans1[state] - MARKER_DIFFERENCE; + +#define IS_MARKED(state) (trans1[state] < 0) + +#define UNMARK_STATE(state) \ +trans1[state] = trans1[state] + MARKER_DIFFERENCE; + +#define CHECK_ACCEPT(state) \ +{ \ +nfaccnum = accptnum[state]; \ +if ( nfaccnum != NIL ) \ +accset[++nacc] = nfaccnum; \ +} + +#define DO_REALLOCATION \ +{ \ +current_max_dfa_size += MAX_DFA_SIZE_INCREMENT; \ +++num_reallocs; \ +t = reallocate_integer_array( t, current_max_dfa_size ); \ +stk = reallocate_integer_array( stk, current_max_dfa_size ); \ +} \ + +#define PUT_ON_STACK(state) \ +{ \ +if ( ++stkend >= current_max_dfa_size ) \ +DO_REALLOCATION \ +stk[stkend] = state; \ +MARK_STATE(state) \ +} + +#define ADD_STATE(state) \ +{ \ +if ( ++numstates >= current_max_dfa_size ) \ +DO_REALLOCATION \ +t[numstates] = state; \ +hashval += state; \ +} + +#define STACK_STATE(state) \ +{ \ +PUT_ON_STACK(state) \ +CHECK_ACCEPT(state) \ +if ( nfaccnum != NIL || transchar[state] != SYM_EPSILON ) \ +ADD_STATE(state) \ +} + + + if ( ! did_stk_init ) + { + stk = allocate_integer_array( current_max_dfa_size ); + did_stk_init = true; + } + + nacc = stkend = hashval = 0; + + for ( nstate = 1; nstate <= numstates; ++nstate ) + { + ns = t[nstate]; + + /* The state could be marked if we've already pushed it onto + * the stack. + */ + if ( ! IS_MARKED(ns) ) + { + PUT_ON_STACK(ns) + CHECK_ACCEPT(ns) + hashval += ns; + } + } + + for ( stkpos = 1; stkpos <= stkend; ++stkpos ) + { + ns = stk[stkpos]; + transsym = transchar[ns]; + + if ( transsym == SYM_EPSILON ) + { + tsp = trans1[ns] + MARKER_DIFFERENCE; + + if ( tsp != NO_TRANSITION ) + { + if ( ! IS_MARKED(tsp) ) + STACK_STATE(tsp) + + tsp = trans2[ns]; + + if ( tsp != NO_TRANSITION && ! IS_MARKED(tsp) ) + STACK_STATE(tsp) + } + } + } + + /* Clear out "visit" markers. */ + + for ( stkpos = 1; stkpos <= stkend; ++stkpos ) + { + if ( IS_MARKED(stk[stkpos]) ) + UNMARK_STATE(stk[stkpos]) + else + flexfatal( + _( "consistency check failed in epsclosure()" ) ); + } + + *ns_addr = numstates; + *hv_addr = hashval; + *nacc_addr = nacc; + + return t; + } + + +/* increase_max_dfas - increase the maximum number of DFAs */ + +void increase_max_dfas() + { + current_max_dfas += MAX_DFAS_INCREMENT; + + ++num_reallocs; + + base = reallocate_integer_array( base, current_max_dfas ); + def = reallocate_integer_array( def, current_max_dfas ); + dfasiz = reallocate_integer_array( dfasiz, current_max_dfas ); + accsiz = reallocate_integer_array( accsiz, current_max_dfas ); + dhash = reallocate_integer_array( dhash, current_max_dfas ); + dss = reallocate_int_ptr_array( dss, current_max_dfas ); + dfaacc = reallocate_dfaacc_union( dfaacc, current_max_dfas ); + + if ( nultrans ) + nultrans = + reallocate_integer_array( nultrans, current_max_dfas ); + } + + +/* ntod - convert an ndfa to a dfa + * + * Creates the dfa corresponding to the ndfa we've constructed. The + * dfa starts out in state #1. + */ + +void ntod() + { + int *accset, ds, nacc, newds; + int sym, hashval, numstates, dsize; + int num_full_table_rows; /* used only for -f */ + int *nset, *dset; + int targptr, totaltrans, i, comstate, comfreq, targ; + int symlist[CSIZE + 1]; + int num_start_states; + int todo_head, todo_next; + + /* Note that the following are indexed by *equivalence classes* + * and not by characters. Since equivalence classes are indexed + * beginning with 1, even if the scanner accepts NUL's, this + * means that (since every character is potentially in its own + * equivalence class) these arrays must have room for indices + * from 1 to CSIZE, so their size must be CSIZE + 1. + */ + int duplist[CSIZE + 1], state[CSIZE + 1]; + int targfreq[CSIZE + 1], targstate[CSIZE + 1]; + + accset = allocate_integer_array( num_rules + 1 ); + nset = allocate_integer_array( current_max_dfa_size ); + + /* The "todo" queue is represented by the head, which is the DFA + * state currently being processed, and the "next", which is the + * next DFA state number available (not in use). We depend on the + * fact that snstods() returns DFA's \in increasing order/, and thus + * need only know the bounds of the dfas to be processed. + */ + todo_head = todo_next = 0; + + for ( i = 0; i <= csize; ++i ) + { + duplist[i] = NIL; + symlist[i] = false; + } + + for ( i = 0; i <= num_rules; ++i ) + accset[i] = NIL; + + if ( trace ) + { + dumpnfa( scset[1] ); + fputs( _( "\n\nDFA Dump:\n\n" ), stderr ); + } + + inittbl(); + + /* Check to see whether we should build a separate table for + * transitions on NUL characters. We don't do this for full-speed + * (-F) scanners, since for them we don't have a simple state + * number lying around with which to index the table. We also + * don't bother doing it for scanners unless (1) NUL is in its own + * equivalence class (indicated by a positive value of + * ecgroup[NUL]), (2) NUL's equivalence class is the last + * equivalence class, and (3) the number of equivalence classes is + * the same as the number of characters. This latter case comes + * about when useecs is false or when it's true but every character + * still manages to land in its own class (unlikely, but it's + * cheap to check for). If all these things are true then the + * character code needed to represent NUL's equivalence class for + * indexing the tables is going to take one more bit than the + * number of characters, and therefore we won't be assured of + * being able to fit it into a YY_CHAR variable. This rules out + * storing the transitions in a compressed table, since the code + * for interpreting them uses a YY_CHAR variable (perhaps it + * should just use an integer, though; this is worth pondering ... + * ###). + * + * Finally, for full tables, we want the number of entries in the + * table to be a power of two so the array references go fast (it + * will just take a shift to compute the major index). If + * encoding NUL's transitions in the table will spoil this, we + * give it its own table (note that this will be the case if we're + * not using equivalence classes). + */ + + /* Note that the test for ecgroup[0] == numecs below accomplishes + * both (1) and (2) above + */ + if ( ! fullspd && ecgroup[0] == numecs ) + { + /* NUL is alone in its equivalence class, which is the + * last one. + */ + int use_NUL_table = (numecs == csize); + + if ( fulltbl && ! use_NUL_table ) + { + /* We still may want to use the table if numecs + * is a power of 2. + */ + int power_of_two; + + for ( power_of_two = 1; power_of_two <= csize; + power_of_two *= 2 ) + if ( numecs == power_of_two ) + { + use_NUL_table = true; + break; + } + } + + if ( use_NUL_table ) + nultrans = allocate_integer_array( current_max_dfas ); + + /* From now on, nultrans != nil indicates that we're + * saving null transitions for later, separate encoding. + */ + } + + + if ( fullspd ) + { + for ( i = 0; i <= numecs; ++i ) + state[i] = 0; + + place_state( state, 0, 0 ); + dfaacc[0].dfaacc_state = 0; + } + + else if ( fulltbl ) + { + if ( nultrans ) + /* We won't be including NUL's transitions in the + * table, so build it for entries from 0 .. numecs - 1. + */ + num_full_table_rows = numecs; + + else + /* Take into account the fact that we'll be including + * the NUL entries in the transition table. Build it + * from 0 .. numecs. + */ + num_full_table_rows = numecs + 1; + + /* Unless -Ca, declare it "short" because it's a real + * long-shot that that won't be large enough. + */ + out_str_dec( "static yyconst %s yy_nxt[][%d] =\n {\n", + /* '}' so vi doesn't get too confused */ + long_align ? "long" : "short", num_full_table_rows ); + + outn( " {" ); + + /* Generate 0 entries for state #0. */ + for ( i = 0; i < num_full_table_rows; ++i ) + mk2data( 0 ); + + dataflush(); + outn( " },\n" ); + } + + /* Create the first states. */ + + num_start_states = lastsc * 2; + + for ( i = 1; i <= num_start_states; ++i ) + { + numstates = 1; + + /* For each start condition, make one state for the case when + * we're at the beginning of the line (the '^' operator) and + * one for the case when we're not. + */ + if ( i % 2 == 1 ) + nset[numstates] = scset[(i / 2) + 1]; + else + nset[numstates] = + mkbranch( scbol[i / 2], scset[i / 2] ); + + nset = epsclosure( nset, &numstates, accset, &nacc, &hashval ); + + if ( snstods( nset, numstates, accset, nacc, hashval, &ds ) ) + { + numas += nacc; + totnst += numstates; + ++todo_next; + + if ( variable_trailing_context_rules && nacc > 0 ) + check_trailing_context( nset, numstates, + accset, nacc ); + } + } + + if ( ! fullspd ) + { + if ( ! snstods( nset, 0, accset, 0, 0, &end_of_buffer_state ) ) + flexfatal( + _( "could not create unique end-of-buffer state" ) ); + + ++numas; + ++num_start_states; + ++todo_next; + } + + while ( todo_head < todo_next ) + { + targptr = 0; + totaltrans = 0; + + for ( i = 1; i <= numecs; ++i ) + state[i] = 0; + + ds = ++todo_head; + + dset = dss[ds]; + dsize = dfasiz[ds]; + + if ( trace ) + fprintf( stderr, _( "state # %d:\n" ), ds ); + + sympartition( dset, dsize, symlist, duplist ); + + for ( sym = 1; sym <= numecs; ++sym ) + { + if ( symlist[sym] ) + { + symlist[sym] = 0; + + if ( duplist[sym] == NIL ) + { + /* Symbol has unique out-transitions. */ + numstates = symfollowset( dset, dsize, + sym, nset ); + nset = epsclosure( nset, &numstates, + accset, &nacc, &hashval ); + + if ( snstods( nset, numstates, accset, + nacc, hashval, &newds ) ) + { + totnst = totnst + numstates; + ++todo_next; + numas += nacc; + + if ( + variable_trailing_context_rules && + nacc > 0 ) + check_trailing_context( + nset, numstates, + accset, nacc ); + } + + state[sym] = newds; + + if ( trace ) + fprintf( stderr, "\t%d\t%d\n", + sym, newds ); + + targfreq[++targptr] = 1; + targstate[targptr] = newds; + ++numuniq; + } + + else + { + /* sym's equivalence class has the same + * transitions as duplist(sym)'s + * equivalence class. + */ + targ = state[duplist[sym]]; + state[sym] = targ; + + if ( trace ) + fprintf( stderr, "\t%d\t%d\n", + sym, targ ); + + /* Update frequency count for + * destination state. + */ + + i = 0; + while ( targstate[++i] != targ ) + ; + + ++targfreq[i]; + ++numdup; + } + + ++totaltrans; + duplist[sym] = NIL; + } + } + + if ( caseins && ! useecs ) + { + int j; + + for ( i = 'A', j = 'a'; i <= 'Z'; ++i, ++j ) + { + if ( state[i] == 0 && state[j] != 0 ) + /* We're adding a transition. */ + ++totaltrans; + + else if ( state[i] != 0 && state[j] == 0 ) + /* We're taking away a transition. */ + --totaltrans; + + state[i] = state[j]; + } + } + + numsnpairs += totaltrans; + + if ( ds > num_start_states ) + check_for_backing_up( ds, state ); + + if ( nultrans ) + { + nultrans[ds] = state[NUL_ec]; + state[NUL_ec] = 0; /* remove transition */ + } + + if ( fulltbl ) + { + outn( " {" ); + + /* Supply array's 0-element. */ + if ( ds == end_of_buffer_state ) + mk2data( -end_of_buffer_state ); + else + mk2data( end_of_buffer_state ); + + for ( i = 1; i < num_full_table_rows; ++i ) + /* Jams are marked by negative of state + * number. + */ + mk2data( state[i] ? state[i] : -ds ); + + dataflush(); + outn( " },\n" ); + } + + else if ( fullspd ) + place_state( state, ds, totaltrans ); + + else if ( ds == end_of_buffer_state ) + /* Special case this state to make sure it does what + * it's supposed to, i.e., jam on end-of-buffer. + */ + stack1( ds, 0, 0, JAMSTATE ); + + else /* normal, compressed state */ + { + /* Determine which destination state is the most + * common, and how many transitions to it there are. + */ + + comfreq = 0; + comstate = 0; + + for ( i = 1; i <= targptr; ++i ) + if ( targfreq[i] > comfreq ) + { + comfreq = targfreq[i]; + comstate = targstate[i]; + } + + bldtbl( state, ds, totaltrans, comstate, comfreq ); + } + } + + if ( fulltbl ) + dataend(); + + else if ( ! fullspd ) + { + cmptmps(); /* create compressed template entries */ + + /* Create tables for all the states with only one + * out-transition. + */ + while ( onesp > 0 ) + { + mk1tbl( onestate[onesp], onesym[onesp], onenext[onesp], + onedef[onesp] ); + --onesp; + } + + mkdeftbl(); + } + + flex_free( (void *) accset ); + flex_free( (void *) nset ); + } + + +/* snstods - converts a set of ndfa states into a dfa state + * + * synopsis + * is_new_state = snstods( int sns[numstates], int numstates, + * int accset[num_rules+1], int nacc, + * int hashval, int *newds_addr ); + * + * On return, the dfa state number is in newds. + */ + +int snstods( sns, numstates, accset, nacc, hashval, newds_addr ) +int sns[], numstates, accset[], nacc, hashval, *newds_addr; + { + int didsort = 0; + int i, j; + int newds, *oldsns; + + for ( i = 1; i <= lastdfa; ++i ) + if ( hashval == dhash[i] ) + { + if ( numstates == dfasiz[i] ) + { + oldsns = dss[i]; + + if ( ! didsort ) + { + /* We sort the states in sns so we + * can compare it to oldsns quickly. + * We use bubble because there probably + * aren't very many states. + */ + bubble( sns, numstates ); + didsort = 1; + } + + for ( j = 1; j <= numstates; ++j ) + if ( sns[j] != oldsns[j] ) + break; + + if ( j > numstates ) + { + ++dfaeql; + *newds_addr = i; + return 0; + } + + ++hshcol; + } + + else + ++hshsave; + } + + /* Make a new dfa. */ + + if ( ++lastdfa >= current_max_dfas ) + increase_max_dfas(); + + newds = lastdfa; + + dss[newds] = allocate_integer_array( numstates + 1 ); + + /* If we haven't already sorted the states in sns, we do so now, + * so that future comparisons with it can be made quickly. + */ + + if ( ! didsort ) + bubble( sns, numstates ); + + for ( i = 1; i <= numstates; ++i ) + dss[newds][i] = sns[i]; + + dfasiz[newds] = numstates; + dhash[newds] = hashval; + + if ( nacc == 0 ) + { + if ( reject ) + dfaacc[newds].dfaacc_set = (int *) 0; + else + dfaacc[newds].dfaacc_state = 0; + + accsiz[newds] = 0; + } + + else if ( reject ) + { + /* We sort the accepting set in increasing order so the + * disambiguating rule that the first rule listed is considered + * match in the event of ties will work. We use a bubble + * sort since the list is probably quite small. + */ + + bubble( accset, nacc ); + + dfaacc[newds].dfaacc_set = allocate_integer_array( nacc + 1 ); + + /* Save the accepting set for later */ + for ( i = 1; i <= nacc; ++i ) + { + dfaacc[newds].dfaacc_set[i] = accset[i]; + + if ( accset[i] <= num_rules ) + /* Who knows, perhaps a REJECT can yield + * this rule. + */ + rule_useful[accset[i]] = true; + } + + accsiz[newds] = nacc; + } + + else + { + /* Find lowest numbered rule so the disambiguating rule + * will work. + */ + j = num_rules + 1; + + for ( i = 1; i <= nacc; ++i ) + if ( accset[i] < j ) + j = accset[i]; + + dfaacc[newds].dfaacc_state = j; + + if ( j <= num_rules ) + rule_useful[j] = true; + } + + *newds_addr = newds; + + return 1; + } + + +/* symfollowset - follow the symbol transitions one step + * + * synopsis + * numstates = symfollowset( int ds[current_max_dfa_size], int dsize, + * int transsym, int nset[current_max_dfa_size] ); + */ + +int symfollowset( ds, dsize, transsym, nset ) +int ds[], dsize, transsym, nset[]; + { + int ns, tsp, sym, i, j, lenccl, ch, numstates, ccllist; + + numstates = 0; + + for ( i = 1; i <= dsize; ++i ) + { /* for each nfa state ns in the state set of ds */ + ns = ds[i]; + sym = transchar[ns]; + tsp = trans1[ns]; + + if ( sym < 0 ) + { /* it's a character class */ + sym = -sym; + ccllist = cclmap[sym]; + lenccl = ccllen[sym]; + + if ( cclng[sym] ) + { + for ( j = 0; j < lenccl; ++j ) + { + /* Loop through negated character + * class. + */ + ch = ccltbl[ccllist + j]; + + if ( ch == 0 ) + ch = NUL_ec; + + if ( ch > transsym ) + /* Transsym isn't in negated + * ccl. + */ + break; + + else if ( ch == transsym ) + /* next 2 */ goto bottom; + } + + /* Didn't find transsym in ccl. */ + nset[++numstates] = tsp; + } + + else + for ( j = 0; j < lenccl; ++j ) + { + ch = ccltbl[ccllist + j]; + + if ( ch == 0 ) + ch = NUL_ec; + + if ( ch > transsym ) + break; + else if ( ch == transsym ) + { + nset[++numstates] = tsp; + break; + } + } + } + + else if ( sym >= 'A' && sym <= 'Z' && caseins ) + flexfatal( + _( "consistency check failed in symfollowset" ) ); + + else if ( sym == SYM_EPSILON ) + { /* do nothing */ + } + + else if ( ABS( ecgroup[sym] ) == transsym ) + nset[++numstates] = tsp; + + bottom: ; + } + + return numstates; + } + + +/* sympartition - partition characters with same out-transitions + * + * synopsis + * sympartition( int ds[current_max_dfa_size], int numstates, + * int symlist[numecs], int duplist[numecs] ); + */ + +void sympartition( ds, numstates, symlist, duplist ) +int ds[], numstates; +int symlist[], duplist[]; + { + int tch, i, j, k, ns, dupfwd[CSIZE + 1], lenccl, cclp, ich; + + /* Partitioning is done by creating equivalence classes for those + * characters which have out-transitions from the given state. Thus + * we are really creating equivalence classes of equivalence classes. + */ + + for ( i = 1; i <= numecs; ++i ) + { /* initialize equivalence class list */ + duplist[i] = i - 1; + dupfwd[i] = i + 1; + } + + duplist[1] = NIL; + dupfwd[numecs] = NIL; + + for ( i = 1; i <= numstates; ++i ) + { + ns = ds[i]; + tch = transchar[ns]; + + if ( tch != SYM_EPSILON ) + { + if ( tch < -lastccl || tch >= csize ) + { + flexfatal( + _( "bad transition character detected in sympartition()" ) ); + } + + if ( tch >= 0 ) + { /* character transition */ + int ec = ecgroup[tch]; + + mkechar( ec, dupfwd, duplist ); + symlist[ec] = 1; + } + + else + { /* character class */ + tch = -tch; + + lenccl = ccllen[tch]; + cclp = cclmap[tch]; + mkeccl( ccltbl + cclp, lenccl, dupfwd, + duplist, numecs, NUL_ec ); + + if ( cclng[tch] ) + { + j = 0; + + for ( k = 0; k < lenccl; ++k ) + { + ich = ccltbl[cclp + k]; + + if ( ich == 0 ) + ich = NUL_ec; + + for ( ++j; j < ich; ++j ) + symlist[j] = 1; + } + + for ( ++j; j <= numecs; ++j ) + symlist[j] = 1; + } + + else + for ( k = 0; k < lenccl; ++k ) + { + ich = ccltbl[cclp + k]; + + if ( ich == 0 ) + ich = NUL_ec; + + symlist[ich] = 1; + } + } + } + } + } diff --git a/usr.bin/lex/ecs.c b/usr.bin/lex/ecs.c new file mode 100644 index 0000000..4f7748d --- /dev/null +++ b/usr.bin/lex/ecs.c @@ -0,0 +1,227 @@ +/* ecs - equivalence class routines */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/ecs.c,v 2.9 93/12/07 10:18:20 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + +/* ccl2ecl - convert character classes to set of equivalence classes */ + +void ccl2ecl() + { + int i, ich, newlen, cclp, ccls, cclmec; + + for ( i = 1; i <= lastccl; ++i ) + { + /* We loop through each character class, and for each character + * in the class, add the character's equivalence class to the + * new "character" class we are creating. Thus when we are all + * done, character classes will really consist of collections + * of equivalence classes + */ + + newlen = 0; + cclp = cclmap[i]; + + for ( ccls = 0; ccls < ccllen[i]; ++ccls ) + { + ich = ccltbl[cclp + ccls]; + cclmec = ecgroup[ich]; + + if ( cclmec > 0 ) + { + ccltbl[cclp + newlen] = cclmec; + ++newlen; + } + } + + ccllen[i] = newlen; + } + } + + +/* cre8ecs - associate equivalence class numbers with class members + * + * fwd is the forward linked-list of equivalence class members. bck + * is the backward linked-list, and num is the number of class members. + * + * Returned is the number of classes. + */ + +int cre8ecs( fwd, bck, num ) +int fwd[], bck[], num; + { + int i, j, numcl; + + numcl = 0; + + /* Create equivalence class numbers. From now on, ABS( bck(x) ) + * is the equivalence class number for object x. If bck(x) + * is positive, then x is the representative of its equivalence + * class. + */ + for ( i = 1; i <= num; ++i ) + if ( bck[i] == NIL ) + { + bck[i] = ++numcl; + for ( j = fwd[i]; j != NIL; j = fwd[j] ) + bck[j] = -numcl; + } + + return numcl; + } + + +/* mkeccl - update equivalence classes based on character class xtions + * + * synopsis + * Char ccls[]; + * int lenccl, fwd[llsiz], bck[llsiz], llsiz, NUL_mapping; + * void mkeccl( Char ccls[], int lenccl, int fwd[llsiz], int bck[llsiz], + * int llsiz, int NUL_mapping ); + * + * ccls contains the elements of the character class, lenccl is the + * number of elements in the ccl, fwd is the forward link-list of equivalent + * characters, bck is the backward link-list, and llsiz size of the link-list. + * + * NUL_mapping is the value which NUL (0) should be mapped to. + */ + +void mkeccl( ccls, lenccl, fwd, bck, llsiz, NUL_mapping ) +Char ccls[]; +int lenccl, fwd[], bck[], llsiz, NUL_mapping; + { + int cclp, oldec, newec; + int cclm, i, j; + static unsigned char cclflags[CSIZE]; /* initialized to all '\0' */ + + /* Note that it doesn't matter whether or not the character class is + * negated. The same results will be obtained in either case. + */ + + cclp = 0; + + while ( cclp < lenccl ) + { + cclm = ccls[cclp]; + + if ( NUL_mapping && cclm == 0 ) + cclm = NUL_mapping; + + oldec = bck[cclm]; + newec = cclm; + + j = cclp + 1; + + for ( i = fwd[cclm]; i != NIL && i <= llsiz; i = fwd[i] ) + { /* look for the symbol in the character class */ + for ( ; j < lenccl; ++j ) + { + int ccl_char; + + if ( NUL_mapping && ccls[j] == 0 ) + ccl_char = NUL_mapping; + else + ccl_char = ccls[j]; + + if ( ccl_char > i ) + break; + + if ( ccl_char == i && ! cclflags[j] ) + { + /* We found an old companion of cclm + * in the ccl. Link it into the new + * equivalence class and flag it as + * having been processed. + */ + + bck[i] = newec; + fwd[newec] = i; + newec = i; + /* Set flag so we don't reprocess. */ + cclflags[j] = 1; + + /* Get next equivalence class member. */ + /* continue 2 */ + goto next_pt; + } + } + + /* Symbol isn't in character class. Put it in the old + * equivalence class. + */ + + bck[i] = oldec; + + if ( oldec != NIL ) + fwd[oldec] = i; + + oldec = i; + + next_pt: ; + } + + if ( bck[cclm] != NIL || oldec != bck[cclm] ) + { + bck[cclm] = NIL; + fwd[oldec] = NIL; + } + + fwd[newec] = NIL; + + /* Find next ccl member to process. */ + + for ( ++cclp; cclflags[cclp] && cclp < lenccl; ++cclp ) + { + /* Reset "doesn't need processing" flag. */ + cclflags[cclp] = 0; + } + } + } + + +/* mkechar - create equivalence class for single character */ + +void mkechar( tch, fwd, bck ) +int tch, fwd[], bck[]; + { + /* If until now the character has been a proper subset of + * an equivalence class, break it away to create a new ec + */ + + if ( fwd[tch] != NIL ) + bck[fwd[tch]] = bck[tch]; + + if ( bck[tch] != NIL ) + fwd[bck[tch]] = fwd[tch]; + + fwd[tch] = NIL; + bck[tch] = NIL; + } diff --git a/usr.bin/lex/flex.skl b/usr.bin/lex/flex.skl new file mode 100644 index 0000000..ef5ee14 --- /dev/null +++ b/usr.bin/lex/flex.skl @@ -0,0 +1,1574 @@ +/* A lexical scanner generated by flex */ + +/* Scanner skeleton version: + * $Header: /home/daffy/u0/vern/flex/RCS/flex.skl,v 2.91 96/09/10 16:58:48 vern Exp $ + * $FreeBSD$ + */ + +#if defined(__FreeBSD__) +#include <sys/cdefs.h> +#else +#define __unused +#define __dead2 +#endif + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 5 + +%- +#include <stdio.h> +%* + + +/* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */ +#ifdef c_plusplus +#ifndef __cplusplus +#define __cplusplus +#endif +#endif + + +#ifdef __cplusplus + +#include <stdlib.h> +%+ +#include <iosfwd> +using namespace std; +%* +#include <unistd.h> + +/* Use prototypes in function declarations. */ +#define YY_USE_PROTOS + +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST + +#else /* ! __cplusplus */ + +#if __STDC__ + +#define YY_USE_PROTOS +#define YY_USE_CONST + +#endif /* __STDC__ */ +#endif /* ! __cplusplus */ + +#ifdef __TURBOC__ + #pragma warn -rch + #pragma warn -use +#include <io.h> +#include <stdlib.h> +#define YY_USE_CONST +#define YY_USE_PROTOS +#endif + +#ifdef YY_USE_CONST +#define yyconst const +#else +#define yyconst +#endif + + +#ifdef YY_USE_PROTOS +#define YY_PROTO(proto) proto +#else +#define YY_PROTO(proto) () +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. + */ +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN yy_start = 1 + 2 * + +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START ((yy_start - 1) / 2) +#define YYSTATE YY_START + +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) + +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#define YY_BUF_SIZE 16384 + +typedef struct yy_buffer_state *YY_BUFFER_STATE; + +extern int yyleng; +%- +extern FILE *yyin, *yyout; +%* + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + +/* The funky do-while in the following #define is used to turn the definition + * into a single C statement (which needs a semi-colon terminator). This + * avoids problems with code like: + * + * if ( condition_holds ) + * yyless( 5 ); + * else + * do_something_else(); + * + * Prior to using the do-while the compiler would get upset at the + * "else" because it interpreted the "if" statement as being all + * done when it reached the ';' after the yyless() call. + */ + +/* Return all but the first 'n' matched characters back to the input stream. */ + +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + *yy_cp = yy_hold_char; \ + YY_RESTORE_YY_MORE_OFFSET \ + yy_c_buf_p = yy_cp = yy_bp + (n) - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) + +#define unput(c) yyunput( (c), yytext_ptr ) + +/* The following is because we cannot portably get our hands on size_t + * (without autoconf's help, which isn't available because we want + * flex-generated scanners to compile on their own). + */ +typedef unsigned int yy_size_t; + + +struct yy_buffer_state + { +%- + FILE *yy_input_file; +%+ + istream* yy_input_file; +%* + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + yy_size_t yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + }; + +%- Standard (non-C++) definition +static YY_BUFFER_STATE yy_current_buffer = 0; +%* + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + */ +#define YY_CURRENT_BUFFER yy_current_buffer + + +%- Standard (non-C++) definition +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; + +static int yy_n_chars; /* number of characters read into yy_ch_buf */ + + +int yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = (char *) 0; +static int yy_init = 1; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart YY_PROTO(( FILE *input_file )); + +void yy_switch_to_buffer YY_PROTO(( YY_BUFFER_STATE new_buffer )); +void yy_load_buffer_state YY_PROTO(( void )); +YY_BUFFER_STATE yy_create_buffer YY_PROTO(( FILE *file, int size )); +void yy_delete_buffer YY_PROTO(( YY_BUFFER_STATE b )); +void yy_init_buffer YY_PROTO(( YY_BUFFER_STATE b, FILE *file )); +void yy_flush_buffer YY_PROTO(( YY_BUFFER_STATE b )); +#define YY_FLUSH_BUFFER yy_flush_buffer( yy_current_buffer ) + +YY_BUFFER_STATE yy_scan_buffer YY_PROTO(( char *base, yy_size_t size )); +YY_BUFFER_STATE yy_scan_string YY_PROTO(( yyconst char *yy_str )); +YY_BUFFER_STATE yy_scan_bytes YY_PROTO(( yyconst char *bytes, int len )); +%* + +static void *yy_flex_alloc YY_PROTO(( yy_size_t )); +static void *yy_flex_realloc YY_PROTO(( void *, yy_size_t )) __unused; +static void yy_flex_free YY_PROTO(( void * )); + +#define yy_new_buffer yy_create_buffer + +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! yy_current_buffer ) \ + yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ + yy_current_buffer->yy_is_interactive = is_interactive; \ + } + +#define yy_set_bol(at_bol) \ + { \ + if ( ! yy_current_buffer ) \ + yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ + yy_current_buffer->yy_at_bol = at_bol; \ + } + +#define YY_AT_BOL() (yy_current_buffer->yy_at_bol) + +%% yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here + +%- Standard (non-C++) definition +static yy_state_type yy_get_previous_state YY_PROTO(( void )); +static yy_state_type yy_try_NUL_trans YY_PROTO(( yy_state_type current_state )); +static int yy_get_next_buffer YY_PROTO(( void )); +static void yy_fatal_error YY_PROTO(( yyconst char msg[] )) __dead2; +%* + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + yytext_ptr = yy_bp; \ +%% code to fiddle yytext and yyleng for yymore() goes here + yy_hold_char = *yy_cp; \ + *yy_cp = '\0'; \ +%% code to copy yytext_ptr to yytext[] goes here, if %array + yy_c_buf_p = yy_cp; + +%% data tables for the DFA and the user's section 1 definitions go here + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap YY_PROTO(( void )); +#else +extern int yywrap YY_PROTO(( void )); +#endif +#endif + +%- +#ifndef YY_NO_UNPUT +static void yyunput YY_PROTO(( int c, char *buf_ptr )); +#endif +%* + +#ifndef yytext_ptr +static void yy_flex_strncpy YY_PROTO(( char *, yyconst char *, int )); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen YY_PROTO(( yyconst char * )); +#endif + +#ifndef YY_NO_INPUT +%- Standard (non-C++) definition +#ifdef __cplusplus +static int yyinput YY_PROTO(( void )); +#else +static int input YY_PROTO(( void )); +#endif +%* +#endif + +#if YY_STACK_USED +static int yy_start_stack_ptr = 0; +static int yy_start_stack_depth = 0; +static int *yy_start_stack = 0; +#ifndef YY_NO_PUSH_STATE +static void yy_push_state YY_PROTO(( int new_state )); +#endif +#ifndef YY_NO_POP_STATE +static void yy_pop_state YY_PROTO(( void )); +#endif +#ifndef YY_NO_TOP_STATE +static int yy_top_state YY_PROTO(( void )); +#endif + +#else +#define YY_NO_PUSH_STATE 1 +#define YY_NO_POP_STATE 1 +#define YY_NO_TOP_STATE 1 +#endif + +#ifdef YY_MALLOC_DECL +YY_MALLOC_DECL +#else +#if __STDC__ +#ifndef __cplusplus +#include <stdlib.h> +#endif +#else +/* Just try to get by without declaring the routines. This will fail + * miserably on non-ANSI systems for which sizeof(size_t) != sizeof(int) + * or sizeof(void*) != sizeof(int). + */ +#endif +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#define YY_READ_BUF_SIZE 8192 +#endif + +/* Copy whatever the last rule matched to the standard output. */ + +#ifndef ECHO +%- Standard (non-C++) definition +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO (void) fwrite( yytext, yyleng, 1, yyout ) +%+ C++ definition +#define ECHO LexerOutput( yytext, yyleng ) +%* +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ +%% fread()/read() definition of YY_INPUT goes here unless we're doing C++ +%+ C++ definition + if ( (result = LexerInput( (char *) buf, max_size )) < 0 ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); +%* +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +%- +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +%+ +#define YY_FATAL_ERROR(msg) LexerError( msg ) +%* +#endif + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +%- Standard (non-C++) definition +#define YY_DECL int yylex YY_PROTO(( void )) +%+ C++ definition +#define YY_DECL int yyFlexLexer::yylex() +%* +#endif + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK break; +#endif + +%% YY_RULE_SETUP definition goes here + +YY_DECL + { + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + +%% user's declarations go here + + if ( yy_init ) + { + yy_init = 0; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! yy_start ) + yy_start = 1; /* first start state */ + + if ( ! yyin ) +%- + yyin = stdin; +%+ + yyin = &cin; +%* + + if ( ! yyout ) +%- + yyout = stdout; +%+ + yyout = &cout; +%* + + if ( ! yy_current_buffer ) + yy_current_buffer = + yy_create_buffer( yyin, YY_BUF_SIZE ); + + yy_load_buffer_state(); + } + + while ( 1 ) /* loops until end-of-file is reached */ + { +%% yymore()-related code goes here + yy_cp = yy_c_buf_p; + + /* Support of yytext. */ + *yy_cp = yy_hold_char; + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + +%% code to set up and find next match goes here + +yy_find_action: +%% code to find the action number goes here + + YY_DO_BEFORE_ACTION; + +%% code for yylineno update goes here + +do_action: /* This label is used only to access EOF actions. */ + +%% debug code goes here + + switch ( yy_act ) + { /* beginning of action switch */ +%% actions go here + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - yytext_ptr) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = yy_hold_char; + YY_RESTORE_YY_MORE_OFFSET + + if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between yy_current_buffer and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + yy_n_chars = yy_current_buffer->yy_n_chars; + yy_current_buffer->yy_input_file = yyin; + yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = yytext_ptr + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++yy_c_buf_p; + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { +%% code to do back-up for compressed tables and set up yy_cp goes here + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer() ) + { + case EOB_ACT_END_OF_FILE: + { + yy_did_buffer_switch_on_eof = 0; + + if ( yywrap() ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! yy_did_buffer_switch_on_eof ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = + yytext_ptr + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + yy_c_buf_p = + &yy_current_buffer->yy_ch_buf[yy_n_chars]; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of yylex */ + +%+ +yyFlexLexer::yyFlexLexer( istream* arg_yyin, ostream* arg_yyout ) + { + yyin = arg_yyin; + yyout = arg_yyout; + yy_c_buf_p = 0; + yy_init = 1; + yy_start = 0; + yy_flex_debug = 0; + yylineno = 1; // this will only get updated if %option yylineno + + yy_did_buffer_switch_on_eof = 0; + + yy_looking_for_trail_begin = 0; + yy_more_flag = 0; + yy_more_len = 0; + yy_more_offset = yy_prev_more_offset = 0; + + yy_start_stack_ptr = yy_start_stack_depth = 0; + yy_start_stack = 0; + + yy_current_buffer = 0; + +#ifdef YY_USES_REJECT + yy_state_buf = new yy_state_type[YY_BUF_SIZE + 2]; +#else + yy_state_buf = 0; +#endif + } + +yyFlexLexer::~yyFlexLexer() + { + delete yy_state_buf; + yy_delete_buffer( yy_current_buffer ); + } + +void yyFlexLexer::switch_streams( istream* new_in, ostream* new_out ) + { + if ( new_in ) + { + yy_delete_buffer( yy_current_buffer ); + yy_switch_to_buffer( yy_create_buffer( new_in, YY_BUF_SIZE ) ); + } + + if ( new_out ) + yyout = new_out; + } + +#ifdef YY_INTERACTIVE +int yyFlexLexer::LexerInput( char* buf, int /* max_size */ ) +#else +int yyFlexLexer::LexerInput( char* buf, int max_size ) +#endif + { + if ( yyin->eof() || yyin->fail() ) + return 0; + +#ifdef YY_INTERACTIVE + yyin->get( buf[0] ); + + if ( yyin->eof() ) + return 0; + + if ( yyin->bad() ) + return -1; + + return 1; + +#else + (void) yyin->read( buf, max_size ); + + if ( yyin->bad() ) + return -1; + else + return yyin->gcount(); +#endif + } + +void yyFlexLexer::LexerOutput( const char* buf, int size ) + { + (void) yyout->write( buf, size ); + } +%* + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ + +%- +#ifdef YY_USE_PROTOS +static int yy_get_next_buffer(void) +#else +static int yy_get_next_buffer() +#endif +%+ +int yyFlexLexer::yy_get_next_buffer() +%* + { + char *dest = yy_current_buffer->yy_ch_buf; + char *source = yytext_ptr; + int number_to_move, i; + int ret_val; + + if ( yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( yy_current_buffer->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) (yy_c_buf_p - yytext_ptr) - 1; + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + yy_current_buffer->yy_n_chars = yy_n_chars = 0; + + else + { + int num_to_read = + yy_current_buffer->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ +#ifdef YY_USES_REJECT + YY_FATAL_ERROR( +"input buffer overflow, can't enlarge buffer because scanner uses REJECT" ); +#else + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = yy_current_buffer; + + int yy_c_buf_p_offset = + (int) (yy_c_buf_p - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + int new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yy_flex_realloc( (void *) b->yy_ch_buf, + b->yy_buf_size + 2 ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = 0; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = yy_current_buffer->yy_buf_size - + number_to_move - 1; +#endif + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&yy_current_buffer->yy_ch_buf[number_to_move]), + yy_n_chars, num_to_read ); + + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + if ( yy_n_chars == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + yy_current_buffer->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + yy_n_chars += number_to_move; + yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; + yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; + + yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + + return ret_val; + } + + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + +%- +#ifdef YY_USE_PROTOS +static yy_state_type yy_get_previous_state(void) +#else +static yy_state_type yy_get_previous_state() +#endif +%+ +yy_state_type yyFlexLexer::yy_get_previous_state() +%* + { + yy_state_type yy_current_state; + char *yy_cp; + +%% code to get the start state into yy_current_state goes here + + for ( yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp ) + { +%% code to find the next state goes here + } + + return yy_current_state; + } + + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + +%- +#ifdef YY_USE_PROTOS +static yy_state_type yy_try_NUL_trans( yy_state_type yy_current_state ) +#else +static yy_state_type yy_try_NUL_trans( yy_current_state ) +yy_state_type yy_current_state; +#endif +%+ +yy_state_type yyFlexLexer::yy_try_NUL_trans( yy_state_type yy_current_state ) +%* + { + int yy_is_jam; +%% code to find the next state, and perhaps do backing up, goes here + + return yy_is_jam ? 0 : yy_current_state; + } + + +%- +#ifndef YY_NO_UNPUT +#ifdef YY_USE_PROTOS +static void yyunput( int c, char *yy_bp ) +#else +static void yyunput( c, yy_bp ) +int c; +char *yy_bp; +#endif +%+ +void yyFlexLexer::yyunput( int c, char* yy_bp ) +%* + { + char *yy_cp = yy_c_buf_p; + + /* undo effects of setting up yytext */ + *yy_cp = yy_hold_char; + + if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) + { /* need to shift things up to make room */ + /* +2 for EOB chars. */ + int number_to_move = yy_n_chars + 2; + char *dest = &yy_current_buffer->yy_ch_buf[ + yy_current_buffer->yy_buf_size + 2]; + char *source = + &yy_current_buffer->yy_ch_buf[number_to_move]; + + while ( source > yy_current_buffer->yy_ch_buf ) + *--dest = *--source; + + yy_cp += (int) (dest - source); + yy_bp += (int) (dest - source); + yy_current_buffer->yy_n_chars = + yy_n_chars = yy_current_buffer->yy_buf_size; + + if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) + YY_FATAL_ERROR( "flex scanner push-back overflow" ); + } + + *--yy_cp = (char) c; + +%% update yylineno here + + yytext_ptr = yy_bp; + yy_hold_char = *yy_cp; + yy_c_buf_p = yy_cp; + } +%- +#endif /* ifndef YY_NO_UNPUT */ +%* + + +%- +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput() +#else +#ifdef YY_USE_PROTOS +static int input(void) +#else +static int input() +#endif +#endif +%+ +int yyFlexLexer::yyinput() +%* + { + int c; + + *yy_c_buf_p = yy_hold_char; + + if ( *yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars] ) + /* This was really a NUL. */ + *yy_c_buf_p = '\0'; + + else + { /* need more input */ + int offset = yy_c_buf_p - yytext_ptr; + ++yy_c_buf_p; + + switch ( yy_get_next_buffer() ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /* fall through */ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap() ) + return EOF; + + if ( ! yy_did_buffer_switch_on_eof ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext_ptr + offset; + break; + } + } + } + + c = *(unsigned char *) yy_c_buf_p; /* cast for 8-bit char's */ + *yy_c_buf_p = '\0'; /* preserve yytext */ + yy_hold_char = *++yy_c_buf_p; + +%% update BOL and yylineno + + return c; + } +%- +#endif /* ifndef YY_NO_INPUT */ +%* + + +%- +#ifdef YY_USE_PROTOS +void yyrestart( FILE *input_file ) +#else +void yyrestart( input_file ) +FILE *input_file; +#endif +%+ +void yyFlexLexer::yyrestart( istream* input_file ) +%* + { + if ( ! yy_current_buffer ) + yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); + + yy_init_buffer( yy_current_buffer, input_file ); + yy_load_buffer_state(); + } + + +%- +#ifdef YY_USE_PROTOS +void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +#else +void yy_switch_to_buffer( new_buffer ) +YY_BUFFER_STATE new_buffer; +#endif +%+ +void yyFlexLexer::yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +%* + { + if ( yy_current_buffer == new_buffer ) + return; + + if ( yy_current_buffer ) + { + /* Flush out information for old buffer. */ + *yy_c_buf_p = yy_hold_char; + yy_current_buffer->yy_buf_pos = yy_c_buf_p; + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + yy_current_buffer = new_buffer; + yy_load_buffer_state(); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yy_did_buffer_switch_on_eof = 1; + } + + +%- +#ifdef YY_USE_PROTOS +void yy_load_buffer_state( void ) +#else +void yy_load_buffer_state() +#endif +%+ +void yyFlexLexer::yy_load_buffer_state() +%* + { + yy_n_chars = yy_current_buffer->yy_n_chars; + yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; + yyin = yy_current_buffer->yy_input_file; + yy_hold_char = *yy_c_buf_p; + } + + +%- +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_create_buffer( FILE *file, int size ) +#else +YY_BUFFER_STATE yy_create_buffer( file, size ) +FILE *file; +int size; +#endif +%+ +YY_BUFFER_STATE yyFlexLexer::yy_create_buffer( istream* file, int size ) +%* + { + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yy_flex_alloc( b->yy_buf_size + 2 ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; + } + + +%- +#ifdef YY_USE_PROTOS +void yy_delete_buffer( YY_BUFFER_STATE b ) +#else +void yy_delete_buffer( b ) +YY_BUFFER_STATE b; +#endif +%+ +void yyFlexLexer::yy_delete_buffer( YY_BUFFER_STATE b ) +%* + { + if ( ! b ) + return; + + if ( b == yy_current_buffer ) + yy_current_buffer = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yy_flex_free( (void *) b->yy_ch_buf ); + + yy_flex_free( (void *) b ); + } + + +%- +#ifndef YY_ALWAYS_INTERACTIVE +#ifndef YY_NEVER_INTERACTIVE +extern int isatty YY_PROTO(( int )); +#endif +#endif + +#ifdef YY_USE_PROTOS +void yy_init_buffer( YY_BUFFER_STATE b, FILE *file ) +#else +void yy_init_buffer( b, file ) +YY_BUFFER_STATE b; +FILE *file; +#endif + +%+ +extern "C" int isatty YY_PROTO(( int )); +void yyFlexLexer::yy_init_buffer( YY_BUFFER_STATE b, istream* file ) +%* + + { + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + +%- +#if YY_ALWAYS_INTERACTIVE + b->yy_is_interactive = 1; +#else +#if YY_NEVER_INTERACTIVE + b->yy_is_interactive = 0; +#else + b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; +#endif +#endif +%+ + b->yy_is_interactive = 0; +%* + } + + +%- +#ifdef YY_USE_PROTOS +void yy_flush_buffer( YY_BUFFER_STATE b ) +#else +void yy_flush_buffer( b ) +YY_BUFFER_STATE b; +#endif + +%+ +void yyFlexLexer::yy_flush_buffer( YY_BUFFER_STATE b ) +%* + { + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == yy_current_buffer ) + yy_load_buffer_state(); + } +%* + + +#ifndef YY_NO_SCAN_BUFFER +%- +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_scan_buffer( char *base, yy_size_t size ) +#else +YY_BUFFER_STATE yy_scan_buffer( base, size ) +char *base; +yy_size_t size; +#endif + { + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return 0; + + b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = 0; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; + } +%* +#endif + + +#ifndef YY_NO_SCAN_STRING +%- +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_scan_string( yyconst char *yy_str ) +#else +YY_BUFFER_STATE yy_scan_string( yy_str ) +yyconst char *yy_str; +#endif + { + int len; + for ( len = 0; yy_str[len]; ++len ) + ; + + return yy_scan_bytes( yy_str, len ); + } +%* +#endif + + +#ifndef YY_NO_SCAN_BYTES +%- +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_scan_bytes( yyconst char *bytes, int len ) +#else +YY_BUFFER_STATE yy_scan_bytes( bytes, len ) +yyconst char *bytes; +int len; +#endif + { + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + int i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = len + 2; + buf = (char *) yy_flex_alloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < len; ++i ) + buf[i] = bytes[i]; + + buf[len] = buf[len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; + } +%* +#endif + + +#ifndef YY_NO_PUSH_STATE +%- +#ifdef YY_USE_PROTOS +static void yy_push_state( int new_state ) +#else +static void yy_push_state( new_state ) +int new_state; +#endif +%+ +void yyFlexLexer::yy_push_state( int new_state ) +%* + { + if ( yy_start_stack_ptr >= yy_start_stack_depth ) + { + yy_size_t new_size; + + yy_start_stack_depth += YY_START_STACK_INCR; + new_size = yy_start_stack_depth * sizeof( int ); + + if ( ! yy_start_stack ) + yy_start_stack = (int *) yy_flex_alloc( new_size ); + + else + yy_start_stack = (int *) yy_flex_realloc( + (void *) yy_start_stack, new_size ); + + if ( ! yy_start_stack ) + YY_FATAL_ERROR( + "out of memory expanding start-condition stack" ); + } + + yy_start_stack[yy_start_stack_ptr++] = YY_START; + + BEGIN(new_state); + } +#endif + + +#ifndef YY_NO_POP_STATE +%- +#ifdef YY_USE_PROTOS +static void yy_pop_state(void) +#else +static void yy_pop_state() +#endif +%+ +void yyFlexLexer::yy_pop_state() +%* + { + if ( --yy_start_stack_ptr < 0 ) + YY_FATAL_ERROR( "start-condition stack underflow" ); + + BEGIN(yy_start_stack[yy_start_stack_ptr]); + } +#endif + + +#ifndef YY_NO_TOP_STATE +%- +#ifdef YY_USE_PROTOS +static int yy_top_state(void) +#else +static int yy_top_state() +#endif +%+ +int yyFlexLexer::yy_top_state() +%* + { + return yy_start_stack[yy_start_stack_ptr - 1]; + } +#endif + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +%- +#ifdef YY_USE_PROTOS +static void yy_fatal_error( yyconst char msg[] ) +#else +static void yy_fatal_error( msg ) +char msg[]; +#endif + { + (void) fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); + } + +%+ + +void yyFlexLexer::LexerError( yyconst char msg[] ) + { + cerr << msg << '\n'; + exit( YY_EXIT_FAILURE ); + } +%* + + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yytext[yyleng] = yy_hold_char; \ + yy_c_buf_p = yytext + n; \ + yy_hold_char = *yy_c_buf_p; \ + *yy_c_buf_p = '\0'; \ + yyleng = n; \ + } \ + while ( 0 ) + + +/* Internal utility routines. */ + +#ifndef yytext_ptr +#ifdef YY_USE_PROTOS +static void yy_flex_strncpy( char *s1, yyconst char *s2, int n ) +#else +static void yy_flex_strncpy( s1, s2, n ) +char *s1; +yyconst char *s2; +int n; +#endif + { + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; + } +#endif + +#ifdef YY_NEED_STRLEN +#ifdef YY_USE_PROTOS +static int yy_flex_strlen( yyconst char *s ) +#else +static int yy_flex_strlen( s ) +yyconst char *s; +#endif + { + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; + } +#endif + + +#ifdef YY_USE_PROTOS +static void *yy_flex_alloc( yy_size_t size ) +#else +static void *yy_flex_alloc( size ) +yy_size_t size; +#endif + { + return (void *) malloc( size ); + } + +#ifdef YY_USE_PROTOS +static void *yy_flex_realloc( void *ptr, yy_size_t size ) +#else +static void *yy_flex_realloc( ptr, size ) +void *ptr; +yy_size_t size; +#endif + { + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return (void *) realloc( (char *) ptr, size ); + } + +#ifdef YY_USE_PROTOS +static void yy_flex_free( void *ptr ) +#else +static void yy_flex_free( ptr ) +void *ptr; +#endif + { + free( ptr ); + } + +#if YY_MAIN +int main() + { + yylex(); + return 0; + } +#endif diff --git a/usr.bin/lex/flexdef.h b/usr.bin/lex/flexdef.h new file mode 100644 index 0000000..1c51ec9 --- /dev/null +++ b/usr.bin/lex/flexdef.h @@ -0,0 +1,1049 @@ +/* flexdef - definitions file for flex */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* @(#) $Header: /home/daffy/u0/vern/flex/RCS/flexdef.h,v 2.53 95/04/20 11:17:36 vern Exp $ (LBL) */ +/* $FreeBSD$ */ + +#include <stdio.h> +#include <ctype.h> + +#include "config.h" + +#ifdef __TURBOC__ +#define HAVE_STRING_H 1 +#define MS_DOS 1 +#ifndef __STDC__ +#define __STDC__ 1 +#endif + #pragma warn -pro + #pragma warn -rch + #pragma warn -use + #pragma warn -aus + #pragma warn -par + #pragma warn -pia +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#else +#include <strings.h> +#endif + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif + +#ifdef HAVE_MALLOC_H +#include <malloc.h> +#endif + +#ifdef STDC_HEADERS +#include <stdlib.h> +#endif + +/* As an aid for the internationalization patch to flex, which + * is maintained outside this distribution for copyright reasons. + */ +#define _(String) (String) + +/* Always be prepared to generate an 8-bit scanner. */ +#define CSIZE 256 +#define Char unsigned char + +/* Size of input alphabet - should be size of ASCII set. */ +#ifndef DEFAULT_CSIZE +#define DEFAULT_CSIZE 128 +#endif + +#ifndef PROTO +#if __STDC__ +#define PROTO(proto) proto +#else +#define PROTO(proto) () +#endif +#endif + +#ifdef VMS +#ifndef __VMS_POSIX +#define unlink remove +#define SHORT_FILE_NAMES +#endif +#endif + +#ifdef MS_DOS +#define SHORT_FILE_NAMES +#endif + + +/* Maximum line length we'll have to deal with. */ +#define MAXLINE 2048 + +#ifndef MIN +#define MIN(x,y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x,y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(x) ((x) < 0 ? -(x) : (x)) +#endif + + +/* ANSI C does not guarantee that isascii() is defined */ +#ifndef isascii +#define isascii(c) ((c) <= 0177) +#endif + + +#define true 1 +#define false 0 +#define unspecified -1 + + +/* Special chk[] values marking the slots taking by end-of-buffer and action + * numbers. + */ +#define EOB_POSITION -1 +#define ACTION_POSITION -2 + +/* Number of data items per line for -f output. */ +#define NUMDATAITEMS 10 + +/* Number of lines of data in -f output before inserting a blank line for + * readability. + */ +#define NUMDATALINES 10 + +/* transition_struct_out() definitions. */ +#define TRANS_STRUCT_PRINT_LENGTH 14 + +/* Returns true if an nfa state has an epsilon out-transition slot + * that can be used. This definition is currently not used. + */ +#define FREE_EPSILON(state) \ + (transchar[state] == SYM_EPSILON && \ + trans2[state] == NO_TRANSITION && \ + finalst[state] != state) + +/* Returns true if an nfa state has an epsilon out-transition character + * and both slots are free + */ +#define SUPER_FREE_EPSILON(state) \ + (transchar[state] == SYM_EPSILON && \ + trans1[state] == NO_TRANSITION) \ + +/* Maximum number of NFA states that can comprise a DFA state. It's real + * big because if there's a lot of rules, the initial state will have a + * huge epsilon closure. + */ +#define INITIAL_MAX_DFA_SIZE 750 +#define MAX_DFA_SIZE_INCREMENT 750 + + +/* A note on the following masks. They are used to mark accepting numbers + * as being special. As such, they implicitly limit the number of accepting + * numbers (i.e., rules) because if there are too many rules the rule numbers + * will overload the mask bits. Fortunately, this limit is \large/ (0x2000 == + * 8192) so unlikely to actually cause any problems. A check is made in + * new_rule() to ensure that this limit is not reached. + */ + +/* Mask to mark a trailing context accepting number. */ +#define YY_TRAILING_MASK 0x2000 + +/* Mask to mark the accepting number of the "head" of a trailing context + * rule. + */ +#define YY_TRAILING_HEAD_MASK 0x4000 + +/* Maximum number of rules, as outlined in the above note. */ +#define MAX_RULE (YY_TRAILING_MASK - 1) + + +/* NIL must be 0. If not, its special meaning when making equivalence classes + * (it marks the representative of a given e.c.) will be unidentifiable. + */ +#define NIL 0 + +#define JAM -1 /* to mark a missing DFA transition */ +#define NO_TRANSITION NIL +#define UNIQUE -1 /* marks a symbol as an e.c. representative */ +#define INFINITY -1 /* for x{5,} constructions */ + +#define INITIAL_MAX_CCLS 100 /* max number of unique character classes */ +#define MAX_CCLS_INCREMENT 100 + +/* Size of table holding members of character classes. */ +#define INITIAL_MAX_CCL_TBL_SIZE 500 +#define MAX_CCL_TBL_SIZE_INCREMENT 250 + +#define INITIAL_MAX_RULES 100 /* default maximum number of rules */ +#define MAX_RULES_INCREMENT 100 + +#define INITIAL_MNS 2000 /* default maximum number of nfa states */ +#define MNS_INCREMENT 1000 /* amount to bump above by if it's not enough */ + +#define INITIAL_MAX_DFAS 1000 /* default maximum number of dfa states */ +#define MAX_DFAS_INCREMENT 1000 + +#define JAMSTATE -32766 /* marks a reference to the state that always jams */ + +/* Maximum number of NFA states. */ +#define MAXIMUM_MNS 31999 + +/* Enough so that if it's subtracted from an NFA state number, the result + * is guaranteed to be negative. + */ +#define MARKER_DIFFERENCE (MAXIMUM_MNS+2) + +/* Maximum number of nxt/chk pairs for non-templates. */ +#define INITIAL_MAX_XPAIRS 2000 +#define MAX_XPAIRS_INCREMENT 2000 + +/* Maximum number of nxt/chk pairs needed for templates. */ +#define INITIAL_MAX_TEMPLATE_XPAIRS 2500 +#define MAX_TEMPLATE_XPAIRS_INCREMENT 2500 + +#define SYM_EPSILON (CSIZE + 1) /* to mark transitions on the symbol epsilon */ + +#define INITIAL_MAX_SCS 40 /* maximum number of start conditions */ +#define MAX_SCS_INCREMENT 40 /* amount to bump by if it's not enough */ + +#define ONE_STACK_SIZE 500 /* stack of states with only one out-transition */ +#define SAME_TRANS -1 /* transition is the same as "default" entry for state */ + +/* The following percentages are used to tune table compression: + + * The percentage the number of out-transitions a state must be of the + * number of equivalence classes in order to be considered for table + * compaction by using protos. + */ +#define PROTO_SIZE_PERCENTAGE 15 + +/* The percentage the number of homogeneous out-transitions of a state + * must be of the number of total out-transitions of the state in order + * that the state's transition table is first compared with a potential + * template of the most common out-transition instead of with the first + * proto in the proto queue. + */ +#define CHECK_COM_PERCENTAGE 50 + +/* The percentage the number of differences between a state's transition + * table and the proto it was first compared with must be of the total + * number of out-transitions of the state in order to keep the first + * proto as a good match and not search any further. + */ +#define FIRST_MATCH_DIFF_PERCENTAGE 10 + +/* The percentage the number of differences between a state's transition + * table and the most similar proto must be of the state's total number + * of out-transitions to use the proto as an acceptable close match. + */ +#define ACCEPTABLE_DIFF_PERCENTAGE 50 + +/* The percentage the number of homogeneous out-transitions of a state + * must be of the number of total out-transitions of the state in order + * to consider making a template from the state. + */ +#define TEMPLATE_SAME_PERCENTAGE 60 + +/* The percentage the number of differences between a state's transition + * table and the most similar proto must be of the state's total number + * of out-transitions to create a new proto from the state. + */ +#define NEW_PROTO_DIFF_PERCENTAGE 20 + +/* The percentage the total number of out-transitions of a state must be + * of the number of equivalence classes in order to consider trying to + * fit the transition table into "holes" inside the nxt/chk table. + */ +#define INTERIOR_FIT_PERCENTAGE 15 + +/* Size of region set aside to cache the complete transition table of + * protos on the proto queue to enable quick comparisons. + */ +#define PROT_SAVE_SIZE 2000 + +#define MSP 50 /* maximum number of saved protos (protos on the proto queue) */ + +/* Maximum number of out-transitions a state can have that we'll rummage + * around through the interior of the internal fast table looking for a + * spot for it. + */ +#define MAX_XTIONS_FULL_INTERIOR_FIT 4 + +/* Maximum number of rules which will be reported as being associated + * with a DFA state. + */ +#define MAX_ASSOC_RULES 100 + +/* Number that, if used to subscript an array, has a good chance of producing + * an error; should be small enough to fit into a short. + */ +#define BAD_SUBSCRIPT -32767 + +/* Absolute value of largest number that can be stored in a short, with a + * bit of slop thrown in for general paranoia. + */ +#define MAX_SHORT 32700 + + +/* Declarations for global variables. */ + +/* Variables for symbol tables: + * sctbl - start-condition symbol table + * ndtbl - name-definition symbol table + * ccltab - character class text symbol table + */ + +struct hash_entry + { + struct hash_entry *prev, *next; + char *name; + char *str_val; + int int_val; + } ; + +typedef struct hash_entry **hash_table; + +#define NAME_TABLE_HASH_SIZE 101 +#define START_COND_HASH_SIZE 101 +#define CCL_HASH_SIZE 101 + +extern struct hash_entry *ndtbl[NAME_TABLE_HASH_SIZE]; +extern struct hash_entry *sctbl[START_COND_HASH_SIZE]; +extern struct hash_entry *ccltab[CCL_HASH_SIZE]; + + +/* Variables for flags: + * printstats - if true (-v), dump statistics + * syntaxerror - true if a syntax error has been found + * eofseen - true if we've seen an eof in the input file + * ddebug - if true (-d), make a "debug" scanner + * trace - if true (-T), trace processing + * nowarn - if true (-w), do not generate warnings + * spprdflt - if true (-s), suppress the default rule + * interactive - if true (-I), generate an interactive scanner + * caseins - if true (-i), generate a case-insensitive scanner + * lex_compat - if true (-l), maximize compatibility with AT&T lex + * do_yylineno - if true, generate code to maintain yylineno + * useecs - if true (-Ce flag), use equivalence classes + * fulltbl - if true (-Cf flag), don't compress the DFA state table + * usemecs - if true (-Cm flag), use meta-equivalence classes + * fullspd - if true (-F flag), use Jacobson method of table representation + * gen_line_dirs - if true (i.e., no -L flag), generate #line directives + * performance_report - if > 0 (i.e., -p flag), generate a report relating + * to scanner performance; if > 1 (-p -p), report on minor performance + * problems, too + * backing_up_report - if true (i.e., -b flag), generate "lex.backup" file + * listing backing-up states + * C_plus_plus - if true (i.e., -+ flag), generate a C++ scanner class; + * otherwise, a standard C scanner + * long_align - if true (-Ca flag), favor long-word alignment. + * use_read - if true (-f, -F, or -Cr) then use read() for scanner input; + * otherwise, use fread(). + * yytext_is_array - if true (i.e., %array directive), then declare + * yytext as an array instead of a character pointer. Nice and inefficient. + * do_yywrap - do yywrap() processing on EOF. If false, EOF treated as + * "no more files". + * csize - size of character set for the scanner we're generating; + * 128 for 7-bit chars and 256 for 8-bit + * yymore_used - if true, yymore() is used in input rules + * reject - if true, generate back-up tables for REJECT macro + * real_reject - if true, scanner really uses REJECT (as opposed to just + * having "reject" set for variable trailing context) + * continued_action - true if this rule's action is to "fall through" to + * the next rule's action (i.e., the '|' action) + * in_rule - true if we're inside an individual rule, false if not. + * yymore_really_used - whether to treat yymore() as really used, regardless + * of what we think based on references to it in the user's actions. + * reject_really_used - same for REJECT + */ + +extern int printstats, syntaxerror, eofseen, ddebug, trace, nowarn, spprdflt; +extern int interactive, caseins, lex_compat, do_yylineno; +extern int useecs, fulltbl, usemecs, fullspd; +extern int gen_line_dirs, performance_report, backing_up_report; +extern int C_plus_plus, long_align, use_read, yytext_is_array, do_yywrap; +extern int csize; +extern int yymore_used, reject, real_reject, continued_action, in_rule; + +extern int yymore_really_used, reject_really_used; + + +/* Variables used in the flex input routines: + * datapos - characters on current output line + * dataline - number of contiguous lines of data in current data + * statement. Used to generate readable -f output + * linenum - current input line number + * out_linenum - current output line number + * skelfile - the skeleton file + * skel - compiled-in skeleton array + * skel_ind - index into "skel" array, if skelfile is nil + * yyin - input file + * backing_up_file - file to summarize backing-up states to + * infilename - name of input file + * outfilename - name of output file + * did_outfilename - whether outfilename was explicitly set + * prefix - the prefix used for externally visible names ("yy" by default) + * yyclass - yyFlexLexer subclass to use for YY_DECL + * do_stdinit - whether to initialize yyin/yyout to stdin/stdout + * use_stdout - the -t flag + * input_files - array holding names of input files + * num_input_files - size of input_files array + * program_name - name with which program was invoked + * + * action_array - array to hold the rule actions + * action_size - size of action_array + * defs1_offset - index where the user's section 1 definitions start + * in action_array + * prolog_offset - index where the prolog starts in action_array + * action_offset - index where the non-prolog starts in action_array + * action_index - index where the next action should go, with respect + * to "action_array" + */ + +extern int datapos, dataline, linenum, out_linenum; +extern FILE *skelfile, *yyin, *backing_up_file; +extern const char *skel[]; +extern int skel_ind; +extern char *infilename, *outfilename; +extern int did_outfilename; +extern char *prefix, *yyclass; +extern int do_stdinit, use_stdout; +extern char **input_files; +extern int num_input_files; +extern char *program_name; + +extern char *action_array; +extern int action_size; +extern int defs1_offset, prolog_offset, action_offset, action_index; + + +/* Variables for stack of states having only one out-transition: + * onestate - state number + * onesym - transition symbol + * onenext - target state + * onedef - default base entry + * onesp - stack pointer + */ + +extern int onestate[ONE_STACK_SIZE], onesym[ONE_STACK_SIZE]; +extern int onenext[ONE_STACK_SIZE], onedef[ONE_STACK_SIZE], onesp; + + +/* Variables for nfa machine data: + * current_mns - current maximum on number of NFA states + * num_rules - number of the last accepting state; also is number of + * rules created so far + * num_eof_rules - number of <<EOF>> rules + * default_rule - number of the default rule + * current_max_rules - current maximum number of rules + * lastnfa - last nfa state number created + * firstst - physically the first state of a fragment + * lastst - last physical state of fragment + * finalst - last logical state of fragment + * transchar - transition character + * trans1 - transition state + * trans2 - 2nd transition state for epsilons + * accptnum - accepting number + * assoc_rule - rule associated with this NFA state (or 0 if none) + * state_type - a STATE_xxx type identifying whether the state is part + * of a normal rule, the leading state in a trailing context + * rule (i.e., the state which marks the transition from + * recognizing the text-to-be-matched to the beginning of + * the trailing context), or a subsequent state in a trailing + * context rule + * rule_type - a RULE_xxx type identifying whether this a ho-hum + * normal rule or one which has variable head & trailing + * context + * rule_linenum - line number associated with rule + * rule_useful - true if we've determined that the rule can be matched + */ + +extern int current_mns, current_max_rules; +extern int num_rules, num_eof_rules, default_rule, lastnfa; +extern int *firstst, *lastst, *finalst, *transchar, *trans1, *trans2; +extern int *accptnum, *assoc_rule, *state_type; +extern int *rule_type, *rule_linenum, *rule_useful; + +/* Different types of states; values are useful as masks, as well, for + * routines like check_trailing_context(). + */ +#define STATE_NORMAL 0x1 +#define STATE_TRAILING_CONTEXT 0x2 + +/* Global holding current type of state we're making. */ + +extern int current_state_type; + +/* Different types of rules. */ +#define RULE_NORMAL 0 +#define RULE_VARIABLE 1 + +/* True if the input rules include a rule with both variable-length head + * and trailing context, false otherwise. + */ +extern int variable_trailing_context_rules; + + +/* Variables for protos: + * numtemps - number of templates created + * numprots - number of protos created + * protprev - backlink to a more-recently used proto + * protnext - forward link to a less-recently used proto + * prottbl - base/def table entry for proto + * protcomst - common state of proto + * firstprot - number of the most recently used proto + * lastprot - number of the least recently used proto + * protsave contains the entire state array for protos + */ + +extern int numtemps, numprots, protprev[MSP], protnext[MSP], prottbl[MSP]; +extern int protcomst[MSP], firstprot, lastprot, protsave[PROT_SAVE_SIZE]; + + +/* Variables for managing equivalence classes: + * numecs - number of equivalence classes + * nextecm - forward link of Equivalence Class members + * ecgroup - class number or backward link of EC members + * nummecs - number of meta-equivalence classes (used to compress + * templates) + * tecfwd - forward link of meta-equivalence classes members + * tecbck - backward link of MEC's + */ + +/* Reserve enough room in the equivalence class arrays so that we + * can use the CSIZE'th element to hold equivalence class information + * for the NUL character. Later we'll move this information into + * the 0th element. + */ +extern int numecs, nextecm[CSIZE + 1], ecgroup[CSIZE + 1], nummecs; + +/* Meta-equivalence classes are indexed starting at 1, so it's possible + * that they will require positions from 1 .. CSIZE, i.e., CSIZE + 1 + * slots total (since the arrays are 0-based). nextecm[] and ecgroup[] + * don't require the extra position since they're indexed from 1 .. CSIZE - 1. + */ +extern int tecfwd[CSIZE + 1], tecbck[CSIZE + 1]; + + +/* Variables for start conditions: + * lastsc - last start condition created + * current_max_scs - current limit on number of start conditions + * scset - set of rules active in start condition + * scbol - set of rules active only at the beginning of line in a s.c. + * scxclu - true if start condition is exclusive + * sceof - true if start condition has EOF rule + * scname - start condition name + */ + +extern int lastsc, *scset, *scbol, *scxclu, *sceof; +extern int current_max_scs; +extern char **scname; + + +/* Variables for dfa machine data: + * current_max_dfa_size - current maximum number of NFA states in DFA + * current_max_xpairs - current maximum number of non-template xtion pairs + * current_max_template_xpairs - current maximum number of template pairs + * current_max_dfas - current maximum number DFA states + * lastdfa - last dfa state number created + * nxt - state to enter upon reading character + * chk - check value to see if "nxt" applies + * tnxt - internal nxt table for templates + * base - offset into "nxt" for given state + * def - where to go if "chk" disallows "nxt" entry + * nultrans - NUL transition for each state + * NUL_ec - equivalence class of the NUL character + * tblend - last "nxt/chk" table entry being used + * firstfree - first empty entry in "nxt/chk" table + * dss - nfa state set for each dfa + * dfasiz - size of nfa state set for each dfa + * dfaacc - accepting set for each dfa state (if using REJECT), or accepting + * number, if not + * accsiz - size of accepting set for each dfa state + * dhash - dfa state hash value + * numas - number of DFA accepting states created; note that this + * is not necessarily the same value as num_rules, which is the analogous + * value for the NFA + * numsnpairs - number of state/nextstate transition pairs + * jambase - position in base/def where the default jam table starts + * jamstate - state number corresponding to "jam" state + * end_of_buffer_state - end-of-buffer dfa state number + */ + +extern int current_max_dfa_size, current_max_xpairs; +extern int current_max_template_xpairs, current_max_dfas; +extern int lastdfa, *nxt, *chk, *tnxt; +extern int *base, *def, *nultrans, NUL_ec, tblend, firstfree, **dss, *dfasiz; +extern union dfaacc_union + { + int *dfaacc_set; + int dfaacc_state; + } *dfaacc; +extern int *accsiz, *dhash, numas; +extern int numsnpairs, jambase, jamstate; +extern int end_of_buffer_state; + +/* Variables for ccl information: + * lastccl - ccl index of the last created ccl + * current_maxccls - current limit on the maximum number of unique ccl's + * cclmap - maps a ccl index to its set pointer + * ccllen - gives the length of a ccl + * cclng - true for a given ccl if the ccl is negated + * cclreuse - counts how many times a ccl is re-used + * current_max_ccl_tbl_size - current limit on number of characters needed + * to represent the unique ccl's + * ccltbl - holds the characters in each ccl - indexed by cclmap + */ + +extern int lastccl, *cclmap, *ccllen, *cclng, cclreuse; +extern int current_maxccls, current_max_ccl_tbl_size; +extern Char *ccltbl; + + +/* Variables for miscellaneous information: + * nmstr - last NAME scanned by the scanner + * sectnum - section number currently being parsed + * nummt - number of empty nxt/chk table entries + * hshcol - number of hash collisions detected by snstods + * dfaeql - number of times a newly created dfa was equal to an old one + * numeps - number of epsilon NFA states created + * eps2 - number of epsilon states which have 2 out-transitions + * num_reallocs - number of times it was necessary to realloc() a group + * of arrays + * tmpuses - number of DFA states that chain to templates + * totnst - total number of NFA states used to make DFA states + * peakpairs - peak number of transition pairs we had to store internally + * numuniq - number of unique transitions + * numdup - number of duplicate transitions + * hshsave - number of hash collisions saved by checking number of states + * num_backing_up - number of DFA states requiring backing up + * bol_needed - whether scanner needs beginning-of-line recognition + */ + +extern char nmstr[MAXLINE]; +extern int sectnum, nummt, hshcol, dfaeql, numeps, eps2, num_reallocs; +extern int tmpuses, totnst, peakpairs, numuniq, numdup, hshsave; +extern int num_backing_up, bol_needed; + +void *allocate_array PROTO((int, size_t)); +void *reallocate_array PROTO((void*, int, size_t)); + +void *flex_alloc PROTO((size_t)); +void *flex_realloc PROTO((void*, size_t)); +void flex_free PROTO((void*)); + +#define allocate_integer_array(size) \ + (int *) allocate_array( size, sizeof( int ) ) + +#define reallocate_integer_array(array,size) \ + (int *) reallocate_array( (void *) array, size, sizeof( int ) ) + +#define allocate_int_ptr_array(size) \ + (int **) allocate_array( size, sizeof( int * ) ) + +#define allocate_char_ptr_array(size) \ + (char **) allocate_array( size, sizeof( char * ) ) + +#define allocate_dfaacc_union(size) \ + (union dfaacc_union *) \ + allocate_array( size, sizeof( union dfaacc_union ) ) + +#define reallocate_int_ptr_array(array,size) \ + (int **) reallocate_array( (void *) array, size, sizeof( int * ) ) + +#define reallocate_char_ptr_array(array,size) \ + (char **) reallocate_array( (void *) array, size, sizeof( char * ) ) + +#define reallocate_dfaacc_union(array, size) \ + (union dfaacc_union *) \ + reallocate_array( (void *) array, size, sizeof( union dfaacc_union ) ) + +#define allocate_character_array(size) \ + (char *) allocate_array( size, sizeof( char ) ) + +#define reallocate_character_array(array,size) \ + (char *) reallocate_array( (void *) array, size, sizeof( char ) ) + +#define allocate_Character_array(size) \ + (Char *) allocate_array( size, sizeof( Char ) ) + +#define reallocate_Character_array(array,size) \ + (Char *) reallocate_array( (void *) array, size, sizeof( Char ) ) + + +/* Used to communicate between scanner and parser. The type should really + * be YYSTYPE, but we can't easily get our hands on it. + */ +extern int yylval; + + +/* External functions that are cross-referenced among the flex source files. */ + + +/* from file ccl.c */ + +extern void ccladd PROTO((int, int)); /* add a single character to a ccl */ +extern int cclinit PROTO((void)); /* make an empty ccl */ +extern void cclnegate PROTO((int)); /* negate a ccl */ + +/* List the members of a set of characters in CCL form. */ +extern void list_character_set PROTO((FILE*, int[])); + + +/* from file dfa.c */ + +/* Check a DFA state for backing up. */ +extern void check_for_backing_up PROTO((int, int[])); + +/* Check to see if NFA state set constitutes "dangerous" trailing context. */ +extern void check_trailing_context PROTO((int*, int, int*, int)); + +/* Construct the epsilon closure of a set of ndfa states. */ +extern int *epsclosure PROTO((int*, int*, int[], int*, int*)); + +/* Increase the maximum number of dfas. */ +extern void increase_max_dfas PROTO((void)); + +extern void ntod PROTO((void)); /* convert a ndfa to a dfa */ + +/* Converts a set of ndfa states into a dfa state. */ +extern int snstods PROTO((int[], int, int[], int, int, int*)); + + +/* from file ecs.c */ + +/* Convert character classes to set of equivalence classes. */ +extern void ccl2ecl PROTO((void)); + +/* Associate equivalence class numbers with class members. */ +extern int cre8ecs PROTO((int[], int[], int)); + +/* Update equivalence classes based on character class transitions. */ +extern void mkeccl PROTO((Char[], int, int[], int[], int, int)); + +/* Create equivalence class for single character. */ +extern void mkechar PROTO((int, int[], int[])); + + +/* from file gen.c */ + +extern void do_indent PROTO((void)); /* indent to the current level */ + +/* Generate the code to keep backing-up information. */ +extern void gen_backing_up PROTO((void)); + +/* Generate the code to perform the backing up. */ +extern void gen_bu_action PROTO((void)); + +/* Generate full speed compressed transition table. */ +extern void genctbl PROTO((void)); + +/* Generate the code to find the action number. */ +extern void gen_find_action PROTO((void)); + +extern void genftbl PROTO((void)); /* generate full transition table */ + +/* Generate the code to find the next compressed-table state. */ +extern void gen_next_compressed_state PROTO((char*)); + +/* Generate the code to find the next match. */ +extern void gen_next_match PROTO((void)); + +/* Generate the code to find the next state. */ +extern void gen_next_state PROTO((int)); + +/* Generate the code to make a NUL transition. */ +extern void gen_NUL_trans PROTO((void)); + +/* Generate the code to find the start state. */ +extern void gen_start_state PROTO((void)); + +/* Generate data statements for the transition tables. */ +extern void gentabs PROTO((void)); + +/* Write out a formatted string at the current indentation level. */ +extern void indent_put2s PROTO((char[], char[])); + +/* Write out a string + newline at the current indentation level. */ +extern void indent_puts PROTO((char[])); + +extern void make_tables PROTO((void)); /* generate transition tables */ + + +/* from file main.c */ + +extern void check_options PROTO((void)); +extern void flexend PROTO((int)); +extern void usage PROTO((void)); + + +/* from file misc.c */ + +/* Add a #define to the action file. */ +extern void action_define PROTO(( char *defname, int value )); + +/* Add the given text to the stored actions. */ +extern void add_action PROTO(( char *new_text )); + +/* True if a string is all lower case. */ +extern int all_lower PROTO((char *)); + +/* True if a string is all upper case. */ +extern int all_upper PROTO((char *)); + +/* Bubble sort an integer array. */ +extern void bubble PROTO((int [], int)); + +/* Check a character to make sure it's in the expected range. */ +extern void check_char PROTO((int c)); + +/* Replace upper-case letter to lower-case. */ +extern Char clower PROTO((int)); + +/* Returns a dynamically allocated copy of a string. */ +extern char *copy_string PROTO((const char *)); + +/* Returns a dynamically allocated copy of a (potentially) unsigned string. */ +extern Char *copy_unsigned_string PROTO((Char *)); + +/* Shell sort a character array. */ +extern void cshell PROTO((Char [], int, int)); + +/* Finish up a block of data declarations. */ +extern void dataend PROTO((void)); + +/* Flush generated data statements. */ +extern void dataflush PROTO((void)); + +/* Report an error message and terminate. */ +extern void flexerror PROTO((const char[])); + +/* Report a fatal error message and terminate. */ +extern void flexfatal PROTO((const char[])); + +/* Convert a hexadecimal digit string to an integer value. */ +extern int htoi PROTO((Char[])); + +/* Report an error message formatted with one integer argument. */ +extern void lerrif PROTO((const char[], int)); + +/* Report an error message formatted with one string argument. */ +extern void lerrsf PROTO((const char[], const char[])); + +/* Spit out a "#line" statement. */ +extern void line_directive_out PROTO((FILE*, int)); + +/* Mark the current position in the action array as the end of the section 1 + * user defs. + */ +extern void mark_defs1 PROTO((void)); + +/* Mark the current position in the action array as the end of the prolog. */ +extern void mark_prolog PROTO((void)); + +/* Generate a data statement for a two-dimensional array. */ +extern void mk2data PROTO((int)); + +extern void mkdata PROTO((int)); /* generate a data statement */ + +/* Return the integer represented by a string of digits. */ +extern int myctoi PROTO((char [])); + +/* Return character corresponding to escape sequence. */ +extern Char myesc PROTO((Char[])); + +/* Convert an octal digit string to an integer value. */ +extern int otoi PROTO((Char [] )); + +/* Output a (possibly-formatted) string to the generated scanner. */ +extern void out PROTO((const char [])); +extern void out_dec PROTO((const char [], int)); +extern void out_dec2 PROTO((const char [], int, int)); +extern void out_hex PROTO((const char [], unsigned int)); +extern void out_line_count PROTO((const char [])); +extern void out_str PROTO((const char [], const char [])); +extern void out_str3 + PROTO((const char [], const char [], const char [], const char [])); +extern void out_str_dec PROTO((const char [], const char [], int)); +extern void outc PROTO((int)); +extern void outn PROTO((const char [])); + +/* Return a printable version of the given character, which might be + * 8-bit. + */ +extern char *readable_form PROTO((int)); + +/* Write out one section of the skeleton file. */ +extern void skelout PROTO((void)); + +/* Output a yy_trans_info structure. */ +extern void transition_struct_out PROTO((int, int)); + +/* Only needed when using certain broken versions of bison to build parse.c. */ +extern void *yy_flex_xmalloc PROTO(( int )); + +/* Set a region of memory to 0. */ +extern void zero_out PROTO((char *, size_t)); + + +/* from file nfa.c */ + +/* Add an accepting state to a machine. */ +extern void add_accept PROTO((int, int)); + +/* Make a given number of copies of a singleton machine. */ +extern int copysingl PROTO((int, int)); + +/* Debugging routine to write out an nfa. */ +extern void dumpnfa PROTO((int)); + +/* Finish up the processing for a rule. */ +extern void finish_rule PROTO((int, int, int, int)); + +/* Connect two machines together. */ +extern int link_machines PROTO((int, int)); + +/* Mark each "beginning" state in a machine as being a "normal" (i.e., + * not trailing context associated) state. + */ +extern void mark_beginning_as_normal PROTO((int)); + +/* Make a machine that branches to two machines. */ +extern int mkbranch PROTO((int, int)); + +extern int mkclos PROTO((int)); /* convert a machine into a closure */ +extern int mkopt PROTO((int)); /* make a machine optional */ + +/* Make a machine that matches either one of two machines. */ +extern int mkor PROTO((int, int)); + +/* Convert a machine into a positive closure. */ +extern int mkposcl PROTO((int)); + +extern int mkrep PROTO((int, int, int)); /* make a replicated machine */ + +/* Create a state with a transition on a given symbol. */ +extern int mkstate PROTO((int)); + +extern void new_rule PROTO((void)); /* initialize for a new rule */ + + +/* from file parse.y */ + +/* Build the "<<EOF>>" action for the active start conditions. */ +extern void build_eof_action PROTO((void)); + +/* Write out a message formatted with one string, pinpointing its location. */ +extern void format_pinpoint_message PROTO((char[], char[])); + +/* Write out a message, pinpointing its location. */ +extern void pinpoint_message PROTO((char[])); + +/* Write out a warning, pinpointing it at the given line. */ +extern void line_warning PROTO(( char[], int )); + +/* Write out a message, pinpointing it at the given line. */ +extern void line_pinpoint PROTO(( char[], int )); + +/* Report a formatted syntax error. */ +extern void format_synerr PROTO((char [], char[])); +extern void synerr PROTO((char [])); /* report a syntax error */ +extern void format_warn PROTO((char [], char[])); +extern void warn PROTO((char [])); /* report a warning */ +extern void yyerror PROTO((char [])); /* report a parse error */ +extern int yyparse PROTO((void)); /* the YACC parser */ + + +/* from file scan.l */ + +/* The Flex-generated scanner for flex. */ +extern int flexscan PROTO((void)); + +/* Open the given file (if NULL, stdin) for scanning. */ +extern void set_input_file PROTO((char*)); + +/* Wrapup a file in the lexical analyzer. */ +extern int yywrap PROTO((void)); + + +/* from file sym.c */ + +/* Add symbol and definitions to symbol table. */ +extern int addsym PROTO((char[], char*, int, hash_table, int)); + +/* Save the text of a character class. */ +extern void cclinstal PROTO ((Char [], int)); + +/* Lookup the number associated with character class. */ +extern int ccllookup PROTO((Char [])); + +/* Find symbol in symbol table. */ +extern struct hash_entry *findsym PROTO((char[], hash_table, int )); + +extern void ndinstal PROTO((char[], Char[])); /* install a name definition */ +extern Char *ndlookup PROTO((char[])); /* lookup a name definition */ + +/* Increase maximum number of SC's. */ +extern void scextend PROTO((void)); +extern void scinstal PROTO((char[], int)); /* make a start condition */ + +/* Lookup the number associated with a start condition. */ +extern int sclookup PROTO((char[])); + + +/* from file tblcmp.c */ + +/* Build table entries for dfa state. */ +extern void bldtbl PROTO((int[], int, int, int, int)); + +extern void cmptmps PROTO((void)); /* compress template table entries */ +extern void expand_nxt_chk PROTO((void)); /* increase nxt/chk arrays */ +/* Finds a space in the table for a state to be placed. */ +extern int find_table_space PROTO((int*, int)); +extern void inittbl PROTO((void)); /* initialize transition tables */ +/* Make the default, "jam" table entries. */ +extern void mkdeftbl PROTO((void)); + +/* Create table entries for a state (or state fragment) which has + * only one out-transition. + */ +extern void mk1tbl PROTO((int, int, int, int)); + +/* Place a state into full speed transition table. */ +extern void place_state PROTO((int*, int, int)); + +/* Save states with only one out-transition to be processed later. */ +extern void stack1 PROTO((int, int, int, int)); + + +/* from file yylex.c */ + +extern int yylex PROTO((void)); diff --git a/usr.bin/lex/gen.c b/usr.bin/lex/gen.c new file mode 100644 index 0000000..b72688b --- /dev/null +++ b/usr.bin/lex/gen.c @@ -0,0 +1,1629 @@ +/* gen - actual generation (writing) of flex scanners */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/gen.c,v 2.56 96/05/25 20:43:38 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + + +/* declare functions that have forward references */ + +void gen_next_state PROTO((int)); +void genecs PROTO((void)); +void indent_put2s PROTO((char [], char [])); +void indent_puts PROTO((char [])); + + +static int indent_level = 0; /* each level is 8 spaces */ + +#define indent_up() (++indent_level) +#define indent_down() (--indent_level) +#define set_indent(indent_val) indent_level = indent_val + +/* Almost everything is done in terms of arrays starting at 1, so provide + * a null entry for the zero element of all C arrays. (The exception + * to this is that the fast table representation generally uses the + * 0 elements of its arrays, too.) + */ +static char C_int_decl[] = "static yyconst int %s[%d] =\n { 0,\n"; +static char C_short_decl[] = "static yyconst short int %s[%d] =\n { 0,\n"; +static char C_long_decl[] = "static yyconst long int %s[%d] =\n { 0,\n"; +static char C_state_decl[] = + "static yyconst yy_state_type %s[%d] =\n { 0,\n"; + + +/* Indent to the current level. */ + +void do_indent() + { + int i = indent_level * 8; + + while ( i >= 8 ) + { + outc( '\t' ); + i -= 8; + } + + while ( i > 0 ) + { + outc( ' ' ); + --i; + } + } + + +/* Generate the code to keep backing-up information. */ + +void gen_backing_up() + { + if ( reject || num_backing_up == 0 ) + return; + + if ( fullspd ) + indent_puts( "if ( yy_current_state[-1].yy_nxt )" ); + else + indent_puts( "if ( yy_accept[yy_current_state] )" ); + + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_last_accepting_state = yy_current_state;" ); + indent_puts( "yy_last_accepting_cpos = yy_cp;" ); + indent_puts( "}" ); + indent_down(); + } + + +/* Generate the code to perform the backing up. */ + +void gen_bu_action() + { + if ( reject || num_backing_up == 0 ) + return; + + set_indent( 3 ); + + indent_puts( "case 0: /* must back up */" ); + indent_puts( "/* undo the effects of YY_DO_BEFORE_ACTION */" ); + indent_puts( "*yy_cp = yy_hold_char;" ); + + if ( fullspd || fulltbl ) + indent_puts( "yy_cp = yy_last_accepting_cpos + 1;" ); + else + /* Backing-up info for compressed tables is taken \after/ + * yy_cp has been incremented for the next state. + */ + indent_puts( "yy_cp = yy_last_accepting_cpos;" ); + + indent_puts( "yy_current_state = yy_last_accepting_state;" ); + indent_puts( "goto yy_find_action;" ); + outc( '\n' ); + + set_indent( 0 ); + } + + +/* genctbl - generates full speed compressed transition table */ + +void genctbl() + { + int i; + int end_of_buffer_action = num_rules + 1; + + /* Table of verify for transition and offset to next state. */ + out_dec( "static yyconst struct yy_trans_info yy_transition[%d] =\n", + tblend + numecs + 1 ); + outn( " {" ); + + /* We want the transition to be represented as the offset to the + * next state, not the actual state number, which is what it currently + * is. The offset is base[nxt[i]] - (base of current state)]. That's + * just the difference between the starting points of the two involved + * states (to - from). + * + * First, though, we need to find some way to put in our end-of-buffer + * flags and states. We do this by making a state with absolutely no + * transitions. We put it at the end of the table. + */ + + /* We need to have room in nxt/chk for two more slots: One for the + * action and one for the end-of-buffer transition. We now *assume* + * that we're guaranteed the only character we'll try to index this + * nxt/chk pair with is EOB, i.e., 0, so we don't have to make sure + * there's room for jam entries for other characters. + */ + + while ( tblend + 2 >= current_max_xpairs ) + expand_nxt_chk(); + + while ( lastdfa + 1 >= current_max_dfas ) + increase_max_dfas(); + + base[lastdfa + 1] = tblend + 2; + nxt[tblend + 1] = end_of_buffer_action; + chk[tblend + 1] = numecs + 1; + chk[tblend + 2] = 1; /* anything but EOB */ + + /* So that "make test" won't show arb. differences. */ + nxt[tblend + 2] = 0; + + /* Make sure every state has an end-of-buffer transition and an + * action #. + */ + for ( i = 0; i <= lastdfa; ++i ) + { + int anum = dfaacc[i].dfaacc_state; + int offset = base[i]; + + chk[offset] = EOB_POSITION; + chk[offset - 1] = ACTION_POSITION; + nxt[offset - 1] = anum; /* action number */ + } + + for ( i = 0; i <= tblend; ++i ) + { + if ( chk[i] == EOB_POSITION ) + transition_struct_out( 0, base[lastdfa + 1] - i ); + + else if ( chk[i] == ACTION_POSITION ) + transition_struct_out( 0, nxt[i] ); + + else if ( chk[i] > numecs || chk[i] == 0 ) + transition_struct_out( 0, 0 ); /* unused slot */ + + else /* verify, transition */ + transition_struct_out( chk[i], + base[nxt[i]] - (i - chk[i]) ); + } + + + /* Here's the final, end-of-buffer state. */ + transition_struct_out( chk[tblend + 1], nxt[tblend + 1] ); + transition_struct_out( chk[tblend + 2], nxt[tblend + 2] ); + + outn( " };\n" ); + + /* Table of pointers to start states. */ + out_dec( + "static yyconst struct yy_trans_info *yy_start_state_list[%d] =\n", + lastsc * 2 + 1 ); + outn( " {" ); /* } so vi doesn't get confused */ + + for ( i = 0; i <= lastsc * 2; ++i ) + out_dec( " &yy_transition[%d],\n", base[i] ); + + dataend(); + + if ( useecs ) + genecs(); + } + + +/* Generate equivalence-class tables. */ + +void genecs() + { + int i, j; + int numrows; + + out_str_dec( C_int_decl, "yy_ec", csize ); + + for ( i = 1; i < csize; ++i ) + { + if ( caseins && (i >= 'A') && (i <= 'Z') ) + ecgroup[i] = ecgroup[clower( i )]; + + ecgroup[i] = ABS( ecgroup[i] ); + mkdata( ecgroup[i] ); + } + + dataend(); + + if ( trace ) + { + fputs( _( "\n\nEquivalence Classes:\n\n" ), stderr ); + + numrows = csize / 8; + + for ( j = 0; j < numrows; ++j ) + { + for ( i = j; i < csize; i = i + numrows ) + { + fprintf( stderr, "%4s = %-2d", + readable_form( i ), ecgroup[i] ); + + putc( ' ', stderr ); + } + + putc( '\n', stderr ); + } + } + } + + +/* Generate the code to find the action number. */ + +void gen_find_action() + { + if ( fullspd ) + indent_puts( "yy_act = yy_current_state[-1].yy_nxt;" ); + + else if ( fulltbl ) + indent_puts( "yy_act = yy_accept[yy_current_state];" ); + + else if ( reject ) + { + indent_puts( "yy_current_state = *--yy_state_ptr;" ); + indent_puts( "yy_lp = yy_accept[yy_current_state];" ); + + outn( + "goto find_rule; /* avoid `defined but not used' warning */"); + outn( + "find_rule: /* we branch to this label when backing up */" ); + + indent_puts( + "for ( ; ; ) /* until we find what rule we matched */" ); + + indent_up(); + + indent_puts( "{" ); + + indent_puts( + "if ( yy_lp && yy_lp < yy_accept[yy_current_state + 1] )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_act = yy_acclist[yy_lp];" ); + + if ( variable_trailing_context_rules ) + { + indent_puts( "if ( yy_act & YY_TRAILING_HEAD_MASK ||" ); + indent_puts( " yy_looking_for_trail_begin )" ); + indent_up(); + indent_puts( "{" ); + + indent_puts( + "if ( yy_act == yy_looking_for_trail_begin )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_looking_for_trail_begin = 0;" ); + indent_puts( "yy_act &= ~YY_TRAILING_HEAD_MASK;" ); + indent_puts( "break;" ); + indent_puts( "}" ); + indent_down(); + + indent_puts( "}" ); + indent_down(); + + indent_puts( "else if ( yy_act & YY_TRAILING_MASK )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( + "yy_looking_for_trail_begin = yy_act & ~YY_TRAILING_MASK;" ); + indent_puts( + "yy_looking_for_trail_begin |= YY_TRAILING_HEAD_MASK;" ); + + if ( real_reject ) + { + /* Remember matched text in case we back up + * due to REJECT. + */ + indent_puts( "yy_full_match = yy_cp;" ); + indent_puts( "yy_full_state = yy_state_ptr;" ); + indent_puts( "yy_full_lp = yy_lp;" ); + } + + indent_puts( "}" ); + indent_down(); + + indent_puts( "else" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_full_match = yy_cp;" ); + indent_puts( "yy_full_state = yy_state_ptr;" ); + indent_puts( "yy_full_lp = yy_lp;" ); + indent_puts( "break;" ); + indent_puts( "}" ); + indent_down(); + + indent_puts( "++yy_lp;" ); + indent_puts( "goto find_rule;" ); + } + + else + { + /* Remember matched text in case we back up due to + * trailing context plus REJECT. + */ + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_full_match = yy_cp;" ); + indent_puts( "break;" ); + indent_puts( "}" ); + indent_down(); + } + + indent_puts( "}" ); + indent_down(); + + indent_puts( "--yy_cp;" ); + + /* We could consolidate the following two lines with those at + * the beginning, but at the cost of complaints that we're + * branching inside a loop. + */ + indent_puts( "yy_current_state = *--yy_state_ptr;" ); + indent_puts( "yy_lp = yy_accept[yy_current_state];" ); + + indent_puts( "}" ); + + indent_down(); + } + + else + { /* compressed */ + indent_puts( "yy_act = yy_accept[yy_current_state];" ); + + if ( interactive && ! reject ) + { + /* Do the guaranteed-needed backing up to figure out + * the match. + */ + indent_puts( "if ( yy_act == 0 )" ); + indent_up(); + indent_puts( "{ /* have to back up */" ); + indent_puts( "yy_cp = yy_last_accepting_cpos;" ); + indent_puts( + "yy_current_state = yy_last_accepting_state;" ); + indent_puts( "yy_act = yy_accept[yy_current_state];" ); + indent_puts( "}" ); + indent_down(); + } + } + } + + +/* genftbl - generate full transition table */ + +void genftbl() + { + int i; + int end_of_buffer_action = num_rules + 1; + + out_str_dec( long_align ? C_long_decl : C_short_decl, + "yy_accept", lastdfa + 1 ); + + dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action; + + for ( i = 1; i <= lastdfa; ++i ) + { + int anum = dfaacc[i].dfaacc_state; + + mkdata( anum ); + + if ( trace && anum ) + fprintf( stderr, _( "state # %d accepts: [%d]\n" ), + i, anum ); + } + + dataend(); + + if ( useecs ) + genecs(); + + /* Don't have to dump the actual full table entries - they were + * created on-the-fly. + */ + } + + +/* Generate the code to find the next compressed-table state. */ + +void gen_next_compressed_state( char_map ) +char *char_map; + { + indent_put2s( "YY_CHAR yy_c = %s;", char_map ); + + /* Save the backing-up info \before/ computing the next state + * because we always compute one more state than needed - we + * always proceed until we reach a jam state + */ + gen_backing_up(); + + indent_puts( +"while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_current_state = (int) yy_def[yy_current_state];" ); + + if ( usemecs ) + { + /* We've arrange it so that templates are never chained + * to one another. This means we can afford to make a + * very simple test to see if we need to convert to + * yy_c's meta-equivalence class without worrying + * about erroneously looking up the meta-equivalence + * class twice + */ + do_indent(); + + /* lastdfa + 2 is the beginning of the templates */ + out_dec( "if ( yy_current_state >= %d )\n", lastdfa + 2 ); + + indent_up(); + indent_puts( "yy_c = yy_meta[(unsigned int) yy_c];" ); + indent_down(); + } + + indent_puts( "}" ); + indent_down(); + + indent_puts( +"yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];" ); + } + + +/* Generate the code to find the next match. */ + +void gen_next_match() + { + /* NOTE - changes in here should be reflected in gen_next_state() and + * gen_NUL_trans(). + */ + char *char_map = useecs ? + "yy_ec[YY_SC_TO_UI(*yy_cp)]" : + "YY_SC_TO_UI(*yy_cp)"; + + char *char_map_2 = useecs ? + "yy_ec[YY_SC_TO_UI(*++yy_cp)]" : + "YY_SC_TO_UI(*++yy_cp)"; + + if ( fulltbl ) + { + indent_put2s( + "while ( (yy_current_state = yy_nxt[yy_current_state][%s]) > 0 )", + char_map ); + + indent_up(); + + if ( num_backing_up > 0 ) + { + indent_puts( "{" ); /* } for vi */ + gen_backing_up(); + outc( '\n' ); + } + + indent_puts( "++yy_cp;" ); + + if ( num_backing_up > 0 ) + /* { for vi */ + indent_puts( "}" ); + + indent_down(); + + outc( '\n' ); + indent_puts( "yy_current_state = -yy_current_state;" ); + } + + else if ( fullspd ) + { + indent_puts( "{" ); /* } for vi */ + indent_puts( + "yyconst struct yy_trans_info *yy_trans_info;\n" ); + indent_puts( "YY_CHAR yy_c;\n" ); + indent_put2s( "for ( yy_c = %s;", char_map ); + indent_puts( + " (yy_trans_info = &yy_current_state[(unsigned int) yy_c])->" ); + indent_puts( "yy_verify == yy_c;" ); + indent_put2s( " yy_c = %s )", char_map_2 ); + + indent_up(); + + if ( num_backing_up > 0 ) + indent_puts( "{" ); /* } for vi */ + + indent_puts( "yy_current_state += yy_trans_info->yy_nxt;" ); + + if ( num_backing_up > 0 ) + { + outc( '\n' ); + gen_backing_up(); /* { for vi */ + indent_puts( "}" ); + } + + indent_down(); /* { for vi */ + indent_puts( "}" ); + } + + else + { /* compressed */ + indent_puts( "do" ); + + indent_up(); + indent_puts( "{" ); /* } for vi */ + + gen_next_state( false ); + + indent_puts( "++yy_cp;" ); + + /* { for vi */ + indent_puts( "}" ); + indent_down(); + + do_indent(); + + if ( interactive ) + out_dec( "while ( yy_base[yy_current_state] != %d );\n", + jambase ); + else + out_dec( "while ( yy_current_state != %d );\n", + jamstate ); + + if ( ! reject && ! interactive ) + { + /* Do the guaranteed-needed backing up to figure out + * the match. + */ + indent_puts( "yy_cp = yy_last_accepting_cpos;" ); + indent_puts( + "yy_current_state = yy_last_accepting_state;" ); + } + } + } + + +/* Generate the code to find the next state. */ + +void gen_next_state( worry_about_NULs ) +int worry_about_NULs; + { /* NOTE - changes in here should be reflected in gen_next_match() */ + char char_map[256]; + + if ( worry_about_NULs && ! nultrans ) + { + if ( useecs ) + (void) sprintf( char_map, + "(*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : %d)", + NUL_ec ); + else + (void) sprintf( char_map, + "(*yy_cp ? YY_SC_TO_UI(*yy_cp) : %d)", NUL_ec ); + } + + else + strcpy( char_map, useecs ? + "yy_ec[YY_SC_TO_UI(*yy_cp)]" : "YY_SC_TO_UI(*yy_cp)" ); + + if ( worry_about_NULs && nultrans ) + { + if ( ! fulltbl && ! fullspd ) + /* Compressed tables back up *before* they match. */ + gen_backing_up(); + + indent_puts( "if ( *yy_cp )" ); + indent_up(); + indent_puts( "{" ); /* } for vi */ + } + + if ( fulltbl ) + indent_put2s( + "yy_current_state = yy_nxt[yy_current_state][%s];", + char_map ); + + else if ( fullspd ) + indent_put2s( + "yy_current_state += yy_current_state[%s].yy_nxt;", + char_map ); + + else + gen_next_compressed_state( char_map ); + + if ( worry_about_NULs && nultrans ) + { + /* { for vi */ + indent_puts( "}" ); + indent_down(); + indent_puts( "else" ); + indent_up(); + indent_puts( + "yy_current_state = yy_NUL_trans[yy_current_state];" ); + indent_down(); + } + + if ( fullspd || fulltbl ) + gen_backing_up(); + + if ( reject ) + indent_puts( "*yy_state_ptr++ = yy_current_state;" ); + } + + +/* Generate the code to make a NUL transition. */ + +void gen_NUL_trans() + { /* NOTE - changes in here should be reflected in gen_next_match() */ + /* Only generate a definition for "yy_cp" if we'll generate code + * that uses it. Otherwise lint and the like complain. + */ + int need_backing_up = (num_backing_up > 0 && ! reject); + + if ( need_backing_up && (! nultrans || fullspd || fulltbl) ) + /* We're going to need yy_cp lying around for the call + * below to gen_backing_up(). + */ + indent_puts( "char *yy_cp = yy_c_buf_p;" ); + + outc( '\n' ); + + if ( nultrans ) + { + indent_puts( + "yy_current_state = yy_NUL_trans[yy_current_state];" ); + indent_puts( "yy_is_jam = (yy_current_state == 0);" ); + } + + else if ( fulltbl ) + { + do_indent(); + out_dec( "yy_current_state = yy_nxt[yy_current_state][%d];\n", + NUL_ec ); + indent_puts( "yy_is_jam = (yy_current_state <= 0);" ); + } + + else if ( fullspd ) + { + do_indent(); + out_dec( "int yy_c = %d;\n", NUL_ec ); + + indent_puts( + "yyconst struct yy_trans_info *yy_trans_info;\n" ); + indent_puts( + "yy_trans_info = &yy_current_state[(unsigned int) yy_c];" ); + indent_puts( "yy_current_state += yy_trans_info->yy_nxt;" ); + + indent_puts( + "yy_is_jam = (yy_trans_info->yy_verify != yy_c);" ); + } + + else + { + char NUL_ec_str[20]; + + (void) sprintf( NUL_ec_str, "%d", NUL_ec ); + gen_next_compressed_state( NUL_ec_str ); + + do_indent(); + out_dec( "yy_is_jam = (yy_current_state == %d);\n", jamstate ); + + if ( reject ) + { + /* Only stack this state if it's a transition we + * actually make. If we stack it on a jam, then + * the state stack and yy_c_buf_p get out of sync. + */ + indent_puts( "if ( ! yy_is_jam )" ); + indent_up(); + indent_puts( "*yy_state_ptr++ = yy_current_state;" ); + indent_down(); + } + } + + /* If we've entered an accepting state, back up; note that + * compressed tables have *already* done such backing up, so + * we needn't bother with it again. + */ + if ( need_backing_up && (fullspd || fulltbl) ) + { + outc( '\n' ); + indent_puts( "if ( ! yy_is_jam )" ); + indent_up(); + indent_puts( "{" ); + gen_backing_up(); + indent_puts( "}" ); + indent_down(); + } + } + + +/* Generate the code to find the start state. */ + +void gen_start_state() + { + if ( fullspd ) + { + if ( bol_needed ) + { + indent_puts( + "yy_current_state = yy_start_state_list[yy_start + YY_AT_BOL()];" ); + } + else + indent_puts( + "yy_current_state = yy_start_state_list[yy_start];" ); + } + + else + { + indent_puts( "yy_current_state = yy_start;" ); + + if ( bol_needed ) + indent_puts( "yy_current_state += YY_AT_BOL();" ); + + if ( reject ) + { + /* Set up for storing up states. */ + indent_puts( "yy_state_ptr = yy_state_buf;" ); + indent_puts( "*yy_state_ptr++ = yy_current_state;" ); + } + } + } + + +/* gentabs - generate data statements for the transition tables */ + +void gentabs() + { + int i, j, k, *accset, nacc, *acc_array, total_states; + int end_of_buffer_action = num_rules + 1; + + acc_array = allocate_integer_array( current_max_dfas ); + nummt = 0; + + /* The compressed table format jams by entering the "jam state", + * losing information about the previous state in the process. + * In order to recover the previous state, we effectively need + * to keep backing-up information. + */ + ++num_backing_up; + + if ( reject ) + { + /* Write out accepting list and pointer list. + * + * First we generate the "yy_acclist" array. In the process, + * we compute the indices that will go into the "yy_accept" + * array, and save the indices in the dfaacc array. + */ + int EOB_accepting_list[2]; + + /* Set up accepting structures for the End Of Buffer state. */ + EOB_accepting_list[0] = 0; + EOB_accepting_list[1] = end_of_buffer_action; + accsiz[end_of_buffer_state] = 1; + dfaacc[end_of_buffer_state].dfaacc_set = EOB_accepting_list; + + out_str_dec( long_align ? C_long_decl : C_short_decl, + "yy_acclist", MAX( numas, 1 ) + 1 ); + + j = 1; /* index into "yy_acclist" array */ + + for ( i = 1; i <= lastdfa; ++i ) + { + acc_array[i] = j; + + if ( accsiz[i] != 0 ) + { + accset = dfaacc[i].dfaacc_set; + nacc = accsiz[i]; + + if ( trace ) + fprintf( stderr, + _( "state # %d accepts: " ), + i ); + + for ( k = 1; k <= nacc; ++k ) + { + int accnum = accset[k]; + + ++j; + + if ( variable_trailing_context_rules && + ! (accnum & YY_TRAILING_HEAD_MASK) && + accnum > 0 && accnum <= num_rules && + rule_type[accnum] == RULE_VARIABLE ) + { + /* Special hack to flag + * accepting number as part + * of trailing context rule. + */ + accnum |= YY_TRAILING_MASK; + } + + mkdata( accnum ); + + if ( trace ) + { + fprintf( stderr, "[%d]", + accset[k] ); + + if ( k < nacc ) + fputs( ", ", stderr ); + else + putc( '\n', stderr ); + } + } + } + } + + /* add accepting number for the "jam" state */ + acc_array[i] = j; + + dataend(); + } + + else + { + dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action; + + for ( i = 1; i <= lastdfa; ++i ) + acc_array[i] = dfaacc[i].dfaacc_state; + + /* add accepting number for jam state */ + acc_array[i] = 0; + } + + /* Spit out "yy_accept" array. If we're doing "reject", it'll be + * pointers into the "yy_acclist" array. Otherwise it's actual + * accepting numbers. In either case, we just dump the numbers. + */ + + /* "lastdfa + 2" is the size of "yy_accept"; includes room for C arrays + * beginning at 0 and for "jam" state. + */ + k = lastdfa + 2; + + if ( reject ) + /* We put a "cap" on the table associating lists of accepting + * numbers with state numbers. This is needed because we tell + * where the end of an accepting list is by looking at where + * the list for the next state starts. + */ + ++k; + + out_str_dec( long_align ? C_long_decl : C_short_decl, "yy_accept", k ); + + for ( i = 1; i <= lastdfa; ++i ) + { + mkdata( acc_array[i] ); + + if ( ! reject && trace && acc_array[i] ) + fprintf( stderr, _( "state # %d accepts: [%d]\n" ), + i, acc_array[i] ); + } + + /* Add entry for "jam" state. */ + mkdata( acc_array[i] ); + + if ( reject ) + /* Add "cap" for the list. */ + mkdata( acc_array[i] ); + + dataend(); + + if ( useecs ) + genecs(); + + if ( usemecs ) + { + /* Write out meta-equivalence classes (used to index + * templates with). + */ + + if ( trace ) + fputs( _( "\n\nMeta-Equivalence Classes:\n" ), + stderr ); + + out_str_dec( C_int_decl, "yy_meta", numecs + 1 ); + + for ( i = 1; i <= numecs; ++i ) + { + if ( trace ) + fprintf( stderr, "%d = %d\n", + i, ABS( tecbck[i] ) ); + + mkdata( ABS( tecbck[i] ) ); + } + + dataend(); + } + + total_states = lastdfa + numtemps; + + out_str_dec( (tblend >= MAX_SHORT || long_align) ? + C_long_decl : C_short_decl, + "yy_base", total_states + 1 ); + + for ( i = 1; i <= lastdfa; ++i ) + { + int d = def[i]; + + if ( base[i] == JAMSTATE ) + base[i] = jambase; + + if ( d == JAMSTATE ) + def[i] = jamstate; + + else if ( d < 0 ) + { + /* Template reference. */ + ++tmpuses; + def[i] = lastdfa - d + 1; + } + + mkdata( base[i] ); + } + + /* Generate jam state's base index. */ + mkdata( base[i] ); + + for ( ++i /* skip jam state */; i <= total_states; ++i ) + { + mkdata( base[i] ); + def[i] = jamstate; + } + + dataend(); + + out_str_dec( (total_states >= MAX_SHORT || long_align) ? + C_long_decl : C_short_decl, + "yy_def", total_states + 1 ); + + for ( i = 1; i <= total_states; ++i ) + mkdata( def[i] ); + + dataend(); + + out_str_dec( (total_states >= MAX_SHORT || long_align) ? + C_long_decl : C_short_decl, + "yy_nxt", tblend + 1 ); + + for ( i = 1; i <= tblend; ++i ) + { + /* Note, the order of the following test is important. + * If chk[i] is 0, then nxt[i] is undefined. + */ + if ( chk[i] == 0 || nxt[i] == 0 ) + nxt[i] = jamstate; /* new state is the JAM state */ + + mkdata( nxt[i] ); + } + + dataend(); + + out_str_dec( (total_states >= MAX_SHORT || long_align) ? + C_long_decl : C_short_decl, + "yy_chk", tblend + 1 ); + + for ( i = 1; i <= tblend; ++i ) + { + if ( chk[i] == 0 ) + ++nummt; + + mkdata( chk[i] ); + } + + dataend(); + } + + +/* Write out a formatted string (with a secondary string argument) at the + * current indentation level, adding a final newline. + */ + +void indent_put2s( fmt, arg ) +char fmt[], arg[]; + { + do_indent(); + out_str( fmt, arg ); + outn( "" ); + } + + +/* Write out a string at the current indentation level, adding a final + * newline. + */ + +void indent_puts( str ) +char str[]; + { + do_indent(); + outn( str ); + } + + +/* make_tables - generate transition tables and finishes generating output file + */ + +void make_tables() + { + int i; + int did_eof_rule = false; + + skelout(); + + /* First, take care of YY_DO_BEFORE_ACTION depending on yymore + * being used. + */ + set_indent( 1 ); + + if ( yymore_used && ! yytext_is_array ) + { + indent_puts( "yytext_ptr -= yy_more_len; \\" ); + indent_puts( "yyleng = (int) (yy_cp - yytext_ptr); \\" ); + } + + else + indent_puts( "yyleng = (int) (yy_cp - yy_bp); \\" ); + + /* Now also deal with copying yytext_ptr to yytext if needed. */ + skelout(); + if ( yytext_is_array ) + { + if ( yymore_used ) + indent_puts( + "if ( yyleng + yy_more_offset >= YYLMAX ) \\" ); + else + indent_puts( "if ( yyleng >= YYLMAX ) \\" ); + + indent_up(); + indent_puts( + "YY_FATAL_ERROR( \"token too large, exceeds YYLMAX\" ); \\" ); + indent_down(); + + if ( yymore_used ) + { + indent_puts( +"yy_flex_strncpy( &yytext[yy_more_offset], yytext_ptr, yyleng + 1 ); \\" ); + indent_puts( "yyleng += yy_more_offset; \\" ); + indent_puts( + "yy_prev_more_offset = yy_more_offset; \\" ); + indent_puts( "yy_more_offset = 0; \\" ); + } + else + { + indent_puts( + "yy_flex_strncpy( yytext, yytext_ptr, yyleng + 1 ); \\" ); + } + } + + set_indent( 0 ); + + skelout(); + + + out_dec( "#define YY_NUM_RULES %d\n", num_rules ); + out_dec( "#define YY_END_OF_BUFFER %d\n", num_rules + 1 ); + + if ( fullspd ) + { + /* Need to define the transet type as a size large + * enough to hold the biggest offset. + */ + int total_table_size = tblend + numecs + 1; + char *trans_offset_type = + (total_table_size >= MAX_SHORT || long_align) ? + "long" : "short"; + + set_indent( 0 ); + indent_puts( "struct yy_trans_info" ); + indent_up(); + indent_puts( "{" ); /* } for vi */ + + if ( long_align ) + indent_puts( "long yy_verify;" ); + else + indent_puts( "short yy_verify;" ); + + /* In cases where its sister yy_verify *is* a "yes, there is + * a transition", yy_nxt is the offset (in records) to the + * next state. In most cases where there is no transition, + * the value of yy_nxt is irrelevant. If yy_nxt is the -1th + * record of a state, though, then yy_nxt is the action number + * for that state. + */ + + indent_put2s( "%s yy_nxt;", trans_offset_type ); + indent_puts( "};" ); + indent_down(); + } + + if ( fullspd ) + genctbl(); + else if ( fulltbl ) + genftbl(); + else + gentabs(); + + /* Definitions for backing up. We don't need them if REJECT + * is being used because then we use an alternative backin-up + * technique instead. + */ + if ( num_backing_up > 0 && ! reject ) + { + if ( ! C_plus_plus ) + { + indent_puts( + "static yy_state_type yy_last_accepting_state;" ); + indent_puts( + "static char *yy_last_accepting_cpos;\n" ); + } + } + + if ( nultrans ) + { + out_str_dec( C_state_decl, "yy_NUL_trans", lastdfa + 1 ); + + for ( i = 1; i <= lastdfa; ++i ) + { + if ( fullspd ) + out_dec( " &yy_transition[%d],\n", base[i] ); + else + mkdata( nultrans[i] ); + } + + dataend(); + } + + if ( ddebug ) + { /* Spit out table mapping rules to line numbers. */ + if ( ! C_plus_plus ) + { + indent_puts( "extern int yy_flex_debug;" ); + indent_puts( "int yy_flex_debug = 1;\n" ); + } + + out_str_dec( long_align ? C_long_decl : C_short_decl, + "yy_rule_linenum", num_rules ); + for ( i = 1; i < num_rules; ++i ) + mkdata( rule_linenum[i] ); + dataend(); + } + + if ( reject ) + { + /* Declare state buffer variables. */ + if ( ! C_plus_plus ) + { + outn( + "static yy_state_type yy_state_buf[YY_BUF_SIZE + 2], *yy_state_ptr;" ); + outn( "static char *yy_full_match;" ); + outn( "static int yy_lp;" ); + } + + if ( variable_trailing_context_rules ) + { + if ( ! C_plus_plus ) + { + outn( + "static int yy_looking_for_trail_begin = 0;" ); + outn( "static int yy_full_lp;" ); + outn( "static int *yy_full_state;" ); + } + + out_hex( "#define YY_TRAILING_MASK 0x%x\n", + (unsigned int) YY_TRAILING_MASK ); + out_hex( "#define YY_TRAILING_HEAD_MASK 0x%x\n", + (unsigned int) YY_TRAILING_HEAD_MASK ); + } + + outn( "#define REJECT \\" ); + outn( "{ \\" ); /* } for vi */ + outn( + "*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ \\" ); + outn( + "yy_cp = yy_full_match; /* restore poss. backed-over text */ \\" ); + + if ( variable_trailing_context_rules ) + { + outn( + "yy_lp = yy_full_lp; /* restore orig. accepting pos. */ \\" ); + outn( + "yy_state_ptr = yy_full_state; /* restore orig. state */ \\" ); + outn( + "yy_current_state = *yy_state_ptr; /* restore curr. state */ \\" ); + } + + outn( "++yy_lp; \\" ); + outn( "goto find_rule; \\" ); + /* { for vi */ + outn( "}" ); + } + + else + { + outn( + "/* The intent behind this definition is that it'll catch" ); + outn( " * any uses of REJECT which flex missed." ); + outn( " */" ); + outn( "#define REJECT reject_used_but_not_detected" ); + } + + if ( yymore_used ) + { + if ( ! C_plus_plus ) + { + if ( yytext_is_array ) + { + indent_puts( "static int yy_more_offset = 0;" ); + indent_puts( + "static int yy_prev_more_offset = 0;" ); + } + else + { + indent_puts( "static int yy_more_flag = 0;" ); + indent_puts( "static int yy_more_len = 0;" ); + } + } + + if ( yytext_is_array ) + { + indent_puts( + "#define yymore() (yy_more_offset = yy_flex_strlen( yytext ))" ); + indent_puts( "#define YY_NEED_STRLEN" ); + indent_puts( "#define YY_MORE_ADJ 0" ); + indent_puts( "#define YY_RESTORE_YY_MORE_OFFSET \\" ); + indent_up(); + indent_puts( "{ \\" ); + indent_puts( "yy_more_offset = yy_prev_more_offset; \\" ); + indent_puts( "yyleng -= yy_more_offset; \\" ); + indent_puts( "}" ); + indent_down(); + } + else + { + indent_puts( "#define yymore() (yy_more_flag = 1)" ); + indent_puts( "#define YY_MORE_ADJ yy_more_len" ); + indent_puts( "#define YY_RESTORE_YY_MORE_OFFSET" ); + } + } + + else + { + indent_puts( "#define yymore() yymore_used_but_not_detected" ); + indent_puts( "#define YY_MORE_ADJ 0" ); + indent_puts( "#define YY_RESTORE_YY_MORE_OFFSET" ); + } + + if ( ! C_plus_plus ) + { + if ( yytext_is_array ) + { + outn( "#ifndef YYLMAX" ); + outn( "#define YYLMAX 8192" ); + outn( "#endif\n" ); + outn( "char yytext[YYLMAX];" ); + outn( "char *yytext_ptr;" ); + } + + else + outn( "char *yytext;" ); + } + + out( &action_array[defs1_offset] ); + + line_directive_out( stdout, 0 ); + + skelout(); + + if ( ! C_plus_plus ) + { + if ( use_read ) + { + outn( +"\tif ( (result = read( fileno(yyin), (char *) buf, max_size )) < 0 ) \\" ); + outn( + "\t\tYY_FATAL_ERROR( \"input in flex scanner failed\" );" ); + } + + else + { + outn( + "\tif ( yy_current_buffer->yy_is_interactive ) \\" ); + outn( "\t\t{ \\" ); + outn( "\t\tint c = '*', n; \\" ); + outn( "\t\tfor ( n = 0; n < max_size && \\" ); + outn( "\t\t\t (c = getc( yyin )) != EOF && c != '\\n'; ++n ) \\" ); + outn( "\t\t\tbuf[n] = (char) c; \\" ); + outn( "\t\tif ( c == '\\n' ) \\" ); + outn( "\t\t\tbuf[n++] = (char) c; \\" ); + outn( "\t\tif ( c == EOF && ferror( yyin ) ) \\" ); + outn( + "\t\t\tYY_FATAL_ERROR( \"input in flex scanner failed\" ); \\" ); + outn( "\t\tresult = n; \\" ); + outn( "\t\t} \\" ); + outn( + "\telse if ( ((result = fread( buf, 1, max_size, yyin )) == 0) \\" ); + outn( "\t\t && ferror( yyin ) ) \\" ); + outn( + "\t\tYY_FATAL_ERROR( \"input in flex scanner failed\" );" ); + } + } + + skelout(); + + indent_puts( "#define YY_RULE_SETUP \\" ); + indent_up(); + if ( bol_needed ) + { + indent_puts( "if ( yyleng > 0 ) \\" ); + indent_up(); + indent_puts( "yy_current_buffer->yy_at_bol = \\" ); + indent_puts( "\t\t(yytext[yyleng - 1] == '\\n'); \\" ); + indent_down(); + } + indent_puts( "YY_USER_ACTION" ); + indent_down(); + + skelout(); + + /* Copy prolog to output file. */ + out( &action_array[prolog_offset] ); + + line_directive_out( stdout, 0 ); + + skelout(); + + set_indent( 2 ); + + if ( yymore_used && ! yytext_is_array ) + { + indent_puts( "yy_more_len = 0;" ); + indent_puts( "if ( yy_more_flag )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_more_len = yy_c_buf_p - yytext_ptr;" ); + indent_puts( "yy_more_flag = 0;" ); + indent_puts( "}" ); + indent_down(); + } + + skelout(); + + gen_start_state(); + + /* Note, don't use any indentation. */ + outn( "yy_match:" ); + gen_next_match(); + + skelout(); + set_indent( 2 ); + gen_find_action(); + + skelout(); + if ( do_yylineno ) + { + indent_puts( "if ( yy_act != YY_END_OF_BUFFER )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "int yyl;" ); + indent_puts( "for ( yyl = 0; yyl < yyleng; ++yyl )" ); + indent_up(); + indent_puts( "if ( yytext[yyl] == '\\n' )" ); + indent_up(); + indent_puts( "++yylineno;" ); + indent_down(); + indent_down(); + indent_puts( "}" ); + indent_down(); + } + + skelout(); + if ( ddebug ) + { + indent_puts( "if ( yy_flex_debug )" ); + indent_up(); + + indent_puts( "{" ); + indent_puts( "if ( yy_act == 0 )" ); + indent_up(); + indent_puts( C_plus_plus ? + "cerr << \"--scanner backing up\\n\";" : + "fprintf( stderr, \"--scanner backing up\\n\" );" ); + indent_down(); + + do_indent(); + out_dec( "else if ( yy_act < %d )\n", num_rules ); + indent_up(); + + if ( C_plus_plus ) + { + indent_puts( + "cerr << \"--accepting rule at line \" << yy_rule_linenum[yy_act] <<" ); + indent_puts( + " \"(\\\"\" << yytext << \"\\\")\\n\";" ); + } + else + { + indent_puts( + "fprintf( stderr, \"--accepting rule at line %d (\\\"%s\\\")\\n\"," ); + + indent_puts( + " yy_rule_linenum[yy_act], yytext );" ); + } + + indent_down(); + + do_indent(); + out_dec( "else if ( yy_act == %d )\n", num_rules ); + indent_up(); + + if ( C_plus_plus ) + { + indent_puts( +"cerr << \"--accepting default rule (\\\"\" << yytext << \"\\\")\\n\";" ); + } + else + { + indent_puts( + "fprintf( stderr, \"--accepting default rule (\\\"%s\\\")\\n\"," ); + indent_puts( " yytext );" ); + } + + indent_down(); + + do_indent(); + out_dec( "else if ( yy_act == %d )\n", num_rules + 1 ); + indent_up(); + + indent_puts( C_plus_plus ? + "cerr << \"--(end of buffer or a NUL)\\n\";" : + "fprintf( stderr, \"--(end of buffer or a NUL)\\n\" );" ); + + indent_down(); + + do_indent(); + outn( "else" ); + indent_up(); + + if ( C_plus_plus ) + { + indent_puts( + "cerr << \"--EOF (start condition \" << YY_START << \")\\n\";" ); + } + else + { + indent_puts( + "fprintf( stderr, \"--EOF (start condition %d)\\n\", YY_START );" ); + } + + indent_down(); + + indent_puts( "}" ); + indent_down(); + } + + /* Copy actions to output file. */ + skelout(); + indent_up(); + gen_bu_action(); + out( &action_array[action_offset] ); + + line_directive_out( stdout, 0 ); + + /* generate cases for any missing EOF rules */ + for ( i = 1; i <= lastsc; ++i ) + if ( ! sceof[i] ) + { + do_indent(); + out_str( "case YY_STATE_EOF(%s):\n", scname[i] ); + did_eof_rule = true; + } + + if ( did_eof_rule ) + { + indent_up(); + indent_puts( "yyterminate();" ); + indent_down(); + } + + + /* Generate code for handling NUL's, if needed. */ + + /* First, deal with backing up and setting up yy_cp if the scanner + * finds that it should JAM on the NUL. + */ + skelout(); + set_indent( 4 ); + + if ( fullspd || fulltbl ) + indent_puts( "yy_cp = yy_c_buf_p;" ); + + else + { /* compressed table */ + if ( ! reject && ! interactive ) + { + /* Do the guaranteed-needed backing up to figure + * out the match. + */ + indent_puts( "yy_cp = yy_last_accepting_cpos;" ); + indent_puts( + "yy_current_state = yy_last_accepting_state;" ); + } + + else + /* Still need to initialize yy_cp, though + * yy_current_state was set up by + * yy_get_previous_state(). + */ + indent_puts( "yy_cp = yy_c_buf_p;" ); + } + + + /* Generate code for yy_get_previous_state(). */ + set_indent( 1 ); + skelout(); + + gen_start_state(); + + set_indent( 2 ); + skelout(); + gen_next_state( true ); + + set_indent( 1 ); + skelout(); + gen_NUL_trans(); + + skelout(); + if ( do_yylineno ) + { /* update yylineno inside of unput() */ + indent_puts( "if ( c == '\\n' )" ); + indent_up(); + indent_puts( "--yylineno;" ); + indent_down(); + } + + skelout(); + /* Update BOL and yylineno inside of input(). */ + if ( bol_needed ) + { + indent_puts( "yy_current_buffer->yy_at_bol = (c == '\\n');" ); + if ( do_yylineno ) + { + indent_puts( "if ( yy_current_buffer->yy_at_bol )" ); + indent_up(); + indent_puts( "++yylineno;" ); + indent_down(); + } + } + + else if ( do_yylineno ) + { + indent_puts( "if ( c == '\\n' )" ); + indent_up(); + indent_puts( "++yylineno;" ); + indent_down(); + } + + skelout(); + + /* Copy remainder of input to output. */ + + line_directive_out( stdout, 1 ); + + if ( sectnum == 3 ) + (void) flexscan(); /* copy remainder of input to output */ + } diff --git a/usr.bin/lex/initscan.c b/usr.bin/lex/initscan.c new file mode 100644 index 0000000..7620c52 --- /dev/null +++ b/usr.bin/lex/initscan.c @@ -0,0 +1,3728 @@ +#line 2 "scan.c" +/* A lexical scanner generated by flex */ + +/* Scanner skeleton version: + * $Header: /home/daffy/u0/vern/flex/RCS/flex.skl,v 2.91 96/09/10 16:58:48 vern Exp $ + * $FreeBSD$ + */ + +#if defined(__FreeBSD__) +#include <sys/cdefs.h> +#else +#define __unused +#define __dead2 +#endif + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 5 + +#include <stdio.h> + + +/* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */ +#ifdef c_plusplus +#ifndef __cplusplus +#define __cplusplus +#endif +#endif + + +#ifdef __cplusplus + +#include <stdlib.h> +#include <unistd.h> + +/* Use prototypes in function declarations. */ +#define YY_USE_PROTOS + +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST + +#else /* ! __cplusplus */ + +#if __STDC__ + +#define YY_USE_PROTOS +#define YY_USE_CONST + +#endif /* __STDC__ */ +#endif /* ! __cplusplus */ + +#ifdef __TURBOC__ + #pragma warn -rch + #pragma warn -use +#include <io.h> +#include <stdlib.h> +#define YY_USE_CONST +#define YY_USE_PROTOS +#endif + +#ifdef YY_USE_CONST +#define yyconst const +#else +#define yyconst +#endif + + +#ifdef YY_USE_PROTOS +#define YY_PROTO(proto) proto +#else +#define YY_PROTO(proto) () +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. + */ +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN yy_start = 1 + 2 * + +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START ((yy_start - 1) / 2) +#define YYSTATE YY_START + +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) + +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#define YY_BUF_SIZE 16384 + +typedef struct yy_buffer_state *YY_BUFFER_STATE; + +extern int yyleng; +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + +/* The funky do-while in the following #define is used to turn the definition + * int a single C statement (which needs a semi-colon terminator). This + * avoids problems with code like: + * + * if ( condition_holds ) + * yyless( 5 ); + * else + * do_something_else(); + * + * Prior to using the do-while the compiler would get upset at the + * "else" because it interpreted the "if" statement as being all + * done when it reached the ';' after the yyless() call. + */ + +/* Return all but the first 'n' matched characters back to the input stream. */ + +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + *yy_cp = yy_hold_char; \ + YY_RESTORE_YY_MORE_OFFSET \ + yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) + +#define unput(c) yyunput( c, yytext_ptr ) + +/* The following is because we cannot portably get our hands on size_t + * (without autoconf's help, which isn't available because we want + * flex-generated scanners to compile on their own). + */ +typedef unsigned int yy_size_t; + + +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + yy_size_t yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + }; + +static YY_BUFFER_STATE yy_current_buffer = 0; + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + */ +#define YY_CURRENT_BUFFER yy_current_buffer + + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; + +static int yy_n_chars; /* number of characters read into yy_ch_buf */ + + +int yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = (char *) 0; +static int yy_init = 1; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart YY_PROTO(( FILE *input_file )); + +void yy_switch_to_buffer YY_PROTO(( YY_BUFFER_STATE new_buffer )); +void yy_load_buffer_state YY_PROTO(( void )); +YY_BUFFER_STATE yy_create_buffer YY_PROTO(( FILE *file, int size )); +void yy_delete_buffer YY_PROTO(( YY_BUFFER_STATE b )); +void yy_init_buffer YY_PROTO(( YY_BUFFER_STATE b, FILE *file )); +void yy_flush_buffer YY_PROTO(( YY_BUFFER_STATE b )); +#define YY_FLUSH_BUFFER yy_flush_buffer( yy_current_buffer ) + +YY_BUFFER_STATE yy_scan_buffer YY_PROTO(( char *base, yy_size_t size )); +YY_BUFFER_STATE yy_scan_string YY_PROTO(( yyconst char *yy_str )); +YY_BUFFER_STATE yy_scan_bytes YY_PROTO(( yyconst char *bytes, int len )); + +static void *yy_flex_alloc YY_PROTO(( yy_size_t )); +static void *yy_flex_realloc YY_PROTO(( void *, yy_size_t )) __unused; +static void yy_flex_free YY_PROTO(( void * )); + +#define yy_new_buffer yy_create_buffer + +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! yy_current_buffer ) \ + yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ + yy_current_buffer->yy_is_interactive = is_interactive; \ + } + +#define yy_set_bol(at_bol) \ + { \ + if ( ! yy_current_buffer ) \ + yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ + yy_current_buffer->yy_at_bol = at_bol; \ + } + +#define YY_AT_BOL() (yy_current_buffer->yy_at_bol) + +typedef unsigned char YY_CHAR; +FILE *yyin = (FILE *) 0, *yyout = (FILE *) 0; +typedef int yy_state_type; +extern char *yytext; +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state YY_PROTO(( void )); +static yy_state_type yy_try_NUL_trans YY_PROTO(( yy_state_type current_state )); +static int yy_get_next_buffer YY_PROTO(( void )); +static void yy_fatal_error YY_PROTO(( yyconst char msg[] )) __dead2; + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + yytext_ptr = yy_bp; \ + yyleng = (int) (yy_cp - yy_bp); \ + yy_hold_char = *yy_cp; \ + *yy_cp = '\0'; \ + yy_c_buf_p = yy_cp; + +#define YY_NUM_RULES 165 +#define YY_END_OF_BUFFER 166 +static yyconst short int yy_accept[769] = + { 0, + 0, 0, 0, 0, 87, 87, 163, 163, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 166, 164, + 7, 18, 164, 16, 1, 17, 164, 164, 164, 164, + 15, 108, 100, 101, 108, 93, 108, 107, 108, 108, + 108, 107, 99, 89, 108, 108, 91, 92, 87, 88, + 87, 86, 85, 86, 86, 163, 163, 28, 29, 28, + 28, 28, 28, 28, 28, 31, 30, 32, 31, 113, + 109, 110, 112, 114, 141, 142, 141, 139, 138, 140, + + 115, 117, 115, 116, 115, 120, 120, 120, 120, 122, + 124, 122, 122, 122, 122, 123, 151, 155, 151, 154, + 156, 156, 152, 152, 152, 149, 150, 164, 82, 164, + 21, 22, 21, 20, 157, 159, 157, 160, 161, 147, + 147, 148, 147, 147, 147, 147, 147, 147, 147, 81, + 34, 33, 81, 81, 81, 81, 35, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 26, 23, 26, 24, + 7, 18, 0, 16, 1, 17, 0, 0, 0, 14, + 8, 0, 0, 0, 0, 4, 5, 0, 2, 15, + + 100, 101, 0, 0, 0, 95, 0, 0, 105, 105, + 0, 162, 162, 162, 94, 0, 99, 89, 0, 0, + 0, 91, 92, 104, 90, 0, 87, 88, 86, 85, + 85, 83, 84, 163, 163, 28, 29, 28, 28, 28, + 28, 31, 30, 32, 111, 112, 142, 138, 117, 0, + 118, 119, 124, 121, 151, 155, 0, 153, 0, 144, + 152, 152, 152, 0, 82, 0, 21, 22, 21, 19, + 157, 159, 158, 147, 147, 147, 148, 143, 147, 147, + 147, 34, 33, 0, 80, 0, 0, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + + 81, 81, 81, 36, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 0, 25, 24, 0, 14, 8, + 0, 12, 0, 0, 0, 0, 0, 4, 5, 0, + 6, 0, 96, 0, 97, 0, 0, 105, 105, 0, + 105, 105, 105, 162, 162, 0, 106, 90, 98, 0, + 104, 0, 83, 84, 28, 28, 28, 27, 28, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 152, 152, 143, 143, 147, 147, 0, 0, 81, + 81, 81, 81, 81, 44, 81, 81, 81, 49, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + + 81, 81, 81, 81, 81, 81, 81, 81, 0, 81, + 81, 81, 81, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 0, 4, 5, 0, 105, 105, 105, 105, + 105, 105, 162, 0, 0, 28, 28, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 152, 152, 147, 147, 37, 38, 81, 81, 81, 81, + 81, 81, 81, 81, 50, 51, 81, 81, 81, 55, + 81, 81, 81, 81, 81, 81, 60, 81, 81, 81, + 81, 81, 81, 67, 0, 0, 0, 81, 81, 81, + 81, 0, 13, 0, 0, 0, 0, 0, 0, 105, + + 105, 105, 105, 105, 105, 0, 0, 28, 28, 137, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 152, 152, 147, 147, 39, 81, 41, 81, + 43, 81, 81, 81, 47, 81, 52, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 62, 81, 81, 65, + 81, 0, 0, 0, 0, 81, 81, 81, 81, 3, + 0, 0, 0, 0, 105, 105, 105, 0, 0, 28, + 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 145, 146, 145, 146, 81, 42, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + + 81, 78, 61, 81, 64, 81, 0, 0, 0, 0, + 81, 81, 69, 70, 0, 10, 0, 11, 0, 103, + 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 81, 81, 81, 45, 81, 48, + 81, 81, 81, 81, 77, 81, 59, 63, 66, 0, + 0, 0, 0, 79, 81, 0, 102, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, + 81, 81, 46, 81, 81, 56, 81, 81, 0, 0, + 0, 0, 68, 0, 9, 0, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 135, 0, 81, 81, + + 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, + 136, 81, 81, 81, 81, 54, 81, 81, 0, 0, + 0, 0, 0, 0, 81, 81, 81, 53, 81, 58, + 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, + 72, 0, 0, 0, 0, 73, 81, 81, 81, 81, + 71, 0, 75, 0, 81, 81, 81, 74, 76, 81, + 81, 81, 81, 81, 81, 57, 40, 0 + } ; + +static yyconst int yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, + 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 5, 6, 7, 8, 1, 9, 10, + 10, 11, 12, 13, 14, 10, 15, 16, 16, 16, + 16, 16, 16, 16, 17, 18, 19, 20, 1, 21, + 22, 23, 10, 1, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 47, + 26, 27, 28, 29, 30, 1, 31, 32, 33, 34, + + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 47, 56, 57, 58, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + } ; + +static yyconst int yy_meta[59] = + { 0, + 1, 1, 2, 1, 3, 1, 1, 1, 4, 1, + 5, 6, 1, 7, 4, 8, 8, 8, 8, 1, + 1, 1, 1, 9, 10, 1, 11, 12, 1, 13, + 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 4, 1, 16 + } ; + +static yyconst short int yy_base[858] = + { 0, + 0, 58, 115, 172, 120, 129, 2712, 2711, 230, 2705, + 136, 141, 288, 0, 2683, 2682, 144, 151, 185, 191, + 178, 188, 344, 347, 375, 0, 125, 131, 147, 216, + 431, 434, 461, 0, 519, 0, 205, 349, 2710, 2716, + 353, 2716, 2706, 0, 360, 2716, 2705, 144, 570, 2696, + 0, 2716, 577, 2716, 2703, 2716, 438, 2716, 2684, 126, + 149, 427, 591, 2716, 2701, 141, 2682, 2716, 0, 2716, + 2699, 0, 2699, 2697, 155, 2696, 2716, 0, 2716, 2695, + 2716, 0, 2662, 2641, 2637, 0, 2692, 2716, 2690, 2716, + 2716, 2663, 0, 2716, 2716, 2716, 2688, 2716, 431, 2716, + + 2716, 2716, 2687, 2716, 567, 2716, 2669, 571, 164, 2716, + 2716, 2685, 0, 2667, 573, 2716, 0, 2716, 2683, 2716, + 573, 2674, 0, 2649, 2628, 2716, 2716, 222, 2716, 356, + 448, 2716, 450, 2667, 0, 2716, 2678, 2716, 0, 0, + 198, 2716, 2677, 2621, 2716, 2667, 0, 2642, 2621, 2716, + 2673, 2716, 2671, 2668, 2640, 2639, 2716, 544, 2639, 579, + 2634, 2635, 318, 0, 2623, 2631, 424, 562, 2614, 587, + 2629, 2613, 2618, 2626, 2629, 2604, 2716, 2716, 2653, 612, + 634, 2716, 2654, 0, 637, 2716, 2653, 600, 2616, 0, + 0, 641, 647, 651, 669, 0, 0, 453, 2716, 0, + + 672, 2716, 2651, 2597, 605, 2716, 2649, 2616, 620, 657, + 645, 2716, 662, 0, 2716, 2592, 688, 2716, 2646, 2592, + 2636, 2625, 2716, 0, 2716, 2610, 0, 2716, 0, 0, + 2642, 0, 0, 2640, 2716, 0, 2716, 0, 2602, 2598, + 745, 0, 2638, 2716, 2716, 0, 2716, 688, 2716, 773, + 2716, 2716, 2716, 2716, 0, 2716, 673, 2716, 0, 2716, + 0, 2599, 2595, 690, 2716, 698, 707, 2716, 709, 2716, + 0, 2716, 2716, 0, 596, 2579, 2716, 827, 0, 2596, + 2592, 2632, 2716, 2628, 2716, 2593, 2592, 0, 642, 2582, + 563, 2617, 2579, 620, 2578, 2577, 2583, 669, 2570, 2584, + + 2572, 0, 2569, 2716, 2570, 2571, 2579, 2582, 685, 125, + 2570, 2567, 2566, 688, 2608, 2716, 716, 2568, 0, 0, + 720, 2716, 2608, 884, 2562, 2559, 2569, 0, 0, 723, + 2716, 739, 2716, 805, 2716, 808, 2562, 787, 869, 876, + 930, 881, 973, 800, 0, 2548, 2716, 2716, 2716, 2570, + 0, 2559, 0, 0, 2568, 2557, 0, 2716, 0, 1009, + 2581, 678, 870, 871, 874, 879, 913, 992, 974, 1013, + 885, 2565, 2554, 0, 1067, 2563, 2552, 2546, 2545, 2557, + 2562, 2561, 2550, 2557, 0, 2554, 2537, 2556, 0, 2536, + 2543, 2533, 2548, 2568, 2537, 2549, 2544, 2542, 2541, 2532, + + 2539, 2540, 2538, 2539, 578, 2520, 2538, 2525, 860, 2526, + 2528, 2521, 2517, 2529, 817, 1044, 2716, 822, 1095, 914, + 2532, 2523, 2517, 0, 0, 2524, 1102, 1025, 1142, 2539, + 1028, 1163, 2716, 2513, 2521, 2523, 2507, 0, 2526, 1058, + 891, 1014, 1019, 894, 1038, 1080, 1072, 1086, 1083, 1081, + 2520, 2504, 2518, 2502, 2716, 2716, 2505, 2493, 2492, 2495, + 2507, 1148, 2507, 2492, 0, 0, 2492, 2493, 2507, 0, + 2525, 2490, 2498, 2522, 2485, 2495, 0, 2500, 2491, 2487, + 2479, 2479, 2483, 0, 875, 2494, 2481, 2494, 2480, 2475, + 2491, 2519, 2716, 920, 999, 2465, 2474, 2468, 2494, 2496, + + 1105, 1184, 1081, 902, 969, 2479, 2491, 2463, 2477, 2716, + 165, 1090, 1144, 1143, 1147, 1163, 1095, 1145, 1037, 1085, + 1150, 1173, 2461, 2475, 2459, 2473, 0, 2458, 0, 2460, + 0, 1165, 2454, 2469, 0, 2461, 0, 2471, 2410, 2414, + 2434, 2400, 2393, 2405, 2385, 2382, 0, 2383, 2335, 0, + 2335, 2330, 2326, 2309, 2278, 2259, 2269, 2268, 2256, 2297, + 1046, 2238, 2242, 2253, 1179, 1142, 1145, 2247, 2246, 0, + 0, 1191, 1192, 1172, 1201, 1202, 1204, 1205, 1206, 1207, + 1209, 1210, 1208, 0, 0, 0, 0, 2254, 0, 2221, + 2229, 2218, 2208, 2200, 2209, 2198, 2195, 2165, 2168, 2149, + + 2132, 0, 0, 2129, 0, 2139, 2143, 2134, 2124, 2137, + 2117, 2116, 0, 0, 1228, 2716, 1232, 2716, 2111, 2716, + 2117, 2716, 2115, 2114, 2108, 2107, 2106, 2103, 2102, 2098, + 2095, 2063, 2047, 1213, 2012, 1986, 1975, 0, 1954, 0, + 1947, 1950, 1941, 1945, 0, 1942, 0, 0, 0, 1938, + 1940, 1934, 1905, 0, 1872, 1234, 2716, 1888, 1882, 1881, + 1864, 1848, 1832, 1828, 1827, 1826, 1823, 1806, 1809, 1784, + 1787, 1772, 0, 1781, 1786, 0, 1766, 1767, 1759, 1744, + 1213, 1736, 0, 1236, 2716, 1245, 2716, 2716, 2716, 2716, + 2716, 2716, 2716, 2716, 2716, 2716, 2716, 1750, 1727, 1720, + + 1701, 1687, 1670, 1681, 1667, 1679, 1659, 689, 1658, 1671, + 2716, 1657, 1627, 1621, 1635, 0, 1603, 1596, 1595, 1608, + 1602, 1587, 1586, 1583, 1581, 1587, 1555, 0, 1547, 0, + 1527, 1507, 1520, 1503, 1483, 1482, 1485, 1443, 1440, 1228, + 2716, 1225, 1224, 1206, 1210, 2716, 1213, 1202, 1018, 948, + 2716, 945, 2716, 884, 780, 771, 779, 2716, 2716, 689, + 673, 581, 408, 318, 86, 0, 0, 2716, 1263, 1279, + 1295, 1311, 1327, 1343, 1359, 1375, 1391, 1407, 1423, 1439, + 1455, 1471, 1481, 1496, 1505, 1520, 1536, 1545, 1560, 1576, + 1592, 1608, 1624, 1634, 1649, 1659, 1674, 1690, 1706, 1718, + + 1728, 1743, 1759, 1775, 1791, 1807, 1817, 1832, 1843, 1236, + 1858, 1874, 1890, 1898, 1905, 1920, 1936, 1952, 1968, 1977, + 1985, 2001, 2017, 2033, 2049, 2065, 2081, 2097, 2113, 2123, + 2138, 2148, 2155, 2170, 2182, 2192, 2207, 2223, 2239, 2255, + 2265, 2280, 2291, 2306, 2322, 2338, 2354, 2364, 2373, 2388, + 2404, 2420, 2429, 2437, 2453, 2469, 2485 + } ; + +static yyconst short int yy_def[858] = + { 0, + 768, 768, 769, 769, 770, 771, 772, 772, 768, 9, + 773, 773, 768, 13, 774, 774, 775, 775, 776, 776, + 777, 777, 778, 778, 768, 25, 779, 779, 780, 780, + 781, 781, 768, 33, 768, 35, 782, 782, 768, 768, + 768, 768, 768, 783, 768, 768, 768, 768, 784, 768, + 785, 768, 768, 768, 768, 768, 768, 768, 768, 786, + 787, 788, 768, 768, 768, 768, 768, 768, 789, 768, + 789, 790, 791, 790, 790, 792, 768, 793, 768, 793, + 768, 794, 794, 794, 793, 795, 768, 768, 795, 768, + 768, 768, 796, 768, 768, 768, 768, 768, 768, 768, + + 768, 768, 768, 768, 787, 768, 768, 787, 797, 768, + 768, 768, 798, 768, 787, 768, 799, 768, 799, 768, + 800, 768, 801, 801, 801, 768, 768, 802, 768, 802, + 803, 768, 803, 768, 804, 768, 804, 768, 805, 806, + 806, 768, 806, 806, 768, 806, 807, 807, 807, 768, + 768, 768, 768, 808, 768, 768, 768, 809, 809, 809, + 809, 809, 809, 809, 809, 809, 809, 810, 809, 809, + 809, 809, 809, 809, 809, 809, 768, 768, 811, 768, + 768, 768, 768, 783, 768, 768, 768, 768, 768, 812, + 813, 768, 768, 768, 768, 814, 815, 816, 768, 785, + + 768, 768, 768, 768, 817, 768, 768, 768, 818, 818, + 819, 768, 768, 820, 768, 821, 768, 768, 768, 768, + 768, 768, 768, 822, 768, 768, 823, 768, 824, 825, + 825, 826, 827, 828, 768, 829, 768, 830, 830, 830, + 768, 831, 768, 768, 768, 832, 768, 768, 768, 833, + 768, 768, 768, 768, 834, 768, 835, 768, 835, 768, + 836, 836, 836, 837, 768, 837, 838, 768, 838, 768, + 839, 768, 768, 840, 840, 840, 768, 768, 841, 841, + 841, 768, 768, 842, 768, 768, 768, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + + 843, 843, 843, 768, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 844, 768, 768, 768, 845, 846, + 847, 768, 768, 768, 768, 768, 768, 848, 849, 850, + 768, 850, 768, 851, 768, 851, 768, 852, 852, 852, + 768, 852, 852, 768, 853, 854, 768, 768, 768, 768, + 855, 768, 826, 827, 830, 830, 241, 768, 241, 241, + 833, 833, 833, 833, 833, 833, 833, 833, 833, 833, + 833, 836, 836, 278, 278, 841, 841, 768, 768, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + + 843, 843, 843, 843, 843, 843, 843, 843, 768, 843, + 843, 843, 843, 768, 847, 847, 768, 847, 847, 768, + 768, 768, 768, 848, 849, 768, 341, 852, 343, 341, + 852, 343, 768, 768, 768, 830, 830, 360, 768, 833, + 833, 833, 833, 833, 833, 833, 833, 833, 833, 833, + 836, 836, 841, 841, 768, 768, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 768, 768, 768, 843, 843, 843, + 843, 768, 768, 847, 847, 768, 768, 768, 768, 427, + + 852, 343, 852, 852, 852, 768, 768, 830, 830, 768, + 833, 833, 833, 833, 833, 833, 833, 833, 833, 833, + 833, 833, 836, 836, 841, 841, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + 843, 768, 768, 768, 768, 843, 843, 843, 843, 768, + 856, 768, 768, 768, 852, 852, 852, 768, 768, 830, + 830, 833, 833, 833, 833, 833, 833, 833, 833, 833, + 833, 833, 833, 836, 836, 841, 841, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 843, + + 843, 843, 843, 843, 843, 843, 768, 768, 768, 768, + 843, 843, 843, 843, 856, 768, 856, 768, 768, 768, + 768, 768, 833, 833, 833, 833, 833, 833, 833, 833, + 833, 833, 833, 833, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 843, 768, + 768, 768, 768, 843, 843, 857, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 833, 843, + 843, 843, 843, 843, 843, 843, 843, 843, 768, 768, + 768, 768, 843, 857, 768, 857, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 843, 843, + + 843, 843, 843, 843, 843, 768, 768, 768, 768, 768, + 768, 843, 843, 843, 843, 843, 843, 843, 768, 768, + 768, 768, 768, 768, 843, 843, 843, 843, 843, 843, + 768, 768, 768, 768, 768, 768, 843, 843, 843, 843, + 768, 768, 768, 768, 768, 768, 843, 843, 843, 843, + 768, 768, 768, 768, 843, 843, 843, 768, 768, 843, + 843, 843, 843, 843, 843, 843, 843, 0, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768 + } ; + +static yyconst short int yy_nxt[2775] = + { 0, + 40, 41, 42, 43, 40, 40, 40, 40, 40, 40, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 40, 40, 40, 44, 44, 40, 40, 40, 40, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 40, 40, 40, 40, 45, + 46, 47, 40, 48, 40, 49, 40, 40, 40, 40, + 40, 40, 50, 40, 40, 40, 40, 40, 40, 40, + 40, 51, 51, 40, 40, 40, 40, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 40, 40, 40, 53, 54, 55, 56, + 767, 57, 70, 71, 58, 58, 58, 129, 130, 58, + 73, 70, 74, 129, 130, 59, 75, 87, 88, 89, + 60, 61, 87, 88, 89, 188, 96, 97, 224, 132, + 133, 210, 211, 96, 97, 404, 98, 134, 405, 99, + 99, 99, 99, 98, 213, 213, 99, 99, 99, 99, + 62, 58, 58, 63, 64, 65, 56, 252, 57, 66, + 40, 58, 58, 58, 439, 189, 58, 102, 103, 104, + 40, 252, 67, 102, 103, 104, 225, 60, 61, 275, + + 68, 100, 214, 107, 108, 276, 109, 178, 100, 179, + 232, 105, 233, 107, 108, 572, 109, 105, 132, 133, + 180, 180, 180, 180, 265, 266, 134, 62, 58, 58, + 78, 78, 79, 80, 78, 78, 78, 78, 78, 78, + 81, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 82, 82, 78, 78, 78, 78, 82, + 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, + 82, 82, 82, 82, 82, 82, 82, 83, 82, 82, + 82, 82, 82, 82, 84, 78, 78, 78, 90, 90, + 40, 90, 90, 90, 90, 90, 90, 90, 91, 90, + + 91, 90, 90, 90, 90, 90, 90, 90, 90, 90, + 92, 93, 93, 90, 90, 90, 90, 93, 93, 93, + 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, + 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, + 93, 93, 93, 90, 90, 90, 111, 112, 296, 111, + 112, 178, 766, 179, 181, 182, 183, 113, 265, 266, + 113, 185, 186, 187, 180, 180, 180, 180, 297, 114, + 115, 116, 114, 115, 116, 117, 117, 118, 119, 120, + 117, 117, 117, 121, 117, 117, 117, 117, 117, 122, + 117, 117, 117, 117, 117, 117, 117, 117, 123, 123, + + 117, 117, 117, 117, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 124, 123, 123, 123, 123, 123, 123, 125, + 126, 117, 127, 136, 137, 138, 136, 137, 138, 206, + 206, 207, 215, 215, 215, 215, 248, 248, 248, 248, + 268, 269, 268, 269, 300, 331, 332, 139, 301, 765, + 139, 140, 141, 142, 143, 140, 140, 140, 144, 140, + 140, 145, 140, 140, 140, 146, 140, 140, 140, 140, + 140, 140, 140, 140, 147, 147, 140, 140, 140, 140, + 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, + + 147, 147, 147, 147, 147, 147, 147, 147, 148, 147, + 147, 147, 147, 147, 147, 149, 140, 140, 140, 150, + 151, 152, 153, 154, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 155, 156, 150, 150, 150, + 157, 150, 150, 150, 150, 150, 150, 150, 150, 158, + 159, 160, 161, 162, 163, 164, 164, 165, 164, 164, + 166, 167, 168, 169, 170, 164, 171, 172, 164, 173, + 174, 175, 164, 176, 150, 150, 150, 191, 201, 202, + 203, 258, 213, 213, 204, 289, 213, 213, 213, 213, + 292, 290, 217, 218, 219, 383, 303, 275, 220, 259, + + 192, 188, 193, 276, 193, 221, 304, 335, 336, 293, + 193, 222, 384, 193, 194, 195, 480, 193, 196, 223, + 214, 306, 481, 197, 214, 198, 214, 317, 317, 317, + 317, 307, 764, 205, 308, 181, 182, 183, 185, 186, + 187, 189, 321, 322, 323, 339, 340, 205, 321, 322, + 323, 387, 321, 322, 323, 388, 324, 324, 324, 324, + 342, 342, 324, 324, 324, 324, 324, 324, 324, 324, + 321, 322, 323, 201, 202, 203, 341, 344, 344, 204, + 380, 258, 339, 340, 324, 324, 324, 324, 325, 217, + 218, 219, 265, 266, 381, 220, 326, 439, 343, 259, + + 265, 266, 221, 248, 248, 248, 248, 673, 222, 268, + 269, 268, 269, 327, 392, 402, 223, 409, 393, 440, + 410, 416, 417, 418, 403, 331, 332, 763, 205, 411, + 412, 317, 317, 317, 317, 419, 419, 419, 419, 721, + 413, 331, 332, 722, 205, 357, 357, 358, 359, 357, + 357, 357, 357, 357, 357, 360, 357, 357, 357, 357, + 357, 357, 357, 357, 357, 357, 357, 357, 360, 360, + 357, 357, 357, 357, 360, 360, 360, 360, 360, 360, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + + 357, 357, 357, 362, 363, 364, 365, 335, 336, 366, + 335, 336, 339, 340, 367, 212, 212, 762, 368, 493, + 494, 369, 761, 370, 417, 494, 371, 374, 374, 760, + 374, 374, 374, 374, 374, 374, 374, 375, 374, 374, + 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, + 375, 375, 374, 374, 374, 374, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 374, 374, 374, 420, 322, 323, 427, 439, + 439, 428, 428, 439, 339, 340, 431, 431, 439, 324, + + 324, 324, 324, 338, 439, 485, 339, 340, 486, 487, + 439, 441, 443, 439, 442, 420, 322, 323, 450, 552, + 759, 513, 493, 494, 516, 553, 444, 339, 340, 429, + 338, 338, 439, 338, 338, 338, 338, 338, 338, 338, + 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + 338, 338, 338, 430, 430, 339, 340, 445, 338, 338, + 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + 430, 430, 430, 430, 430, 338, 338, 338, 432, 432, + 432, 432, 758, 439, 339, 340, 432, 757, 339, 340, + + 495, 417, 418, 432, 432, 432, 432, 432, 432, 360, + 360, 439, 438, 360, 360, 360, 360, 360, 360, 448, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + 360, 360, 439, 439, 360, 360, 360, 360, 439, 446, + 501, 501, 447, 504, 504, 416, 417, 418, 616, 617, + 339, 340, 638, 339, 340, 515, 439, 439, 449, 419, + 419, 419, 419, 514, 360, 360, 360, 375, 375, 580, + 375, 375, 375, 375, 375, 375, 375, 439, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 517, 439, 375, 375, 375, 375, 495, 417, 418, 439, + + 439, 511, 439, 512, 439, 439, 339, 340, 209, 439, + 419, 419, 419, 419, 439, 519, 520, 581, 518, 522, + 566, 566, 375, 375, 375, 500, 500, 573, 521, 578, + 339, 340, 500, 500, 500, 500, 500, 500, 500, 500, + 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, + 500, 500, 500, 500, 500, 500, 500, 502, 502, 502, + 502, 532, 439, 439, 439, 502, 439, 339, 340, 439, + 339, 340, 502, 502, 502, 502, 502, 502, 505, 505, + 505, 505, 439, 533, 582, 576, 505, 574, 579, 534, + 575, 439, 439, 505, 505, 505, 505, 505, 505, 567, + + 567, 567, 567, 590, 339, 340, 338, 567, 577, 583, + 439, 439, 625, 591, 567, 567, 567, 567, 567, 567, + 439, 439, 624, 439, 439, 439, 439, 439, 439, 439, + 616, 617, 439, 623, 616, 617, 685, 686, 685, 686, + 756, 628, 626, 632, 708, 755, 634, 685, 686, 302, + 302, 627, 629, 754, 753, 630, 631, 633, 752, 751, + 750, 709, 669, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 69, + 69, 69, 69, 69, 69, 69, 69, 69, 69, 69, + 69, 69, 69, 69, 69, 72, 72, 72, 72, 72, + + 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, + 72, 76, 76, 76, 76, 76, 76, 76, 76, 76, + 76, 76, 76, 76, 76, 76, 76, 86, 86, 86, + 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, + 86, 86, 86, 40, 40, 40, 40, 40, 40, 40, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 106, 106, 106, 106, 106, 106, 106, 106, 106, + + 106, 106, 106, 106, 106, 106, 106, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 177, 177, 177, 177, 177, 177, 177, 177, 177, + 177, 177, 177, 177, 177, 177, 177, 184, 184, 184, + 184, 749, 748, 184, 184, 184, 190, 190, 190, 190, + + 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, + 190, 200, 200, 200, 200, 747, 746, 200, 200, 200, + 209, 745, 209, 209, 209, 209, 209, 209, 209, 209, + 209, 209, 209, 209, 209, 209, 212, 744, 212, 212, + 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, + 212, 212, 216, 216, 216, 743, 742, 216, 216, 216, + 227, 741, 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 229, 740, 229, 229, + 229, 229, 229, 229, 229, 229, 229, 229, 229, 229, + 229, 229, 230, 739, 230, 230, 230, 230, 230, 230, + + 230, 230, 230, 230, 230, 230, 230, 230, 234, 234, + 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, + 234, 234, 234, 234, 236, 738, 236, 236, 737, 236, + 236, 236, 736, 735, 236, 236, 734, 733, 732, 236, + 238, 238, 238, 238, 731, 730, 238, 238, 238, 242, + 729, 242, 242, 242, 242, 242, 242, 242, 242, 242, + 242, 242, 242, 242, 242, 246, 246, 246, 246, 728, + 727, 246, 246, 246, 251, 726, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 254, 725, 254, 254, 254, 254, 254, 254, 254, 254, + + 254, 724, 254, 254, 254, 254, 255, 723, 720, 719, + 255, 255, 255, 255, 718, 717, 255, 255, 257, 716, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 261, 261, 261, 261, 715, 714, + 261, 261, 261, 264, 264, 264, 264, 264, 264, 264, + 264, 264, 264, 264, 264, 264, 264, 264, 264, 267, + 267, 267, 267, 713, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 271, 712, 711, 271, 271, + 271, 271, 271, 271, 271, 710, 271, 271, 271, 271, + 271, 273, 707, 273, 273, 273, 273, 273, 273, 273, + + 273, 273, 273, 273, 273, 273, 273, 274, 706, 274, + 274, 705, 274, 274, 274, 704, 703, 274, 274, 702, + 701, 700, 274, 279, 279, 279, 279, 699, 698, 279, + 279, 279, 284, 697, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, 288, 288, + 696, 288, 288, 695, 694, 693, 288, 288, 315, 692, + 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, + 315, 315, 315, 315, 319, 691, 319, 319, 319, 319, + 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, + 320, 690, 320, 320, 320, 320, 320, 320, 320, 320, + + 320, 320, 320, 320, 320, 320, 328, 328, 689, 688, + 328, 328, 328, 329, 329, 687, 683, 329, 329, 329, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + 330, 330, 330, 330, 330, 330, 334, 334, 334, 334, + 334, 334, 334, 334, 334, 334, 334, 334, 334, 334, + 334, 334, 338, 682, 338, 338, 338, 338, 338, 338, + 338, 338, 338, 681, 338, 338, 338, 338, 209, 680, + 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, + 209, 209, 209, 209, 345, 345, 679, 678, 677, 676, + 345, 346, 346, 346, 346, 675, 674, 346, 346, 346, + + 346, 351, 673, 351, 351, 351, 351, 351, 351, 351, + 351, 351, 351, 351, 351, 351, 351, 227, 672, 227, + 227, 227, 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 229, 671, 229, 229, 229, 229, 229, + 229, 229, 229, 229, 229, 229, 229, 229, 229, 230, + 670, 230, 230, 230, 230, 230, 230, 230, 230, 230, + 230, 230, 230, 230, 230, 353, 668, 353, 353, 353, + 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, + 353, 354, 667, 354, 354, 354, 354, 354, 354, 354, + 354, 354, 354, 354, 354, 354, 354, 234, 234, 234, + + 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, + 234, 234, 234, 236, 666, 236, 236, 665, 236, 236, + 236, 664, 663, 236, 236, 662, 661, 660, 236, 238, + 238, 238, 238, 659, 658, 238, 238, 238, 242, 657, + 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, + 242, 242, 242, 242, 246, 246, 246, 246, 656, 655, + 246, 246, 246, 361, 361, 654, 653, 652, 361, 361, + 255, 651, 650, 649, 255, 255, 255, 255, 648, 647, + 255, 255, 257, 646, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 261, 261, + + 261, 261, 645, 644, 261, 261, 261, 264, 264, 264, + 264, 264, 264, 264, 264, 264, 264, 264, 264, 264, + 264, 264, 264, 267, 267, 267, 267, 643, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 271, + 642, 641, 271, 271, 271, 271, 271, 271, 271, 640, + 271, 271, 271, 271, 271, 274, 639, 274, 274, 638, + 274, 274, 274, 637, 636, 274, 274, 635, 622, 621, + 274, 279, 279, 279, 279, 620, 619, 279, 279, 279, + 284, 618, 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 288, 288, 560, 288, + + 288, 614, 613, 612, 288, 288, 315, 611, 315, 315, + 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, + 315, 315, 319, 610, 319, 319, 319, 319, 319, 319, + 319, 319, 319, 319, 319, 319, 319, 319, 320, 609, + 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, + 320, 320, 320, 320, 415, 415, 415, 415, 415, 415, + 415, 415, 415, 415, 415, 415, 415, 415, 415, 415, + 424, 424, 424, 424, 608, 607, 424, 424, 424, 425, + 425, 425, 425, 606, 605, 425, 425, 425, 330, 330, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + + 330, 330, 330, 330, 334, 334, 334, 334, 334, 334, + 334, 334, 334, 334, 334, 334, 334, 334, 334, 334, + 338, 604, 338, 338, 338, 338, 338, 338, 338, 338, + 338, 603, 338, 338, 338, 338, 433, 433, 602, 601, + 600, 599, 433, 346, 346, 346, 346, 598, 597, 346, + 346, 346, 346, 351, 596, 351, 351, 351, 351, 351, + 351, 351, 351, 351, 351, 351, 351, 351, 351, 615, + 615, 615, 615, 615, 615, 615, 615, 615, 615, 615, + 615, 615, 615, 615, 615, 684, 684, 684, 684, 684, + 684, 684, 684, 684, 684, 684, 684, 684, 684, 684, + + 684, 595, 594, 593, 592, 589, 588, 587, 586, 585, + 584, 571, 570, 569, 568, 565, 564, 563, 562, 561, + 560, 559, 558, 557, 556, 555, 554, 551, 550, 549, + 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, + 538, 537, 536, 535, 531, 530, 529, 528, 527, 526, + 525, 524, 523, 510, 509, 508, 507, 506, 503, 499, + 498, 497, 496, 492, 491, 490, 489, 488, 484, 483, + 482, 479, 478, 477, 476, 475, 474, 473, 472, 471, + 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, + 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, + + 439, 437, 436, 435, 434, 347, 426, 423, 422, 421, + 322, 414, 316, 408, 407, 406, 401, 400, 399, 398, + 397, 396, 395, 394, 391, 390, 389, 386, 385, 382, + 379, 378, 285, 282, 377, 376, 278, 373, 372, 243, + 356, 355, 235, 231, 352, 350, 349, 348, 218, 347, + 337, 206, 333, 202, 318, 186, 182, 316, 314, 313, + 312, 311, 310, 309, 305, 299, 298, 295, 294, 291, + 287, 286, 285, 283, 282, 281, 280, 260, 278, 277, + 272, 270, 263, 262, 260, 256, 250, 253, 250, 249, + 247, 245, 244, 243, 241, 240, 239, 237, 235, 228, + + 231, 228, 226, 218, 208, 202, 199, 186, 182, 768, + 94, 94, 85, 77, 77, 39, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768 + } ; + +static yyconst short int yy_chk[2775] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, + 765, 3, 5, 5, 3, 3, 3, 27, 27, 3, + 6, 6, 6, 28, 28, 3, 6, 11, 11, 11, + 3, 3, 12, 12, 12, 48, 17, 17, 66, 29, + 29, 60, 60, 18, 18, 310, 17, 29, 310, 17, + 17, 17, 17, 18, 61, 61, 18, 18, 18, 18, + 3, 3, 3, 4, 4, 4, 4, 109, 4, 4, + 21, 4, 4, 4, 511, 48, 4, 19, 19, 19, + 22, 109, 4, 20, 20, 20, 66, 4, 4, 141, + + 4, 17, 61, 21, 21, 141, 21, 37, 18, 37, + 75, 19, 75, 22, 22, 511, 22, 20, 30, 30, + 37, 37, 37, 37, 128, 128, 30, 4, 4, 4, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 23, 23, 163, 24, + 24, 38, 764, 38, 41, 41, 41, 23, 130, 130, + 24, 45, 45, 45, 38, 38, 38, 38, 163, 23, + 23, 23, 24, 24, 24, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 31, 31, 31, 32, 32, 32, 57, + 57, 57, 62, 62, 62, 62, 99, 99, 99, 99, + 131, 131, 133, 133, 167, 198, 198, 31, 167, 763, + 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 49, 53, 53, + 53, 121, 105, 105, 53, 158, 108, 108, 115, 115, + 160, 158, 63, 63, 63, 291, 168, 275, 63, 121, + + 49, 188, 49, 275, 49, 63, 168, 205, 205, 160, + 49, 63, 291, 49, 49, 49, 405, 49, 49, 63, + 105, 170, 405, 49, 108, 49, 115, 180, 180, 180, + 180, 170, 762, 53, 170, 181, 181, 181, 185, 185, + 185, 188, 192, 192, 192, 209, 209, 63, 193, 193, + 193, 294, 194, 194, 194, 294, 192, 192, 192, 192, + 211, 211, 193, 193, 193, 193, 194, 194, 194, 194, + 195, 195, 195, 201, 201, 201, 210, 213, 213, 201, + 289, 257, 210, 210, 195, 195, 195, 195, 192, 217, + 217, 217, 264, 264, 289, 217, 194, 362, 211, 257, + + 266, 266, 217, 248, 248, 248, 248, 761, 217, 267, + 267, 269, 269, 195, 298, 309, 217, 314, 298, 362, + 314, 321, 321, 321, 309, 330, 330, 760, 201, 314, + 314, 317, 317, 317, 317, 321, 321, 321, 321, 708, + 314, 332, 332, 708, 217, 241, 241, 241, 241, 241, + 241, 241, 241, 241, 241, 241, 241, 241, 241, 241, + 241, 241, 241, 241, 241, 241, 241, 241, 241, 241, + 241, 241, 241, 241, 241, 241, 241, 241, 241, 241, + 241, 241, 241, 241, 241, 241, 241, 241, 241, 241, + 241, 241, 241, 241, 241, 241, 241, 241, 241, 241, + + 241, 241, 241, 250, 250, 250, 250, 334, 334, 250, + 336, 336, 338, 338, 250, 344, 344, 757, 250, 415, + 415, 250, 756, 250, 418, 418, 250, 278, 278, 755, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 278, 278, 278, 324, 324, 324, 339, 363, + 364, 340, 340, 365, 339, 339, 342, 342, 366, 324, + + 324, 324, 324, 340, 371, 409, 342, 342, 409, 409, + 441, 363, 365, 444, 364, 420, 420, 420, 371, 485, + 754, 441, 494, 494, 444, 485, 366, 504, 504, 340, + 341, 341, 367, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 367, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 343, 343, + 343, 343, 752, 369, 505, 505, 343, 750, 343, 343, + + 495, 495, 495, 343, 343, 343, 343, 343, 343, 360, + 360, 368, 360, 360, 360, 360, 360, 360, 360, 369, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + 360, 360, 370, 442, 360, 360, 360, 360, 443, 368, + 428, 428, 368, 431, 431, 416, 416, 416, 561, 561, + 428, 428, 749, 431, 431, 443, 519, 445, 370, 416, + 416, 416, 416, 442, 360, 360, 360, 375, 375, 519, + 375, 375, 375, 375, 375, 375, 375, 440, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 445, 447, 375, 375, 375, 375, 419, 419, 419, 446, + + 450, 440, 449, 440, 520, 448, 503, 503, 503, 512, + 419, 419, 419, 419, 517, 447, 448, 520, 446, 450, + 501, 501, 375, 375, 375, 427, 427, 512, 449, 517, + 501, 501, 427, 427, 427, 427, 427, 427, 427, 427, + 427, 427, 427, 427, 427, 427, 427, 427, 427, 427, + 427, 427, 427, 427, 427, 427, 427, 429, 429, 429, + 429, 462, 514, 513, 518, 429, 515, 566, 566, 521, + 567, 567, 429, 429, 429, 429, 429, 429, 432, 432, + 432, 432, 516, 462, 521, 515, 432, 513, 518, 462, + 514, 574, 522, 432, 432, 432, 432, 432, 432, 502, + + 502, 502, 502, 532, 565, 565, 565, 502, 516, 522, + 572, 573, 574, 532, 502, 502, 502, 502, 502, 502, + 575, 576, 573, 577, 578, 579, 580, 583, 581, 582, + 615, 615, 634, 572, 617, 617, 656, 656, 684, 684, + 748, 577, 575, 581, 681, 747, 583, 686, 686, 810, + 810, 576, 578, 745, 744, 579, 580, 582, 743, 742, + 740, 681, 634, 769, 769, 769, 769, 769, 769, 769, + 769, 769, 769, 769, 769, 769, 769, 769, 769, 770, + 770, 770, 770, 770, 770, 770, 770, 770, 770, 770, + 770, 770, 770, 770, 770, 771, 771, 771, 771, 771, + + 771, 771, 771, 771, 771, 771, 771, 771, 771, 771, + 771, 772, 772, 772, 772, 772, 772, 772, 772, 772, + 772, 772, 772, 772, 772, 772, 772, 773, 773, 773, + 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + 773, 773, 773, 774, 774, 774, 774, 774, 774, 774, + 774, 774, 774, 774, 774, 774, 774, 774, 774, 775, + 775, 775, 775, 775, 775, 775, 775, 775, 775, 775, + 775, 775, 775, 775, 775, 776, 776, 776, 776, 776, + 776, 776, 776, 776, 776, 776, 776, 776, 776, 776, + 776, 777, 777, 777, 777, 777, 777, 777, 777, 777, + + 777, 777, 777, 777, 777, 777, 777, 778, 778, 778, + 778, 778, 778, 778, 778, 778, 778, 778, 778, 778, + 778, 778, 778, 779, 779, 779, 779, 779, 779, 779, + 779, 779, 779, 779, 779, 779, 779, 779, 779, 780, + 780, 780, 780, 780, 780, 780, 780, 780, 780, 780, + 780, 780, 780, 780, 780, 781, 781, 781, 781, 781, + 781, 781, 781, 781, 781, 781, 781, 781, 781, 781, + 781, 782, 782, 782, 782, 782, 782, 782, 782, 782, + 782, 782, 782, 782, 782, 782, 782, 783, 783, 783, + 783, 739, 738, 783, 783, 783, 784, 784, 784, 784, + + 784, 784, 784, 784, 784, 784, 784, 784, 784, 784, + 784, 785, 785, 785, 785, 737, 736, 785, 785, 785, + 786, 735, 786, 786, 786, 786, 786, 786, 786, 786, + 786, 786, 786, 786, 786, 786, 787, 734, 787, 787, + 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, + 787, 787, 788, 788, 788, 733, 732, 788, 788, 788, + 789, 731, 789, 789, 789, 789, 789, 789, 789, 789, + 789, 789, 789, 789, 789, 789, 790, 729, 790, 790, + 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, + 790, 790, 791, 727, 791, 791, 791, 791, 791, 791, + + 791, 791, 791, 791, 791, 791, 791, 791, 792, 792, + 792, 792, 792, 792, 792, 792, 792, 792, 792, 792, + 792, 792, 792, 792, 793, 726, 793, 793, 725, 793, + 793, 793, 724, 723, 793, 793, 722, 721, 720, 793, + 794, 794, 794, 794, 719, 718, 794, 794, 794, 795, + 717, 795, 795, 795, 795, 795, 795, 795, 795, 795, + 795, 795, 795, 795, 795, 796, 796, 796, 796, 715, + 714, 796, 796, 796, 797, 713, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 798, 712, 798, 798, 798, 798, 798, 798, 798, 798, + + 798, 710, 798, 798, 798, 798, 799, 709, 707, 706, + 799, 799, 799, 799, 705, 704, 799, 799, 800, 703, + 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, + 800, 800, 800, 800, 801, 801, 801, 801, 702, 701, + 801, 801, 801, 802, 802, 802, 802, 802, 802, 802, + 802, 802, 802, 802, 802, 802, 802, 802, 802, 803, + 803, 803, 803, 700, 803, 803, 803, 803, 803, 803, + 803, 803, 803, 803, 803, 804, 699, 698, 804, 804, + 804, 804, 804, 804, 804, 682, 804, 804, 804, 804, + 804, 805, 680, 805, 805, 805, 805, 805, 805, 805, + + 805, 805, 805, 805, 805, 805, 805, 806, 679, 806, + 806, 678, 806, 806, 806, 677, 675, 806, 806, 674, + 672, 671, 806, 807, 807, 807, 807, 670, 669, 807, + 807, 807, 808, 668, 808, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 808, 808, 808, 808, 809, 809, + 667, 809, 809, 666, 665, 664, 809, 809, 811, 663, + 811, 811, 811, 811, 811, 811, 811, 811, 811, 811, + 811, 811, 811, 811, 812, 662, 812, 812, 812, 812, + 812, 812, 812, 812, 812, 812, 812, 812, 812, 812, + 813, 661, 813, 813, 813, 813, 813, 813, 813, 813, + + 813, 813, 813, 813, 813, 813, 814, 814, 660, 659, + 814, 814, 814, 815, 815, 658, 655, 815, 815, 815, + 816, 816, 816, 816, 816, 816, 816, 816, 816, 816, + 816, 816, 816, 816, 816, 816, 817, 817, 817, 817, + 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + 817, 817, 818, 653, 818, 818, 818, 818, 818, 818, + 818, 818, 818, 652, 818, 818, 818, 818, 819, 651, + 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, + 819, 819, 819, 819, 820, 820, 650, 646, 644, 643, + 820, 821, 821, 821, 821, 642, 641, 821, 821, 821, + + 821, 822, 639, 822, 822, 822, 822, 822, 822, 822, + 822, 822, 822, 822, 822, 822, 822, 823, 637, 823, + 823, 823, 823, 823, 823, 823, 823, 823, 823, 823, + 823, 823, 823, 824, 636, 824, 824, 824, 824, 824, + 824, 824, 824, 824, 824, 824, 824, 824, 824, 825, + 635, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 826, 633, 826, 826, 826, + 826, 826, 826, 826, 826, 826, 826, 826, 826, 826, + 826, 827, 632, 827, 827, 827, 827, 827, 827, 827, + 827, 827, 827, 827, 827, 827, 827, 828, 828, 828, + + 828, 828, 828, 828, 828, 828, 828, 828, 828, 828, + 828, 828, 828, 829, 631, 829, 829, 630, 829, 829, + 829, 629, 628, 829, 829, 627, 626, 625, 829, 830, + 830, 830, 830, 624, 623, 830, 830, 830, 831, 621, + 831, 831, 831, 831, 831, 831, 831, 831, 831, 831, + 831, 831, 831, 831, 832, 832, 832, 832, 619, 612, + 832, 832, 832, 833, 833, 611, 610, 609, 833, 833, + 834, 608, 607, 606, 834, 834, 834, 834, 604, 601, + 834, 834, 835, 600, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 835, 835, 835, 835, 836, 836, + + 836, 836, 599, 598, 836, 836, 836, 837, 837, 837, + 837, 837, 837, 837, 837, 837, 837, 837, 837, 837, + 837, 837, 837, 838, 838, 838, 838, 597, 838, 838, + 838, 838, 838, 838, 838, 838, 838, 838, 838, 839, + 596, 595, 839, 839, 839, 839, 839, 839, 839, 594, + 839, 839, 839, 839, 839, 840, 593, 840, 840, 592, + 840, 840, 840, 591, 590, 840, 840, 588, 569, 568, + 840, 841, 841, 841, 841, 564, 563, 841, 841, 841, + 842, 562, 842, 842, 842, 842, 842, 842, 842, 842, + 842, 842, 842, 842, 842, 842, 843, 843, 560, 843, + + 843, 559, 558, 557, 843, 843, 844, 556, 844, 844, + 844, 844, 844, 844, 844, 844, 844, 844, 844, 844, + 844, 844, 845, 555, 845, 845, 845, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 845, 846, 554, + 846, 846, 846, 846, 846, 846, 846, 846, 846, 846, + 846, 846, 846, 846, 847, 847, 847, 847, 847, 847, + 847, 847, 847, 847, 847, 847, 847, 847, 847, 847, + 848, 848, 848, 848, 553, 552, 848, 848, 848, 849, + 849, 849, 849, 551, 549, 849, 849, 849, 850, 850, + 850, 850, 850, 850, 850, 850, 850, 850, 850, 850, + + 850, 850, 850, 850, 851, 851, 851, 851, 851, 851, + 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, + 852, 548, 852, 852, 852, 852, 852, 852, 852, 852, + 852, 546, 852, 852, 852, 852, 853, 853, 545, 544, + 543, 542, 853, 854, 854, 854, 854, 541, 540, 854, + 854, 854, 854, 855, 539, 855, 855, 855, 855, 855, + 855, 855, 855, 855, 855, 855, 855, 855, 855, 856, + 856, 856, 856, 856, 856, 856, 856, 856, 856, 856, + 856, 856, 856, 856, 856, 857, 857, 857, 857, 857, + 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, + + 857, 538, 536, 534, 533, 530, 528, 526, 525, 524, + 523, 509, 508, 507, 506, 500, 499, 498, 497, 496, + 492, 491, 490, 489, 488, 487, 486, 483, 482, 481, + 480, 479, 478, 476, 475, 474, 473, 472, 471, 469, + 468, 467, 464, 463, 461, 460, 459, 458, 457, 454, + 453, 452, 451, 439, 437, 436, 435, 434, 430, 426, + 423, 422, 421, 414, 413, 412, 411, 410, 408, 407, + 406, 404, 403, 402, 401, 400, 399, 398, 397, 396, + 395, 394, 393, 392, 391, 390, 388, 387, 386, 384, + 383, 382, 381, 380, 379, 378, 377, 376, 373, 372, + + 361, 356, 355, 352, 350, 346, 337, 327, 326, 325, + 323, 318, 315, 313, 312, 311, 308, 307, 306, 305, + 303, 301, 300, 299, 297, 296, 295, 293, 292, 290, + 287, 286, 284, 282, 281, 280, 276, 263, 262, 243, + 240, 239, 234, 231, 226, 222, 221, 220, 219, 216, + 208, 207, 204, 203, 189, 187, 183, 179, 176, 175, + 174, 173, 172, 171, 169, 166, 165, 162, 161, 159, + 156, 155, 154, 153, 151, 149, 148, 146, 144, 143, + 137, 134, 125, 124, 122, 119, 114, 112, 107, 103, + 97, 92, 89, 87, 85, 84, 83, 80, 76, 74, + + 73, 71, 67, 65, 59, 55, 50, 47, 43, 39, + 16, 15, 10, 8, 7, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, + 768, 768, 768, 768 + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "scan.l" +#define INITIAL 0 +/* scan.l - scanner for flex input */ +#line 4 "scan.l" +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/scan.l,v 2.56 95/04/24 12:17:19 vern Exp $ */ +/* $FreeBSD$ */ + +#include "flexdef.h" +#include "parse.h" + +#define ACTION_ECHO add_action( yytext ) +#define ACTION_IFDEF(def, should_define) \ + { \ + if ( should_define ) \ + action_define( def, 1 ); \ + } + +#define MARK_END_OF_PROLOG mark_prolog(); + +#define YY_DECL \ + int flexscan() + +#define RETURNCHAR \ + yylval = (unsigned char) yytext[0]; \ + return CHAR; + +#define RETURNNAME \ + strcpy( nmstr, yytext ); \ + return NAME; + +#define PUT_BACK_STRING(str, start) \ + for ( i = strlen( str ) - 1; i >= start; --i ) \ + unput((str)[i]) + +#define CHECK_REJECT(str) \ + if ( all_upper( str ) ) \ + reject = true; + +#define CHECK_YYMORE(str) \ + if ( all_lower( str ) ) \ + yymore_used = true; +#define YY_STACK_USED 1 +#define YY_NO_TOP_STATE 1 +#define SECT2 1 +#define SECT2PROLOG 2 +#define SECT3 3 +#define CODEBLOCK 4 +#define PICKUPDEF 5 +#define SC 6 +#define CARETISBOL 7 +#define NUM 8 +#define QUOTE 9 + +#define FIRSTCCL 10 +#define CCL 11 +#define ACTION 12 +#define RECOVER 13 +#define COMMENT 14 +#define ACTION_STRING 15 +#define PERCENT_BRACE_ACTION 16 + +#define OPTION 17 +#define LINEDIR 18 + +#line 1342 "scan.c" + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap YY_PROTO(( void )); +#else +extern int yywrap YY_PROTO(( void )); +#endif +#endif + +#ifndef YY_NO_UNPUT +static void yyunput YY_PROTO(( int c, char *buf_ptr )); +#endif + +#ifndef yytext_ptr +static void yy_flex_strncpy YY_PROTO(( char *, yyconst char *, int )); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen YY_PROTO(( yyconst char * )); +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput YY_PROTO(( void )); +#else +static int input YY_PROTO(( void )); +#endif +#endif + +#if YY_STACK_USED +static int yy_start_stack_ptr = 0; +static int yy_start_stack_depth = 0; +static int *yy_start_stack = 0; +#ifndef YY_NO_PUSH_STATE +static void yy_push_state YY_PROTO(( int new_state )); +#endif +#ifndef YY_NO_POP_STATE +static void yy_pop_state YY_PROTO(( void )); +#endif +#ifndef YY_NO_TOP_STATE +static int yy_top_state YY_PROTO(( void )); +#endif + +#else +#define YY_NO_PUSH_STATE 1 +#define YY_NO_POP_STATE 1 +#define YY_NO_TOP_STATE 1 +#endif + +#ifdef YY_MALLOC_DECL +YY_MALLOC_DECL +#else +#if __STDC__ +#ifndef __cplusplus +#include <stdlib.h> +#endif +#else +/* Just try to get by without declaring the routines. This will fail + * miserably on non-ANSI systems for which sizeof(size_t) != sizeof(int) + * or sizeof(void*) != sizeof(int). + */ +#endif +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#define YY_READ_BUF_SIZE 8192 +#endif + +/* Copy whatever the last rule matched to the standard output. */ + +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO (void) fwrite( yytext, yyleng, 1, yyout ) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( yy_current_buffer->yy_is_interactive ) \ + { \ + int c = '*', n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else if ( ((result = fread( buf, 1, max_size, yyin )) == 0) \ + && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL int yylex YY_PROTO(( void )) +#endif + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK break; +#endif + +#define YY_RULE_SETUP \ + if ( yyleng > 0 ) \ + yy_current_buffer->yy_at_bol = \ + (yytext[yyleng - 1] == '\n'); \ + YY_USER_ACTION + +YY_DECL + { + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + +#line 95 "scan.l" + + static int bracelevel, didadef, indented_code; + static int doing_rule_action = false; + static int option_sense; + + int doing_codeblock = false; + int i; + Char nmdef[MAXLINE], myesc(); + + +#line 1507 "scan.c" + + if ( yy_init ) + { + yy_init = 0; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! yy_start ) + yy_start = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! yy_current_buffer ) + yy_current_buffer = + yy_create_buffer( yyin, YY_BUF_SIZE ); + + yy_load_buffer_state(); + } + + while ( 1 ) /* loops until end-of-file is reached */ + { + yy_cp = yy_c_buf_p; + + /* Support of yytext. */ + *yy_cp = yy_hold_char; + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = yy_start; + yy_current_state += YY_AT_BOL(); +yy_match: + do + { + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + if ( yy_accept[yy_current_state] ) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 769 ) + yy_c = yy_meta[(unsigned int) yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + ++yy_cp; + } + while ( yy_base[yy_current_state] != 2716 ); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + if ( yy_act == 0 ) + { /* have to back up */ + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + yy_act = yy_accept[yy_current_state]; + } + + YY_DO_BEFORE_ACTION; + + +do_action: /* This label is used only to access EOF actions. */ + + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = yy_hold_char; + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + goto yy_find_action; + + +case 1: +YY_RULE_SETUP +#line 106 "scan.l" +indented_code = true; BEGIN(CODEBLOCK); + YY_BREAK +case 2: +YY_RULE_SETUP +#line 107 "scan.l" +ACTION_ECHO; yy_push_state( COMMENT ); + YY_BREAK +case 3: +YY_RULE_SETUP +#line 108 "scan.l" +yy_push_state( LINEDIR ); + YY_BREAK +case 4: +YY_RULE_SETUP +#line 109 "scan.l" +return SCDECL; + YY_BREAK +case 5: +YY_RULE_SETUP +#line 110 "scan.l" +return XSCDECL; + YY_BREAK +case 6: +YY_RULE_SETUP +#line 111 "scan.l" +{ + ++linenum; + line_directive_out( (FILE *) 0, 1 ); + indented_code = false; + BEGIN(CODEBLOCK); + } + YY_BREAK +case 7: +YY_RULE_SETUP +#line 118 "scan.l" +/* discard */ + YY_BREAK +case 8: +YY_RULE_SETUP +#line 120 "scan.l" +{ + sectnum = 2; + bracelevel = 0; + mark_defs1(); + line_directive_out( (FILE *) 0, 1 ); + BEGIN(SECT2PROLOG); + return SECTEND; + } + YY_BREAK +case 9: +YY_RULE_SETUP +#line 129 "scan.l" +yytext_is_array = false; ++linenum; + YY_BREAK +case 10: +YY_RULE_SETUP +#line 130 "scan.l" +yytext_is_array = true; ++linenum; + YY_BREAK +case 11: +YY_RULE_SETUP +#line 132 "scan.l" +BEGIN(OPTION); return OPTION_OP; + YY_BREAK +case 12: +YY_RULE_SETUP +#line 134 "scan.l" +++linenum; /* ignore */ + YY_BREAK +case 13: +YY_RULE_SETUP +#line 135 "scan.l" +++linenum; /* ignore */ + YY_BREAK +case 14: +YY_RULE_SETUP +#line 137 "scan.l" +synerr( _( "unrecognized '%' directive" ) ); + YY_BREAK +case 15: +YY_RULE_SETUP +#line 139 "scan.l" +{ + strcpy( nmstr, yytext ); + didadef = false; + BEGIN(PICKUPDEF); + } + YY_BREAK +case 16: +YY_RULE_SETUP +#line 145 "scan.l" +RETURNNAME; + YY_BREAK +case 17: +YY_RULE_SETUP +#line 146 "scan.l" +++linenum; /* allows blank lines in section 1 */ + YY_BREAK +case 18: +YY_RULE_SETUP +#line 147 "scan.l" +ACTION_ECHO; ++linenum; /* maybe end of comment line */ + YY_BREAK + + +case 19: +YY_RULE_SETUP +#line 152 "scan.l" +ACTION_ECHO; yy_pop_state(); + YY_BREAK +case 20: +YY_RULE_SETUP +#line 153 "scan.l" +ACTION_ECHO; + YY_BREAK +case 21: +YY_RULE_SETUP +#line 154 "scan.l" +ACTION_ECHO; + YY_BREAK +case 22: +YY_RULE_SETUP +#line 155 "scan.l" +++linenum; ACTION_ECHO; + YY_BREAK + + +case 23: +YY_RULE_SETUP +#line 159 "scan.l" +yy_pop_state(); + YY_BREAK +case 24: +YY_RULE_SETUP +#line 160 "scan.l" +linenum = myctoi( yytext ); + YY_BREAK +case 25: +YY_RULE_SETUP +#line 162 "scan.l" +{ + flex_free( (void *) infilename ); + infilename = copy_string( yytext + 1 ); + infilename[strlen( infilename ) - 1] = '\0'; + } + YY_BREAK +case 26: +YY_RULE_SETUP +#line 167 "scan.l" +/* ignore spurious characters */ + YY_BREAK + + +case 27: +YY_RULE_SETUP +#line 171 "scan.l" +++linenum; BEGIN(INITIAL); + YY_BREAK +case 28: +YY_RULE_SETUP +#line 173 "scan.l" +ACTION_ECHO; + YY_BREAK +case 29: +YY_RULE_SETUP +#line 175 "scan.l" +{ + ++linenum; + ACTION_ECHO; + if ( indented_code ) + BEGIN(INITIAL); + } + YY_BREAK + + +case 30: +YY_RULE_SETUP +#line 185 "scan.l" +/* separates name and definition */ + YY_BREAK +case 31: +YY_RULE_SETUP +#line 187 "scan.l" +{ + strcpy( (char *) nmdef, yytext ); + + /* Skip trailing whitespace. */ + for ( i = strlen( (char *) nmdef ) - 1; + i >= 0 && (nmdef[i] == ' ' || nmdef[i] == '\t'); + --i ) + ; + + nmdef[i + 1] = '\0'; + + ndinstal( nmstr, nmdef ); + didadef = true; + } + YY_BREAK +case 32: +YY_RULE_SETUP +#line 202 "scan.l" +{ + if ( ! didadef ) + synerr( _( "incomplete name definition" ) ); + BEGIN(INITIAL); + ++linenum; + } + YY_BREAK + + +case 33: +YY_RULE_SETUP +#line 212 "scan.l" +++linenum; BEGIN(INITIAL); + YY_BREAK +case 34: +YY_RULE_SETUP +#line 213 "scan.l" +option_sense = true; + YY_BREAK +case 35: +YY_RULE_SETUP +#line 215 "scan.l" +return '='; + YY_BREAK +case 36: +YY_RULE_SETUP +#line 217 "scan.l" +option_sense = ! option_sense; + YY_BREAK +case 37: +YY_RULE_SETUP +#line 219 "scan.l" +csize = option_sense ? 128 : 256; + YY_BREAK +case 38: +YY_RULE_SETUP +#line 220 "scan.l" +csize = option_sense ? 256 : 128; + YY_BREAK +case 39: +YY_RULE_SETUP +#line 222 "scan.l" +long_align = option_sense; + YY_BREAK +case 40: +YY_RULE_SETUP +#line 223 "scan.l" +{ + action_define( "YY_ALWAYS_INTERACTIVE", option_sense ); + } + YY_BREAK +case 41: +YY_RULE_SETUP +#line 226 "scan.l" +yytext_is_array = option_sense; + YY_BREAK +case 42: +YY_RULE_SETUP +#line 227 "scan.l" +backing_up_report = option_sense; + YY_BREAK +case 43: +YY_RULE_SETUP +#line 228 "scan.l" +interactive = ! option_sense; + YY_BREAK +case 44: +YY_RULE_SETUP +#line 229 "scan.l" +C_plus_plus = option_sense; + YY_BREAK +case 45: +YY_RULE_SETUP +#line 230 "scan.l" +caseins = ! option_sense; + YY_BREAK +case 46: +YY_RULE_SETUP +#line 231 "scan.l" +caseins = option_sense; + YY_BREAK +case 47: +YY_RULE_SETUP +#line 232 "scan.l" +ddebug = option_sense; + YY_BREAK +case 48: +YY_RULE_SETUP +#line 233 "scan.l" +spprdflt = ! option_sense; + YY_BREAK +case 49: +YY_RULE_SETUP +#line 234 "scan.l" +useecs = option_sense; + YY_BREAK +case 50: +YY_RULE_SETUP +#line 235 "scan.l" +{ + useecs = usemecs = false; + use_read = fullspd = true; + } + YY_BREAK +case 51: +YY_RULE_SETUP +#line 239 "scan.l" +{ + useecs = usemecs = false; + use_read = fulltbl = true; + } + YY_BREAK +case 52: +YY_RULE_SETUP +#line 243 "scan.l" +ACTION_IFDEF("YY_NO_INPUT", ! option_sense); + YY_BREAK +case 53: +YY_RULE_SETUP +#line 244 "scan.l" +interactive = option_sense; + YY_BREAK +case 54: +YY_RULE_SETUP +#line 245 "scan.l" +lex_compat = option_sense; + YY_BREAK +case 55: +YY_RULE_SETUP +#line 246 "scan.l" +{ + action_define( "YY_MAIN", option_sense ); + do_yywrap = ! option_sense; + } + YY_BREAK +case 56: +YY_RULE_SETUP +#line 250 "scan.l" +usemecs = option_sense; + YY_BREAK +case 57: +YY_RULE_SETUP +#line 251 "scan.l" +{ + action_define( "YY_NEVER_INTERACTIVE", option_sense ); + } + YY_BREAK +case 58: +YY_RULE_SETUP +#line 254 "scan.l" +performance_report += option_sense ? 1 : -1; + YY_BREAK +case 59: +YY_RULE_SETUP +#line 255 "scan.l" +yytext_is_array = ! option_sense; + YY_BREAK +case 60: +YY_RULE_SETUP +#line 256 "scan.l" +use_read = option_sense; + YY_BREAK +case 61: +YY_RULE_SETUP +#line 257 "scan.l" +reject_really_used = option_sense; + YY_BREAK +case 62: +YY_RULE_SETUP +#line 258 "scan.l" +action_define( "YY_STACK_USED", option_sense ); + YY_BREAK +case 63: +YY_RULE_SETUP +#line 259 "scan.l" +do_stdinit = option_sense; + YY_BREAK +case 64: +YY_RULE_SETUP +#line 260 "scan.l" +use_stdout = option_sense; + YY_BREAK +case 65: +YY_RULE_SETUP +#line 261 "scan.l" +ACTION_IFDEF("YY_NO_UNPUT", ! option_sense); + YY_BREAK +case 66: +YY_RULE_SETUP +#line 262 "scan.l" +printstats = option_sense; + YY_BREAK +case 67: +YY_RULE_SETUP +#line 263 "scan.l" +nowarn = ! option_sense; + YY_BREAK +case 68: +YY_RULE_SETUP +#line 264 "scan.l" +do_yylineno = option_sense; + YY_BREAK +case 69: +YY_RULE_SETUP +#line 265 "scan.l" +yymore_really_used = option_sense; + YY_BREAK +case 70: +YY_RULE_SETUP +#line 266 "scan.l" +do_yywrap = option_sense; + YY_BREAK +case 71: +YY_RULE_SETUP +#line 268 "scan.l" +ACTION_IFDEF("YY_NO_PUSH_STATE", ! option_sense); + YY_BREAK +case 72: +YY_RULE_SETUP +#line 269 "scan.l" +ACTION_IFDEF("YY_NO_POP_STATE", ! option_sense); + YY_BREAK +case 73: +YY_RULE_SETUP +#line 270 "scan.l" +ACTION_IFDEF("YY_NO_TOP_STATE", ! option_sense); + YY_BREAK +case 74: +YY_RULE_SETUP +#line 272 "scan.l" +ACTION_IFDEF("YY_NO_SCAN_BUFFER", ! option_sense); + YY_BREAK +case 75: +YY_RULE_SETUP +#line 273 "scan.l" +ACTION_IFDEF("YY_NO_SCAN_BYTES", ! option_sense); + YY_BREAK +case 76: +YY_RULE_SETUP +#line 274 "scan.l" +ACTION_IFDEF("YY_NO_SCAN_STRING", ! option_sense); + YY_BREAK +case 77: +YY_RULE_SETUP +#line 276 "scan.l" +return OPT_OUTFILE; + YY_BREAK +case 78: +YY_RULE_SETUP +#line 277 "scan.l" +return OPT_PREFIX; + YY_BREAK +case 79: +YY_RULE_SETUP +#line 278 "scan.l" +return OPT_YYCLASS; + YY_BREAK +case 80: +YY_RULE_SETUP +#line 280 "scan.l" +{ + strcpy( nmstr, yytext + 1 ); + nmstr[strlen( nmstr ) - 1] = '\0'; + return NAME; + } + YY_BREAK +case 81: +YY_RULE_SETUP +#line 286 "scan.l" +{ + format_synerr( _( "unrecognized %%option: %s" ), + yytext ); + BEGIN(RECOVER); + } + YY_BREAK + +case 82: +YY_RULE_SETUP +#line 293 "scan.l" +++linenum; BEGIN(INITIAL); + YY_BREAK + +case 83: +YY_RULE_SETUP +#line 297 "scan.l" +++bracelevel; yyless( 2 ); /* eat only %{ */ + YY_BREAK +case 84: +YY_RULE_SETUP +#line 298 "scan.l" +--bracelevel; yyless( 2 ); /* eat only %} */ + YY_BREAK +case 85: +YY_RULE_SETUP +#line 300 "scan.l" +ACTION_ECHO; /* indented code in prolog */ + YY_BREAK +case 86: +YY_RULE_SETUP +#line 302 "scan.l" +{ /* non-indented code */ + if ( bracelevel <= 0 ) + { /* not in %{ ... %} */ + yyless( 0 ); /* put it all back */ + yy_set_bol( 1 ); + mark_prolog(); + BEGIN(SECT2); + } + else + ACTION_ECHO; + } + YY_BREAK +case 87: +YY_RULE_SETUP +#line 314 "scan.l" +ACTION_ECHO; + YY_BREAK +case 88: +YY_RULE_SETUP +#line 315 "scan.l" +++linenum; ACTION_ECHO; + YY_BREAK +case YY_STATE_EOF(SECT2PROLOG): +#line 317 "scan.l" +{ + mark_prolog(); + sectnum = 0; + yyterminate(); /* to stop the parser */ + } + YY_BREAK + + +case 89: +YY_RULE_SETUP +#line 325 "scan.l" +++linenum; /* allow blank lines in section 2 */ + YY_BREAK +case 90: +YY_RULE_SETUP +#line 327 "scan.l" +{ + indented_code = false; + doing_codeblock = true; + bracelevel = 1; + BEGIN(PERCENT_BRACE_ACTION); + } + YY_BREAK +case 91: +YY_RULE_SETUP +#line 334 "scan.l" +BEGIN(SC); return '<'; + YY_BREAK +case 92: +YY_RULE_SETUP +#line 335 "scan.l" +return '^'; + YY_BREAK +case 93: +YY_RULE_SETUP +#line 336 "scan.l" +BEGIN(QUOTE); return '"'; + YY_BREAK +case 94: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +YY_RULE_SETUP +#line 337 "scan.l" +BEGIN(NUM); return '{'; + YY_BREAK +case 95: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +YY_RULE_SETUP +#line 338 "scan.l" +return '$'; + YY_BREAK +case 96: +YY_RULE_SETUP +#line 340 "scan.l" +{ + bracelevel = 1; + BEGIN(PERCENT_BRACE_ACTION); + + if ( in_rule ) + { + doing_rule_action = true; + in_rule = false; + return '\n'; + } + } + YY_BREAK +case 97: +YY_RULE_SETUP +#line 351 "scan.l" +continued_action = true; ++linenum; return '\n'; + YY_BREAK +case 98: +YY_RULE_SETUP +#line 353 "scan.l" +{ + yyless( yyleng - 2 ); /* put back '/', '*' */ + bracelevel = 0; + continued_action = false; + BEGIN(ACTION); + } + YY_BREAK +case 99: +YY_RULE_SETUP +#line 360 "scan.l" +/* allow indented rules */ + YY_BREAK +case 100: +YY_RULE_SETUP +#line 362 "scan.l" +{ + /* This rule is separate from the one below because + * otherwise we get variable trailing context, so + * we can't build the scanner using -{f,F}. + */ + bracelevel = 0; + continued_action = false; + BEGIN(ACTION); + + if ( in_rule ) + { + doing_rule_action = true; + in_rule = false; + return '\n'; + } + } + YY_BREAK +case 101: +YY_RULE_SETUP +#line 379 "scan.l" +{ + bracelevel = 0; + continued_action = false; + BEGIN(ACTION); + unput( '\n' ); /* so <ACTION> sees it */ + + if ( in_rule ) + { + doing_rule_action = true; + in_rule = false; + return '\n'; + } + } + YY_BREAK +case 102: +#line 394 "scan.l" +case 103: +YY_RULE_SETUP +#line 394 "scan.l" +return EOF_OP; + YY_BREAK +case 104: +YY_RULE_SETUP +#line 396 "scan.l" +{ + sectnum = 3; + BEGIN(SECT3); + yyterminate(); /* to stop the parser */ + } + YY_BREAK +case 105: +YY_RULE_SETUP +#line 402 "scan.l" +{ + int cclval; + + strcpy( nmstr, yytext ); + + /* Check to see if we've already encountered this + * ccl. + */ + if ( (cclval = ccllookup( (Char *) nmstr )) != 0 ) + { + if ( input() != ']' ) + synerr( _( "bad character class" ) ); + + yylval = cclval; + ++cclreuse; + return PREVCCL; + } + else + { + /* We fudge a bit. We know that this ccl will + * soon be numbered as lastccl + 1 by cclinit. + */ + cclinstal( (Char *) nmstr, lastccl + 1 ); + + /* Push back everything but the leading bracket + * so the ccl can be rescanned. + */ + yyless( 1 ); + + BEGIN(FIRSTCCL); + return '['; + } + } + YY_BREAK +case 106: +YY_RULE_SETUP +#line 436 "scan.l" +{ + Char *nmdefptr; + Char *ndlookup(); + + strcpy( nmstr, yytext + 1 ); + nmstr[yyleng - 2] = '\0'; /* chop trailing brace */ + + if ( (nmdefptr = ndlookup( nmstr )) == 0 ) + format_synerr( + _( "undefined definition {%s}" ), + nmstr ); + + else + { /* push back name surrounded by ()'s */ + int len = strlen( (char *) nmdefptr ); + + if ( lex_compat || nmdefptr[0] == '^' || + (len > 0 && nmdefptr[len - 1] == '$') ) + { /* don't use ()'s after all */ + PUT_BACK_STRING((char *) nmdefptr, 0); + + if ( nmdefptr[0] == '^' ) + BEGIN(CARETISBOL); + } + + else + { + unput(')'); + PUT_BACK_STRING((char *) nmdefptr, 0); + unput('('); + } + } + } + YY_BREAK +case 107: +YY_RULE_SETUP +#line 470 "scan.l" +return (unsigned char) yytext[0]; + YY_BREAK +case 108: +YY_RULE_SETUP +#line 471 "scan.l" +RETURNCHAR; + YY_BREAK + + +case 109: +YY_RULE_SETUP +#line 476 "scan.l" +return (unsigned char) yytext[0]; + YY_BREAK +case 110: +YY_RULE_SETUP +#line 477 "scan.l" +BEGIN(SECT2); return '>'; + YY_BREAK +case 111: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +YY_RULE_SETUP +#line 478 "scan.l" +BEGIN(CARETISBOL); return '>'; + YY_BREAK +case 112: +YY_RULE_SETUP +#line 479 "scan.l" +RETURNNAME; + YY_BREAK +case 113: +YY_RULE_SETUP +#line 480 "scan.l" +{ + format_synerr( _( "bad <start condition>: %s" ), + yytext ); + } + YY_BREAK + +case 114: +YY_RULE_SETUP +#line 486 "scan.l" +BEGIN(SECT2); return '^'; + YY_BREAK + +case 115: +YY_RULE_SETUP +#line 490 "scan.l" +RETURNCHAR; + YY_BREAK +case 116: +YY_RULE_SETUP +#line 491 "scan.l" +BEGIN(SECT2); return '"'; + YY_BREAK +case 117: +YY_RULE_SETUP +#line 493 "scan.l" +{ + synerr( _( "missing quote" ) ); + BEGIN(SECT2); + ++linenum; + return '"'; + } + YY_BREAK + + +case 118: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +YY_RULE_SETUP +#line 503 "scan.l" +BEGIN(CCL); return '^'; + YY_BREAK +case 119: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +YY_RULE_SETUP +#line 504 "scan.l" +return '^'; + YY_BREAK +case 120: +YY_RULE_SETUP +#line 505 "scan.l" +BEGIN(CCL); RETURNCHAR; + YY_BREAK + + +case 121: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +YY_RULE_SETUP +#line 509 "scan.l" +return '-'; + YY_BREAK +case 122: +YY_RULE_SETUP +#line 510 "scan.l" +RETURNCHAR; + YY_BREAK +case 123: +YY_RULE_SETUP +#line 511 "scan.l" +BEGIN(SECT2); return ']'; + YY_BREAK +case 124: +YY_RULE_SETUP +#line 512 "scan.l" +{ + synerr( _( "bad character class" ) ); + BEGIN(SECT2); + return ']'; + } + YY_BREAK + + +case 125: +YY_RULE_SETUP +#line 520 "scan.l" +BEGIN(CCL); return CCE_ALNUM; + YY_BREAK +case 126: +YY_RULE_SETUP +#line 521 "scan.l" +BEGIN(CCL); return CCE_ALPHA; + YY_BREAK +case 127: +YY_RULE_SETUP +#line 522 "scan.l" +BEGIN(CCL); return CCE_BLANK; + YY_BREAK +case 128: +YY_RULE_SETUP +#line 523 "scan.l" +BEGIN(CCL); return CCE_CNTRL; + YY_BREAK +case 129: +YY_RULE_SETUP +#line 524 "scan.l" +BEGIN(CCL); return CCE_DIGIT; + YY_BREAK +case 130: +YY_RULE_SETUP +#line 525 "scan.l" +BEGIN(CCL); return CCE_GRAPH; + YY_BREAK +case 131: +YY_RULE_SETUP +#line 526 "scan.l" +BEGIN(CCL); return CCE_LOWER; + YY_BREAK +case 132: +YY_RULE_SETUP +#line 527 "scan.l" +BEGIN(CCL); return CCE_PRINT; + YY_BREAK +case 133: +YY_RULE_SETUP +#line 528 "scan.l" +BEGIN(CCL); return CCE_PUNCT; + YY_BREAK +case 134: +YY_RULE_SETUP +#line 529 "scan.l" +BEGIN(CCL); return CCE_SPACE; + YY_BREAK +case 135: +YY_RULE_SETUP +#line 530 "scan.l" +BEGIN(CCL); return CCE_UPPER; + YY_BREAK +case 136: +YY_RULE_SETUP +#line 531 "scan.l" +BEGIN(CCL); return CCE_XDIGIT; + YY_BREAK +case 137: +YY_RULE_SETUP +#line 532 "scan.l" +{ + format_synerr( + _( "bad character class expression: %s" ), + yytext ); + BEGIN(CCL); return CCE_ALNUM; + } + YY_BREAK + + +case 138: +YY_RULE_SETUP +#line 541 "scan.l" +{ + yylval = myctoi( yytext ); + return NUMBER; + } + YY_BREAK +case 139: +YY_RULE_SETUP +#line 546 "scan.l" +return ','; + YY_BREAK +case 140: +YY_RULE_SETUP +#line 547 "scan.l" +BEGIN(SECT2); return '}'; + YY_BREAK +case 141: +YY_RULE_SETUP +#line 549 "scan.l" +{ + synerr( _( "bad character inside {}'s" ) ); + BEGIN(SECT2); + return '}'; + } + YY_BREAK +case 142: +YY_RULE_SETUP +#line 555 "scan.l" +{ + synerr( _( "missing }" ) ); + BEGIN(SECT2); + ++linenum; + return '}'; + } + YY_BREAK + + +case 143: +YY_RULE_SETUP +#line 565 "scan.l" +bracelevel = 0; + YY_BREAK +case 144: +YY_RULE_SETUP +#line 567 "scan.l" +ACTION_ECHO; yy_push_state( COMMENT ); + YY_BREAK + +case 145: +YY_RULE_SETUP +#line 570 "scan.l" +{ + ACTION_ECHO; + CHECK_REJECT(yytext); + } + YY_BREAK +case 146: +YY_RULE_SETUP +#line 574 "scan.l" +{ + ACTION_ECHO; + CHECK_YYMORE(yytext); + } + YY_BREAK + +case 147: +YY_RULE_SETUP +#line 580 "scan.l" +ACTION_ECHO; + YY_BREAK +case 148: +YY_RULE_SETUP +#line 581 "scan.l" +{ + ++linenum; + ACTION_ECHO; + if ( bracelevel == 0 || + (doing_codeblock && indented_code) ) + { + if ( doing_rule_action ) + add_action( "\tYY_BREAK\n" ); + + doing_rule_action = doing_codeblock = false; + BEGIN(SECT2); + } + } + YY_BREAK + +/* Reject and YYmore() are checked for above, in PERCENT_BRACE_ACTION */ + +case 149: +YY_RULE_SETUP +#line 599 "scan.l" +ACTION_ECHO; ++bracelevel; + YY_BREAK +case 150: +YY_RULE_SETUP +#line 600 "scan.l" +ACTION_ECHO; --bracelevel; + YY_BREAK +case 151: +YY_RULE_SETUP +#line 601 "scan.l" +ACTION_ECHO; + YY_BREAK +case 152: +YY_RULE_SETUP +#line 602 "scan.l" +ACTION_ECHO; + YY_BREAK +case 153: +YY_RULE_SETUP +#line 603 "scan.l" +ACTION_ECHO; /* character constant */ + YY_BREAK +case 154: +YY_RULE_SETUP +#line 604 "scan.l" +ACTION_ECHO; BEGIN(ACTION_STRING); + YY_BREAK +case 155: +YY_RULE_SETUP +#line 605 "scan.l" +{ + ++linenum; + ACTION_ECHO; + if ( bracelevel == 0 ) + { + if ( doing_rule_action ) + add_action( "\tYY_BREAK\n" ); + + doing_rule_action = false; + BEGIN(SECT2); + } + } + YY_BREAK +case 156: +YY_RULE_SETUP +#line 617 "scan.l" +ACTION_ECHO; + YY_BREAK + + +case 157: +YY_RULE_SETUP +#line 621 "scan.l" +ACTION_ECHO; + YY_BREAK +case 158: +YY_RULE_SETUP +#line 622 "scan.l" +ACTION_ECHO; + YY_BREAK +case 159: +YY_RULE_SETUP +#line 623 "scan.l" +++linenum; ACTION_ECHO; + YY_BREAK +case 160: +YY_RULE_SETUP +#line 624 "scan.l" +ACTION_ECHO; BEGIN(ACTION); + YY_BREAK +case 161: +YY_RULE_SETUP +#line 625 "scan.l" +ACTION_ECHO; + YY_BREAK + +case YY_STATE_EOF(COMMENT): +case YY_STATE_EOF(ACTION): +case YY_STATE_EOF(ACTION_STRING): +#line 628 "scan.l" +{ + synerr( _( "EOF encountered inside an action" ) ); + yyterminate(); + } + YY_BREAK +case 162: +YY_RULE_SETUP +#line 634 "scan.l" +{ + yylval = myesc( (Char *) yytext ); + + if ( YY_START == FIRSTCCL ) + BEGIN(CCL); + + return CHAR; + } + YY_BREAK + +case 163: +YY_RULE_SETUP +#line 645 "scan.l" +ECHO; + YY_BREAK +case YY_STATE_EOF(SECT3): +#line 646 "scan.l" +sectnum = 0; yyterminate(); + YY_BREAK + +case 164: +YY_RULE_SETUP +#line 649 "scan.l" +format_synerr( _( "bad character: %s" ), yytext ); + YY_BREAK +case 165: +YY_RULE_SETUP +#line 651 "scan.l" +YY_FATAL_ERROR( "flex scanner jammed" ); + YY_BREAK +#line 2745 "scan.c" +case YY_STATE_EOF(INITIAL): +case YY_STATE_EOF(SECT2): +case YY_STATE_EOF(CODEBLOCK): +case YY_STATE_EOF(PICKUPDEF): +case YY_STATE_EOF(SC): +case YY_STATE_EOF(CARETISBOL): +case YY_STATE_EOF(NUM): +case YY_STATE_EOF(QUOTE): +case YY_STATE_EOF(FIRSTCCL): +case YY_STATE_EOF(CCL): +case YY_STATE_EOF(RECOVER): +case YY_STATE_EOF(PERCENT_BRACE_ACTION): +case YY_STATE_EOF(OPTION): +case YY_STATE_EOF(LINEDIR): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - yytext_ptr) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = yy_hold_char; + YY_RESTORE_YY_MORE_OFFSET + + if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between yy_current_buffer and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + yy_n_chars = yy_current_buffer->yy_n_chars; + yy_current_buffer->yy_input_file = yyin; + yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = yytext_ptr + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++yy_c_buf_p; + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = yy_c_buf_p; + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer() ) + { + case EOB_ACT_END_OF_FILE: + { + yy_did_buffer_switch_on_eof = 0; + + if ( yywrap() ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! yy_did_buffer_switch_on_eof ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = + yytext_ptr + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + yy_c_buf_p = + &yy_current_buffer->yy_ch_buf[yy_n_chars]; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of yylex */ + + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ + +#ifdef YY_USE_PROTOS +static int yy_get_next_buffer(void) +#else +static int yy_get_next_buffer() +#endif + { + char *dest = yy_current_buffer->yy_ch_buf; + char *source = yytext_ptr; + int number_to_move, i; + int ret_val; + + if ( yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( yy_current_buffer->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) (yy_c_buf_p - yytext_ptr) - 1; + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + yy_current_buffer->yy_n_chars = yy_n_chars = 0; + + else + { + int num_to_read = + yy_current_buffer->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ +#ifdef YY_USES_REJECT + YY_FATAL_ERROR( +"input buffer overflow, can't enlarge buffer because scanner uses REJECT" ); +#else + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = yy_current_buffer; + + int yy_c_buf_p_offset = + (int) (yy_c_buf_p - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + int new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yy_flex_realloc( (void *) b->yy_ch_buf, + b->yy_buf_size + 2 ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = 0; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = yy_current_buffer->yy_buf_size - + number_to_move - 1; +#endif + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&yy_current_buffer->yy_ch_buf[number_to_move]), + yy_n_chars, num_to_read ); + + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + if ( yy_n_chars == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + yy_current_buffer->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + yy_n_chars += number_to_move; + yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; + yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; + + yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + + return ret_val; + } + + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + +#ifdef YY_USE_PROTOS +static yy_state_type yy_get_previous_state(void) +#else +static yy_state_type yy_get_previous_state() +#endif + { + yy_state_type yy_current_state; + char *yy_cp; + + yy_current_state = yy_start; + yy_current_state += YY_AT_BOL(); + + for ( yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp ) + { + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 769 ) + yy_c = yy_meta[(unsigned int) yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + } + + return yy_current_state; + } + + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + +#ifdef YY_USE_PROTOS +static yy_state_type yy_try_NUL_trans( yy_state_type yy_current_state ) +#else +static yy_state_type yy_try_NUL_trans( yy_current_state ) +yy_state_type yy_current_state; +#endif + { + int yy_is_jam; + char *yy_cp = yy_c_buf_p; + + YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 769 ) + yy_c = yy_meta[(unsigned int) yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + yy_is_jam = (yy_current_state == 768); + + return yy_is_jam ? 0 : yy_current_state; + } + + +#ifndef YY_NO_UNPUT +#ifdef YY_USE_PROTOS +static void yyunput( int c, char *yy_bp ) +#else +static void yyunput( c, yy_bp ) +int c; +char *yy_bp; +#endif + { + char *yy_cp = yy_c_buf_p; + + /* undo effects of setting up yytext */ + *yy_cp = yy_hold_char; + + if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) + { /* need to shift things up to make room */ + /* +2 for EOB chars. */ + int number_to_move = yy_n_chars + 2; + char *dest = &yy_current_buffer->yy_ch_buf[ + yy_current_buffer->yy_buf_size + 2]; + char *source = + &yy_current_buffer->yy_ch_buf[number_to_move]; + + while ( source > yy_current_buffer->yy_ch_buf ) + *--dest = *--source; + + yy_cp += (int) (dest - source); + yy_bp += (int) (dest - source); + yy_current_buffer->yy_n_chars = + yy_n_chars = yy_current_buffer->yy_buf_size; + + if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) + YY_FATAL_ERROR( "flex scanner push-back overflow" ); + } + + *--yy_cp = (char) c; + + + yytext_ptr = yy_bp; + yy_hold_char = *yy_cp; + yy_c_buf_p = yy_cp; + } +#endif /* ifndef YY_NO_UNPUT */ + + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput() +#else +#ifdef YY_USE_PROTOS +static int input(void) +#else +static int input() +#endif +#endif + { + int c; + + *yy_c_buf_p = yy_hold_char; + + if ( *yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars] ) + /* This was really a NUL. */ + *yy_c_buf_p = '\0'; + + else + { /* need more input */ + int offset = yy_c_buf_p - yytext_ptr; + ++yy_c_buf_p; + + switch ( yy_get_next_buffer() ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /* fall through */ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap() ) + return EOF; + + if ( ! yy_did_buffer_switch_on_eof ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext_ptr + offset; + break; + } + } + } + + c = *(unsigned char *) yy_c_buf_p; /* cast for 8-bit char's */ + *yy_c_buf_p = '\0'; /* preserve yytext */ + yy_hold_char = *++yy_c_buf_p; + + yy_current_buffer->yy_at_bol = (c == '\n'); + + return c; + } +#endif /* ifndef YY_NO_INPUT */ + + +#ifdef YY_USE_PROTOS +void yyrestart( FILE *input_file ) +#else +void yyrestart( input_file ) +FILE *input_file; +#endif + { + if ( ! yy_current_buffer ) + yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); + + yy_init_buffer( yy_current_buffer, input_file ); + yy_load_buffer_state(); + } + + +#ifdef YY_USE_PROTOS +void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +#else +void yy_switch_to_buffer( new_buffer ) +YY_BUFFER_STATE new_buffer; +#endif + { + if ( yy_current_buffer == new_buffer ) + return; + + if ( yy_current_buffer ) + { + /* Flush out information for old buffer. */ + *yy_c_buf_p = yy_hold_char; + yy_current_buffer->yy_buf_pos = yy_c_buf_p; + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + yy_current_buffer = new_buffer; + yy_load_buffer_state(); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yy_did_buffer_switch_on_eof = 1; + } + + +#ifdef YY_USE_PROTOS +void yy_load_buffer_state( void ) +#else +void yy_load_buffer_state() +#endif + { + yy_n_chars = yy_current_buffer->yy_n_chars; + yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; + yyin = yy_current_buffer->yy_input_file; + yy_hold_char = *yy_c_buf_p; + } + + +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_create_buffer( FILE *file, int size ) +#else +YY_BUFFER_STATE yy_create_buffer( file, size ) +FILE *file; +int size; +#endif + { + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yy_flex_alloc( b->yy_buf_size + 2 ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; + } + + +#ifdef YY_USE_PROTOS +void yy_delete_buffer( YY_BUFFER_STATE b ) +#else +void yy_delete_buffer( b ) +YY_BUFFER_STATE b; +#endif + { + if ( ! b ) + return; + + if ( b == yy_current_buffer ) + yy_current_buffer = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yy_flex_free( (void *) b->yy_ch_buf ); + + yy_flex_free( (void *) b ); + } + + +#ifndef YY_ALWAYS_INTERACTIVE +#ifndef YY_NEVER_INTERACTIVE +extern int isatty YY_PROTO(( int )); +#endif +#endif + +#ifdef YY_USE_PROTOS +void yy_init_buffer( YY_BUFFER_STATE b, FILE *file ) +#else +void yy_init_buffer( b, file ) +YY_BUFFER_STATE b; +FILE *file; +#endif + + + { + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + +#if YY_ALWAYS_INTERACTIVE + b->yy_is_interactive = 1; +#else +#if YY_NEVER_INTERACTIVE + b->yy_is_interactive = 0; +#else + b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; +#endif +#endif + } + + +#ifdef YY_USE_PROTOS +void yy_flush_buffer( YY_BUFFER_STATE b ) +#else +void yy_flush_buffer( b ) +YY_BUFFER_STATE b; +#endif + + { + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == yy_current_buffer ) + yy_load_buffer_state(); + } + + +#ifndef YY_NO_SCAN_BUFFER +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_scan_buffer( char *base, yy_size_t size ) +#else +YY_BUFFER_STATE yy_scan_buffer( base, size ) +char *base; +yy_size_t size; +#endif + { + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return 0; + + b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = 0; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; + } +#endif + + +#ifndef YY_NO_SCAN_STRING +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_scan_string( yyconst char *yy_str ) +#else +YY_BUFFER_STATE yy_scan_string( yy_str ) +yyconst char *yy_str; +#endif + { + int len; + for ( len = 0; yy_str[len]; ++len ) + ; + + return yy_scan_bytes( yy_str, len ); + } +#endif + + +#ifndef YY_NO_SCAN_BYTES +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yy_scan_bytes( yyconst char *bytes, int len ) +#else +YY_BUFFER_STATE yy_scan_bytes( bytes, len ) +yyconst char *bytes; +int len; +#endif + { + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + int i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = len + 2; + buf = (char *) yy_flex_alloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < len; ++i ) + buf[i] = bytes[i]; + + buf[len] = buf[len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; + } +#endif + + +#ifndef YY_NO_PUSH_STATE +#ifdef YY_USE_PROTOS +static void yy_push_state( int new_state ) +#else +static void yy_push_state( new_state ) +int new_state; +#endif + { + if ( yy_start_stack_ptr >= yy_start_stack_depth ) + { + yy_size_t new_size; + + yy_start_stack_depth += YY_START_STACK_INCR; + new_size = yy_start_stack_depth * sizeof( int ); + + if ( ! yy_start_stack ) + yy_start_stack = (int *) yy_flex_alloc( new_size ); + + else + yy_start_stack = (int *) yy_flex_realloc( + (void *) yy_start_stack, new_size ); + + if ( ! yy_start_stack ) + YY_FATAL_ERROR( + "out of memory expanding start-condition stack" ); + } + + yy_start_stack[yy_start_stack_ptr++] = YY_START; + + BEGIN(new_state); + } +#endif + + +#ifndef YY_NO_POP_STATE +#ifdef YY_USE_PROTOS +static void yy_pop_state(void) +#else +static void yy_pop_state() +#endif + { + if ( --yy_start_stack_ptr < 0 ) + YY_FATAL_ERROR( "start-condition stack underflow" ); + + BEGIN(yy_start_stack[yy_start_stack_ptr]); + } +#endif + + +#ifndef YY_NO_TOP_STATE +#ifdef YY_USE_PROTOS +static int yy_top_state(void) +#else +static int yy_top_state() +#endif + { + return yy_start_stack[yy_start_stack_ptr - 1]; + } +#endif + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +#ifdef YY_USE_PROTOS +static void yy_fatal_error( yyconst char msg[] ) +#else +static void yy_fatal_error( msg ) +char msg[]; +#endif + { + (void) fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); + } + + + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yytext[yyleng] = yy_hold_char; \ + yy_c_buf_p = yytext + n; \ + yy_hold_char = *yy_c_buf_p; \ + *yy_c_buf_p = '\0'; \ + yyleng = n; \ + } \ + while ( 0 ) + + +/* Internal utility routines. */ + +#ifndef yytext_ptr +#ifdef YY_USE_PROTOS +static void yy_flex_strncpy( char *s1, yyconst char *s2, int n ) +#else +static void yy_flex_strncpy( s1, s2, n ) +char *s1; +yyconst char *s2; +int n; +#endif + { + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; + } +#endif + +#ifdef YY_NEED_STRLEN +#ifdef YY_USE_PROTOS +static int yy_flex_strlen( yyconst char *s ) +#else +static int yy_flex_strlen( s ) +yyconst char *s; +#endif + { + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; + } +#endif + + +#ifdef YY_USE_PROTOS +static void *yy_flex_alloc( yy_size_t size ) +#else +static void *yy_flex_alloc( size ) +yy_size_t size; +#endif + { + return (void *) malloc( size ); + } + +#ifdef YY_USE_PROTOS +static void *yy_flex_realloc( void *ptr, yy_size_t size ) +#else +static void *yy_flex_realloc( ptr, size ) +void *ptr; +yy_size_t size; +#endif + { + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return (void *) realloc( (char *) ptr, size ); + } + +#ifdef YY_USE_PROTOS +static void yy_flex_free( void *ptr ) +#else +static void yy_flex_free( ptr ) +void *ptr; +#endif + { + free( ptr ); + } + +#if YY_MAIN +int main() + { + yylex(); + return 0; + } +#endif +#line 651 "scan.l" + + + +int yywrap() + { + if ( --num_input_files > 0 ) + { + set_input_file( *++input_files ); + return 0; + } + + else + return 1; + } + + +/* set_input_file - open the given file (if NULL, stdin) for scanning */ + +void set_input_file( file ) +char *file; + { + if ( file && strcmp( file, "-" ) ) + { + infilename = copy_string( file ); + yyin = fopen( infilename, "r" ); + + if ( yyin == NULL ) + lerrsf( _( "can't open %s" ), file ); + } + + else + { + yyin = stdin; + infilename = copy_string( "<stdin>" ); + } + + linenum = 1; + } + + +/* Wrapper routines for accessing the scanner's malloc routines. */ + +void *flex_alloc( size ) +size_t size; + { + return (void *) malloc( size ); + } + +void *flex_realloc( ptr, size ) +void *ptr; +size_t size; + { + return (void *) realloc( ptr, size ); + } + +void flex_free( ptr ) +void *ptr; + { + if ( ptr ) + free( ptr ); + } diff --git a/usr.bin/lex/lex.1 b/usr.bin/lex/lex.1 new file mode 100644 index 0000000..38eecf1 --- /dev/null +++ b/usr.bin/lex/lex.1 @@ -0,0 +1,4068 @@ +.\" $FreeBSD$ +.\" +.TH FLEX 1 "April 1995" "Version 2.5" +.SH NAME +flex \- fast lexical analyzer generator +.SH SYNOPSIS +.B flex +.B [\-bcdfhilnpstvwBFILTV78+? \-C[aefFmr] \-ooutput \-Pprefix \-Sskeleton] +.B [\-\-help \-\-version] +.I [filename ...] +.SH OVERVIEW +This manual describes +.I flex, +a tool for generating programs that perform pattern-matching on text. The +manual includes both tutorial and reference sections: +.nf + + Description + a brief overview of the tool + + Some Simple Examples + + Format Of The Input File + + Patterns + the extended regular expressions used by flex + + How The Input Is Matched + the rules for determining what has been matched + + Actions + how to specify what to do when a pattern is matched + + The Generated Scanner + details regarding the scanner that flex produces; + how to control the input source + + Start Conditions + introducing context into your scanners, and + managing "mini-scanners" + + Multiple Input Buffers + how to manipulate multiple input sources; how to + scan from strings instead of files + + End-of-file Rules + special rules for matching the end of the input + + Miscellaneous Macros + a summary of macros available to the actions + + Values Available To The User + a summary of values available to the actions + + Interfacing With Yacc + connecting flex scanners together with yacc parsers + + Options + flex command-line options, and the "%option" + directive + + Performance Considerations + how to make your scanner go as fast as possible + + Generating C++ Scanners + the (experimental) facility for generating C++ + scanner classes + + Incompatibilities With Lex And POSIX + how flex differs from AT&T lex and the POSIX lex + standard + + Diagnostics + those error messages produced by flex (or scanners + it generates) whose meanings might not be apparent + + Files + files used by flex + + Deficiencies / Bugs + known problems with flex + + See Also + other documentation, related tools + + Author + includes contact information + +.fi +.SH DESCRIPTION +.I flex +is a tool for generating +.I scanners: +programs which recognize lexical patterns in text. +.I flex +reads +the given input files, or its standard input if no file names are given, +for a description of a scanner to generate. The description is in +the form of pairs +of regular expressions and C code, called +.I rules. flex +generates as output a C source file, +.B lex.yy.c, +which defines a routine +.B yylex(). +This file is compiled and linked with the +.B \-ll +library to produce an executable. When the executable is run, +it analyzes its input for occurrences +of the regular expressions. Whenever it finds one, it executes +the corresponding C code. +.SH SOME SIMPLE EXAMPLES +First some simple examples to get the flavor of how one uses +.I flex. +The following +.I flex +input specifies a scanner which whenever it encounters the string +"username" will replace it with the user's login name: +.nf + + %% + username printf( "%s", getlogin() ); + +.fi +By default, any text not matched by a +.I flex +scanner +is copied to the output, so the net effect of this scanner is +to copy its input file to its output with each occurrence +of "username" expanded. +In this input, there is just one rule. "username" is the +.I pattern +and the "printf" is the +.I action. +The "%%" marks the beginning of the rules. +.PP +Here's another simple example: +.nf + + %{ + int num_lines = 0, num_chars = 0; + %} + + %% + \\n ++num_lines; ++num_chars; + . ++num_chars; + + %% + main() + { + yylex(); + printf( "# of lines = %d, # of chars = %d\\n", + num_lines, num_chars ); + } + +.fi +This scanner counts the number of characters and the number +of lines in its input (it produces no output other than the +final report on the counts). The first line +declares two globals, "num_lines" and "num_chars", which are accessible +both inside +.B yylex() +and in the +.B main() +routine declared after the second "%%". There are two rules, one +which matches a newline ("\\n") and increments both the line count and +the character count, and one which matches any character other than +a newline (indicated by the "." regular expression). +.PP +A somewhat more complicated example: +.nf + + /* scanner for a toy Pascal-like language */ + + %{ + /* need this for the call to atof() below */ + #include <math.h> + %} + + DIGIT [0-9] + ID [a-z][a-z0-9]* + + %% + + {DIGIT}+ { + printf( "An integer: %s (%d)\\n", yytext, + atoi( yytext ) ); + } + + {DIGIT}+"."{DIGIT}* { + printf( "A float: %s (%g)\\n", yytext, + atof( yytext ) ); + } + + if|then|begin|end|procedure|function { + printf( "A keyword: %s\\n", yytext ); + } + + {ID} printf( "An identifier: %s\\n", yytext ); + + "+"|"-"|"*"|"/" printf( "An operator: %s\\n", yytext ); + + "{"[^}\\n]*"}" /* eat up one-line comments */ + + [ \\t\\n]+ /* eat up whitespace */ + + . printf( "Unrecognized character: %s\\n", yytext ); + + %% + + main( argc, argv ) + int argc; + char **argv; + { + ++argv, --argc; /* skip over program name */ + if ( argc > 0 ) + yyin = fopen( argv[0], "r" ); + else + yyin = stdin; + + yylex(); + } + +.fi +This is the beginnings of a simple scanner for a language like +Pascal. It identifies different types of +.I tokens +and reports on what it has seen. +.PP +The details of this example will be explained in the following +sections. +.SH FORMAT OF THE INPUT FILE +The +.I flex +input file consists of three sections, separated by a line with just +.B %% +in it: +.nf + + definitions + %% + rules + %% + user code + +.fi +The +.I definitions +section contains declarations of simple +.I name +definitions to simplify the scanner specification, and declarations of +.I start conditions, +which are explained in a later section. +.PP +Name definitions have the form: +.nf + + name definition + +.fi +The "name" is a word beginning with a letter or an underscore ('_') +followed by zero or more letters, digits, '_', or '-' (dash). +The definition is taken to begin at the first non-white-space character +following the name and continuing to the end of the line. +The definition can subsequently be referred to using "{name}", which +will expand to "(definition)". For example, +.nf + + DIGIT [0-9] + ID [a-z][a-z0-9]* + +.fi +defines "DIGIT" to be a regular expression which matches a +single digit, and +"ID" to be a regular expression which matches a letter +followed by zero-or-more letters-or-digits. +A subsequent reference to +.nf + + {DIGIT}+"."{DIGIT}* + +.fi +is identical to +.nf + + ([0-9])+"."([0-9])* + +.fi +and matches one-or-more digits followed by a '.' followed +by zero-or-more digits. +.PP +The +.I rules +section of the +.I flex +input contains a series of rules of the form: +.nf + + pattern action + +.fi +where the pattern must be unindented and the action must begin +on the same line. +.PP +See below for a further description of patterns and actions. +.PP +Finally, the user code section is simply copied to +.B lex.yy.c +verbatim. +It is used for companion routines which call or are called +by the scanner. The presence of this section is optional; +if it is missing, the second +.B %% +in the input file may be skipped, too. +.PP +In the definitions and rules sections, any +.I indented +text or text enclosed in +.B %{ +and +.B %} +is copied verbatim to the output (with the %{}'s removed). +The %{}'s must appear unindented on lines by themselves. +.PP +In the rules section, +any indented or %{} text appearing before the +first rule may be used to declare variables +which are local to the scanning routine and (after the declarations) +code which is to be executed whenever the scanning routine is entered. +Other indented or %{} text in the rule section is still copied to the output, +but its meaning is not well-defined and it may well cause compile-time +errors (this feature is present for +.I POSIX +compliance; see below for other such features). +.PP +In the definitions section (but not in the rules section), +an unindented comment (i.e., a line +beginning with "/*") is also copied verbatim to the output up +to the next "*/". +.SH PATTERNS +The patterns in the input are written using an extended set of regular +expressions. These are: +.nf + + x match the character 'x' + . any character (byte) except newline + [xyz] a "character class"; in this case, the pattern + matches either an 'x', a 'y', or a 'z' + [abj-oZ] a "character class" with a range in it; matches + an 'a', a 'b', any letter from 'j' through 'o', + or a 'Z' + [^A-Z] a "negated character class", i.e., any character + but those in the class. In this case, any + character EXCEPT an uppercase letter. + [^A-Z\\n] any character EXCEPT an uppercase letter or + a newline + r* zero or more r's, where r is any regular expression + r+ one or more r's + r? zero or one r's (that is, "an optional r") + r{2,5} anywhere from two to five r's + r{2,} two or more r's + r{4} exactly 4 r's + {name} the expansion of the "name" definition + (see above) + "[xyz]\\"foo" + the literal string: [xyz]"foo + \\X if X is an 'a', 'b', 'f', 'n', 'r', 't', or 'v', + then the ANSI-C interpretation of \\x. + Otherwise, a literal 'X' (used to escape + operators such as '*') + \\0 a NUL character (ASCII code 0) + \\123 the character with octal value 123 + \\x2a the character with hexadecimal value 2a + (r) match an r; parentheses are used to override + precedence (see below) + + + rs the regular expression r followed by the + regular expression s; called "concatenation" + + + r|s either an r or an s + + + r/s an r but only if it is followed by an s. The + text matched by s is included when determining + whether this rule is the "longest match", + but is then returned to the input before + the action is executed. So the action only + sees the text matched by r. This type + of pattern is called trailing context". + (There are some combinations of r/s that flex + cannot match correctly; see notes in the + Deficiencies / Bugs section below regarding + "dangerous trailing context".) + ^r an r, but only at the beginning of a line (i.e., + when just starting to scan, or right after a + newline has been scanned). + r$ an r, but only at the end of a line (i.e., just + before a newline). Equivalent to "r/\\n". + + Note that flex's notion of "newline" is exactly + whatever the C compiler used to compile flex + interprets '\\n' as; in particular, on some DOS + systems you must either filter out \\r's in the + input yourself, or explicitly use r/\\r\\n for "r$". + + + <s>r an r, but only in start condition s (see + below for discussion of start conditions) + <s1,s2,s3>r + same, but in any of start conditions s1, + s2, or s3 + <*>r an r in any start condition, even an exclusive one. + + + <<EOF>> an end-of-file + <s1,s2><<EOF>> + an end-of-file when in start condition s1 or s2 + +.fi +Note that inside of a character class, all regular expression operators +lose their special meaning except escape ('\\') and the character class +operators, '-', ']', and, at the beginning of the class, '^'. +.PP +The regular expressions listed above are grouped according to +precedence, from highest precedence at the top to lowest at the bottom. +Those grouped together have equal precedence. For example, +.nf + + foo|bar* + +.fi +is the same as +.nf + + (foo)|(ba(r*)) + +.fi +since the '*' operator has higher precedence than concatenation, +and concatenation higher than alternation ('|'). This pattern +therefore matches +.I either +the string "foo" +.I or +the string "ba" followed by zero-or-more r's. +To match "foo" or zero-or-more "bar"'s, use: +.nf + + foo|(bar)* + +.fi +and to match zero-or-more "foo"'s-or-"bar"'s: +.nf + + (foo|bar)* + +.fi +.PP +In addition to characters and ranges of characters, character classes +can also contain character class +.I expressions. +These are expressions enclosed inside +.B [: +and +.B :] +delimiters (which themselves must appear between the '[' and ']' of the +character class; other elements may occur inside the character class, too). +The valid expressions are: +.nf + + [:alnum:] [:alpha:] [:blank:] + [:cntrl:] [:digit:] [:graph:] + [:lower:] [:print:] [:punct:] + [:space:] [:upper:] [:xdigit:] + +.fi +These expressions all designate a set of characters equivalent to +the corresponding standard C +.B isXXX +function. For example, +.B [:alnum:] +designates those characters for which +.B isalnum() +returns true - i.e., any alphabetic or numeric. +Some systems don't provide +.B isblank(), +so flex defines +.B [:blank:] +as a blank or a tab. +.PP +For example, the following character classes are all equivalent: +.nf + + [[:alnum:]] + [[:alpha:][:digit:]] + [[:alpha:]0-9] + [a-zA-Z0-9] + +.fi +If your scanner is case-insensitive (the +.B \-i +flag), then +.B [:upper:] +and +.B [:lower:] +are equivalent to +.B [:alpha:]. +.PP +Some notes on patterns: +.IP - +A negated character class such as the example "[^A-Z]" +above +.I will match a newline +unless "\\n" (or an equivalent escape sequence) is one of the +characters explicitly present in the negated character class +(e.g., "[^A-Z\\n]"). This is unlike how many other regular +expression tools treat negated character classes, but unfortunately +the inconsistency is historically entrenched. +Matching newlines means that a pattern like [^"]* can match the entire +input unless there's another quote in the input. +.IP - +A rule can have at most one instance of trailing context (the '/' operator +or the '$' operator). The start condition, '^', and "<<EOF>>" patterns +can only occur at the beginning of a pattern, and, as well as with '/' and '$', +cannot be grouped inside parentheses. A '^' which does not occur at +the beginning of a rule or a '$' which does not occur at the end of +a rule loses its special properties and is treated as a normal character. +.IP +The following are illegal: +.nf + + foo/bar$ + <sc1>foo<sc2>bar + +.fi +Note that the first of these, can be written "foo/bar\\n". +.IP +The following will result in '$' or '^' being treated as a normal character: +.nf + + foo|(bar$) + foo|^bar + +.fi +If what's wanted is a "foo" or a bar-followed-by-a-newline, the following +could be used (the special '|' action is explained below): +.nf + + foo | + bar$ /* action goes here */ + +.fi +A similar trick will work for matching a foo or a +bar-at-the-beginning-of-a-line. +.SH HOW THE INPUT IS MATCHED +When the generated scanner is run, it analyzes its input looking +for strings which match any of its patterns. If it finds more than +one match, it takes the one matching the most text (for trailing +context rules, this includes the length of the trailing part, even +though it will then be returned to the input). If it finds two +or more matches of the same length, the +rule listed first in the +.I flex +input file is chosen. +.PP +Once the match is determined, the text corresponding to the match +(called the +.I token) +is made available in the global character pointer +.B yytext, +and its length in the global integer +.B yyleng. +The +.I action +corresponding to the matched pattern is then executed (a more +detailed description of actions follows), and then the remaining +input is scanned for another match. +.PP +If no match is found, then the +.I default rule +is executed: the next character in the input is considered matched and +copied to the standard output. Thus, the simplest legal +.I flex +input is: +.nf + + %% + +.fi +which generates a scanner that simply copies its input (one character +at a time) to its output. +.PP +Note that +.B yytext +can be defined in two different ways: either as a character +.I pointer +or as a character +.I array. +You can control which definition +.I flex +uses by including one of the special directives +.B %pointer +or +.B %array +in the first (definitions) section of your flex input. The default is +.B %pointer, +unless you use the +.B -l +lex compatibility option, in which case +.B yytext +will be an array. +The advantage of using +.B %pointer +is substantially faster scanning and no buffer overflow when matching +very large tokens (unless you run out of dynamic memory). The disadvantage +is that you are restricted in how your actions can modify +.B yytext +(see the next section), and calls to the +.B unput() +function destroys the present contents of +.B yytext, +which can be a considerable porting headache when moving between different +.I lex +versions. +.PP +The advantage of +.B %array +is that you can then modify +.B yytext +to your heart's content, and calls to +.B unput() +do not destroy +.B yytext +(see below). Furthermore, existing +.I lex +programs sometimes access +.B yytext +externally using declarations of the form: +.nf + extern char yytext[]; +.fi +This definition is erroneous when used with +.B %pointer, +but correct for +.B %array. +.PP +.B %array +defines +.B yytext +to be an array of +.B YYLMAX +characters, which defaults to a fairly large value. You can change +the size by simply #define'ing +.B YYLMAX +to a different value in the first section of your +.I flex +input. As mentioned above, with +.B %pointer +yytext grows dynamically to accommodate large tokens. While this means your +.B %pointer +scanner can accommodate very large tokens (such as matching entire blocks +of comments), bear in mind that each time the scanner must resize +.B yytext +it also must rescan the entire token from the beginning, so matching such +tokens can prove slow. +.B yytext +presently does +.I not +dynamically grow if a call to +.B unput() +results in too much text being pushed back; instead, a run-time error results. +.PP +Also note that you cannot use +.B %array +with C++ scanner classes +(the +.B c++ +option; see below). +.SH ACTIONS +Each pattern in a rule has a corresponding action, which can be any +arbitrary C statement. The pattern ends at the first non-escaped +whitespace character; the remainder of the line is its action. If the +action is empty, then when the pattern is matched the input token +is simply discarded. For example, here is the specification for a program +which deletes all occurrences of "zap me" from its input: +.nf + + %% + "zap me" + +.fi +(It will copy all other characters in the input to the output since +they will be matched by the default rule.) +.PP +Here is a program which compresses multiple blanks and tabs down to +a single blank, and throws away whitespace found at the end of a line: +.nf + + %% + [ \\t]+ putchar( ' ' ); + [ \\t]+$ /* ignore this token */ + +.fi +.PP +If the action contains a '{', then the action spans till the balancing '}' +is found, and the action may cross multiple lines. +.I flex +knows about C strings and comments and won't be fooled by braces found +within them, but also allows actions to begin with +.B %{ +and will consider the action to be all the text up to the next +.B %} +(regardless of ordinary braces inside the action). +.PP +An action consisting solely of a vertical bar ('|') means "same as +the action for the next rule." See below for an illustration. +.PP +Actions can include arbitrary C code, including +.B return +statements to return a value to whatever routine called +.B yylex(). +Each time +.B yylex() +is called it continues processing tokens from where it last left +off until it either reaches +the end of the file or executes a return. +.PP +Actions are free to modify +.B yytext +except for lengthening it (adding +characters to its end--these will overwrite later characters in the +input stream). This however does not apply when using +.B %array +(see above); in that case, +.B yytext +may be freely modified in any way. +.PP +Actions are free to modify +.B yyleng +except they should not do so if the action also includes use of +.B yymore() +(see below). +.PP +There are a number of special directives which can be included within +an action: +.IP - +.B ECHO +copies yytext to the scanner's output. +.IP - +.B BEGIN +followed by the name of a start condition places the scanner in the +corresponding start condition (see below). +.IP - +.B REJECT +directs the scanner to proceed on to the "second best" rule which matched the +input (or a prefix of the input). The rule is chosen as described +above in "How the Input is Matched", and +.B yytext +and +.B yyleng +set up appropriately. +It may either be one which matched as much text +as the originally chosen rule but came later in the +.I flex +input file, or one which matched less text. +For example, the following will both count the +words in the input and call the routine special() whenever "frob" is seen: +.nf + + int word_count = 0; + %% + + frob special(); REJECT; + [^ \\t\\n]+ ++word_count; + +.fi +Without the +.B REJECT, +any "frob"'s in the input would not be counted as words, since the +scanner normally executes only one action per token. +Multiple +.B REJECT's +are allowed, each one finding the next best choice to the currently +active rule. For example, when the following scanner scans the token +"abcd", it will write "abcdabcaba" to the output: +.nf + + %% + a | + ab | + abc | + abcd ECHO; REJECT; + .|\\n /* eat up any unmatched character */ + +.fi +(The first three rules share the fourth's action since they use +the special '|' action.) +.B REJECT +is a particularly expensive feature in terms of scanner performance; +if it is used in +.I any +of the scanner's actions it will slow down +.I all +of the scanner's matching. Furthermore, +.B REJECT +cannot be used with the +.I -Cf +or +.I -CF +options (see below). +.IP +Note also that unlike the other special actions, +.B REJECT +is a +.I branch; +code immediately following it in the action will +.I not +be executed. +.IP - +.B yymore() +tells the scanner that the next time it matches a rule, the corresponding +token should be +.I appended +onto the current value of +.B yytext +rather than replacing it. For example, given the input "mega-kludge" +the following will write "mega-mega-kludge" to the output: +.nf + + %% + mega- ECHO; yymore(); + kludge ECHO; + +.fi +First "mega-" is matched and echoed to the output. Then "kludge" +is matched, but the previous "mega-" is still hanging around at the +beginning of +.B yytext +so the +.B ECHO +for the "kludge" rule will actually write "mega-kludge". +.PP +Two notes regarding use of +.B yymore(). +First, +.B yymore() +depends on the value of +.I yyleng +correctly reflecting the size of the current token, so you must not +modify +.I yyleng +if you are using +.B yymore(). +Second, the presence of +.B yymore() +in the scanner's action entails a minor performance penalty in the +scanner's matching speed. +.IP - +.B yyless(n) +returns all but the first +.I n +characters of the current token back to the input stream, where they +will be rescanned when the scanner looks for the next match. +.B yytext +and +.B yyleng +are adjusted appropriately (e.g., +.B yyleng +will now be equal to +.I n +). For example, on the input "foobar" the following will write out +"foobarbar": +.nf + + %% + foobar ECHO; yyless(3); + [a-z]+ ECHO; + +.fi +An argument of 0 to +.B yyless +will cause the entire current input string to be scanned again. Unless you've +changed how the scanner will subsequently process its input (using +.B BEGIN, +for example), this will result in an endless loop. +.PP +Note that +.B yyless +is a macro and can only be used in the flex input file, not from +other source files. +.IP - +.B unput(c) +puts the character +.I c +back onto the input stream. It will be the next character scanned. +The following action will take the current token and cause it +to be rescanned enclosed in parentheses. +.nf + + { + int i; + /* Copy yytext because unput() trashes yytext */ + char *yycopy = strdup( yytext ); + unput( ')' ); + for ( i = yyleng - 1; i >= 0; --i ) + unput( yycopy[i] ); + unput( '(' ); + free( yycopy ); + } + +.fi +Note that since each +.B unput() +puts the given character back at the +.I beginning +of the input stream, pushing back strings must be done back-to-front. +.PP +An important potential problem when using +.B unput() +is that if you are using +.B %pointer +(the default), a call to +.B unput() +.I destroys +the contents of +.I yytext, +starting with its rightmost character and devouring one character to +the left with each call. If you need the value of yytext preserved +after a call to +.B unput() +(as in the above example), +you must either first copy it elsewhere, or build your scanner using +.B %array +instead (see How The Input Is Matched). +.PP +Finally, note that you cannot put back +.B EOF +to attempt to mark the input stream with an end-of-file. +.IP - +.B input() +reads the next character from the input stream. For example, +the following is one way to eat up C comments: +.nf + + %% + "/*" { + int c; + + for ( ; ; ) + { + while ( (c = input()) != '*' && + c != EOF ) + ; /* eat up text of comment */ + + if ( c == '*' ) + { + while ( (c = input()) == '*' ) + ; + if ( c == '/' ) + break; /* found the end */ + } + + if ( c == EOF ) + { + error( "EOF in comment" ); + break; + } + } + } + +.fi +(Note that if the scanner is compiled using +.B C++, +then +.B input() +is instead referred to as +.B yyinput(), +in order to avoid a name clash with the +.B C++ +stream by the name of +.I input.) +.IP - +.B YY_FLUSH_BUFFER +flushes the scanner's internal buffer +so that the next time the scanner attempts to match a token, it will +first refill the buffer using +.B YY_INPUT +(see The Generated Scanner, below). This action is a special case +of the more general +.B yy_flush_buffer() +function, described below in the section Multiple Input Buffers. +.IP - +.B yyterminate() +can be used in lieu of a return statement in an action. It terminates +the scanner and returns a 0 to the scanner's caller, indicating "all done". +By default, +.B yyterminate() +is also called when an end-of-file is encountered. It is a macro and +may be redefined. +.SH THE GENERATED SCANNER +The output of +.I flex +is the file +.B lex.yy.c, +which contains the scanning routine +.B yylex(), +a number of tables used by it for matching tokens, and a number +of auxiliary routines and macros. By default, +.B yylex() +is declared as follows: +.nf + + int yylex() + { + ... various definitions and the actions in here ... + } + +.fi +(If your environment supports function prototypes, then it will +be "int yylex( void )".) This definition may be changed by defining +the "YY_DECL" macro. For example, you could use: +.nf + + #define YY_DECL float lexscan( a, b ) float a, b; + +.fi +to give the scanning routine the name +.I lexscan, +returning a float, and taking two floats as arguments. Note that +if you give arguments to the scanning routine using a +K&R-style/non-prototyped function declaration, you must terminate +the definition with a semi-colon (;). +.PP +Whenever +.B yylex() +is called, it scans tokens from the global input file +.I yyin +(which defaults to stdin). It continues until it either reaches +an end-of-file (at which point it returns the value 0) or +one of its actions executes a +.I return +statement. +.PP +If the scanner reaches an end-of-file, subsequent calls are undefined +unless either +.I yyin +is pointed at a new input file (in which case scanning continues from +that file), or +.B yyrestart() +is called. +.B yyrestart() +takes one argument, a +.B FILE * +pointer (which can be nil, if you've set up +.B YY_INPUT +to scan from a source other than +.I yyin), +and initializes +.I yyin +for scanning from that file. Essentially there is no difference between +just assigning +.I yyin +to a new input file or using +.B yyrestart() +to do so; the latter is available for compatibility with previous versions +of +.I flex, +and because it can be used to switch input files in the middle of scanning. +It can also be used to throw away the current input buffer, by calling +it with an argument of +.I yyin; +but better is to use +.B YY_FLUSH_BUFFER +(see above). +Note that +.B yyrestart() +does +.I not +reset the start condition to +.B INITIAL +(see Start Conditions, below). +.PP +If +.B yylex() +stops scanning due to executing a +.I return +statement in one of the actions, the scanner may then be called again and it +will resume scanning where it left off. +.PP +By default (and for purposes of efficiency), the scanner uses +block-reads rather than simple +.I getc() +calls to read characters from +.I yyin. +The nature of how it gets its input can be controlled by defining the +.B YY_INPUT +macro. +YY_INPUT's calling sequence is "YY_INPUT(buf,result,max_size)". Its +action is to place up to +.I max_size +characters in the character array +.I buf +and return in the integer variable +.I result +either the +number of characters read or the constant YY_NULL (0 on Unix systems) +to indicate EOF. The default YY_INPUT reads from the +global file-pointer "yyin". +.PP +A sample definition of YY_INPUT (in the definitions +section of the input file): +.nf + + %{ + #define YY_INPUT(buf,result,max_size) \\ + { \\ + int c = getchar(); \\ + result = (c == EOF) ? YY_NULL : (buf[0] = c, 1); \\ + } + %} + +.fi +This definition will change the input processing to occur +one character at a time. +.PP +When the scanner receives an end-of-file indication from YY_INPUT, +it then checks the +.B yywrap() +function. If +.B yywrap() +returns false (zero), then it is assumed that the +function has gone ahead and set up +.I yyin +to point to another input file, and scanning continues. If it returns +true (non-zero), then the scanner terminates, returning 0 to its +caller. Note that in either case, the start condition remains unchanged; +it does +.I not +revert to +.B INITIAL. +.PP +If you do not supply your own version of +.B yywrap(), +then you must either use +.B %option noyywrap +(in which case the scanner behaves as though +.B yywrap() +returned 1), or you must link with +.B \-ll +to obtain the default version of the routine, which always returns 1. +.PP +Three routines are available for scanning from in-memory buffers rather +than files: +.B yy_scan_string(), yy_scan_bytes(), +and +.B yy_scan_buffer(). +See the discussion of them below in the section Multiple Input Buffers. +.PP +The scanner writes its +.B ECHO +output to the +.I yyout +global (default, stdout), which may be redefined by the user simply +by assigning it to some other +.B FILE +pointer. +.SH START CONDITIONS +.I flex +provides a mechanism for conditionally activating rules. Any rule +whose pattern is prefixed with "<sc>" will only be active when +the scanner is in the start condition named "sc". For example, +.nf + + <STRING>[^"]* { /* eat up the string body ... */ + ... + } + +.fi +will be active only when the scanner is in the "STRING" start +condition, and +.nf + + <INITIAL,STRING,QUOTE>\\. { /* handle an escape ... */ + ... + } + +.fi +will be active only when the current start condition is +either "INITIAL", "STRING", or "QUOTE". +.PP +Start conditions +are declared in the definitions (first) section of the input +using unindented lines beginning with either +.B %s +or +.B %x +followed by a list of names. +The former declares +.I inclusive +start conditions, the latter +.I exclusive +start conditions. A start condition is activated using the +.B BEGIN +action. Until the next +.B BEGIN +action is executed, rules with the given start +condition will be active and +rules with other start conditions will be inactive. +If the start condition is +.I inclusive, +then rules with no start conditions at all will also be active. +If it is +.I exclusive, +then +.I only +rules qualified with the start condition will be active. +A set of rules contingent on the same exclusive start condition +describe a scanner which is independent of any of the other rules in the +.I flex +input. Because of this, +exclusive start conditions make it easy to specify "mini-scanners" +which scan portions of the input that are syntactically different +from the rest (e.g., comments). +.PP +If the distinction between inclusive and exclusive start conditions +is still a little vague, here's a simple example illustrating the +connection between the two. The set of rules: +.nf + + %s example + %% + + <example>foo do_something(); + + bar something_else(); + +.fi +is equivalent to +.nf + + %x example + %% + + <example>foo do_something(); + + <INITIAL,example>bar something_else(); + +.fi +Without the +.B <INITIAL,example> +qualifier, the +.I bar +pattern in the second example wouldn't be active (i.e., couldn't match) +when in start condition +.B example. +If we just used +.B <example> +to qualify +.I bar, +though, then it would only be active in +.B example +and not in +.B INITIAL, +while in the first example it's active in both, because in the first +example the +.B example +start condition is an +.I inclusive +.B (%s) +start condition. +.PP +Also note that the special start-condition specifier +.B <*> +matches every start condition. Thus, the above example could also +have been written; +.nf + + %x example + %% + + <example>foo do_something(); + + <*>bar something_else(); + +.fi +.PP +The default rule (to +.B ECHO +any unmatched character) remains active in start conditions. It +is equivalent to: +.nf + + <*>.|\\n ECHO; + +.fi +.PP +.B BEGIN(0) +returns to the original state where only the rules with +no start conditions are active. This state can also be +referred to as the start-condition "INITIAL", so +.B BEGIN(INITIAL) +is equivalent to +.B BEGIN(0). +(The parentheses around the start condition name are not required but +are considered good style.) +.PP +.B BEGIN +actions can also be given as indented code at the beginning +of the rules section. For example, the following will cause +the scanner to enter the "SPECIAL" start condition whenever +.B yylex() +is called and the global variable +.I enter_special +is true: +.nf + + int enter_special; + + %x SPECIAL + %% + if ( enter_special ) + BEGIN(SPECIAL); + + <SPECIAL>blahblahblah + ...more rules follow... + +.fi +.PP +To illustrate the uses of start conditions, +here is a scanner which provides two different interpretations +of a string like "123.456". By default it will treat it as +three tokens, the integer "123", a dot ('.'), and the integer "456". +But if the string is preceded earlier in the line by the string +"expect-floats" +it will treat it as a single token, the floating-point number +123.456: +.nf + + %{ + #include <math.h> + %} + %s expect + + %% + expect-floats BEGIN(expect); + + <expect>[0-9]+"."[0-9]+ { + printf( "found a float, = %f\\n", + atof( yytext ) ); + } + <expect>\\n { + /* that's the end of the line, so + * we need another "expect-number" + * before we'll recognize any more + * numbers + */ + BEGIN(INITIAL); + } + + [0-9]+ { + printf( "found an integer, = %d\\n", + atoi( yytext ) ); + } + + "." printf( "found a dot\\n" ); + +.fi +Here is a scanner which recognizes (and discards) C comments while +maintaining a count of the current input line. +.nf + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\\n]* /* eat anything that's not a '*' */ + <comment>"*"+[^*/\\n]* /* eat up '*'s not followed by '/'s */ + <comment>\\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + +.fi +This scanner goes to a bit of trouble to match as much +text as possible with each rule. In general, when attempting to write +a high-speed scanner try to match as much possible in each rule, as +it's a big win. +.PP +Note that start-conditions names are really integer values and +can be stored as such. Thus, the above could be extended in the +following fashion: +.nf + + %x comment foo + %% + int line_num = 1; + int comment_caller; + + "/*" { + comment_caller = INITIAL; + BEGIN(comment); + } + + ... + + <foo>"/*" { + comment_caller = foo; + BEGIN(comment); + } + + <comment>[^*\\n]* /* eat anything that's not a '*' */ + <comment>"*"+[^*/\\n]* /* eat up '*'s not followed by '/'s */ + <comment>\\n ++line_num; + <comment>"*"+"/" BEGIN(comment_caller); + +.fi +Furthermore, you can access the current start condition using +the integer-valued +.B YY_START +macro. For example, the above assignments to +.I comment_caller +could instead be written +.nf + + comment_caller = YY_START; + +.fi +Flex provides +.B YYSTATE +as an alias for +.B YY_START +(since that is what's used by AT&T +.I lex). +.PP +Note that start conditions do not have their own name-space; %s's and %x's +declare names in the same fashion as #define's. +.PP +Finally, here's an example of how to match C-style quoted strings using +exclusive start conditions, including expanded escape sequences (but +not including checking for a string that's too long): +.nf + + %x str + + %% + char string_buf[MAX_STR_CONST]; + char *string_buf_ptr; + + + \\" string_buf_ptr = string_buf; BEGIN(str); + + <str>\\" { /* saw closing quote - all done */ + BEGIN(INITIAL); + *string_buf_ptr = '\\0'; + /* return string constant token type and + * value to parser + */ + } + + <str>\\n { + /* error - unterminated string constant */ + /* generate error message */ + } + + <str>\\\\[0-7]{1,3} { + /* octal escape sequence */ + int result; + + (void) sscanf( yytext + 1, "%o", &result ); + + if ( result > 0xff ) + /* error, constant is out-of-bounds */ + + *string_buf_ptr++ = result; + } + + <str>\\\\[0-9]+ { + /* generate error - bad escape sequence; something + * like '\\48' or '\\0777777' + */ + } + + <str>\\\\n *string_buf_ptr++ = '\\n'; + <str>\\\\t *string_buf_ptr++ = '\\t'; + <str>\\\\r *string_buf_ptr++ = '\\r'; + <str>\\\\b *string_buf_ptr++ = '\\b'; + <str>\\\\f *string_buf_ptr++ = '\\f'; + + <str>\\\\(.|\\n) *string_buf_ptr++ = yytext[1]; + + <str>[^\\\\\\n\\"]+ { + char *yptr = yytext; + + while ( *yptr ) + *string_buf_ptr++ = *yptr++; + } + +.fi +.PP +Often, such as in some of the examples above, you wind up writing a +whole bunch of rules all preceded by the same start condition(s). Flex +makes this a little easier and cleaner by introducing a notion of +start condition +.I scope. +A start condition scope is begun with: +.nf + + <SCs>{ + +.fi +where +.I SCs +is a list of one or more start conditions. Inside the start condition +scope, every rule automatically has the prefix +.I <SCs> +applied to it, until a +.I '}' +which matches the initial +.I '{'. +So, for example, +.nf + + <ESC>{ + "\\\\n" return '\\n'; + "\\\\r" return '\\r'; + "\\\\f" return '\\f'; + "\\\\0" return '\\0'; + } + +.fi +is equivalent to: +.nf + + <ESC>"\\\\n" return '\\n'; + <ESC>"\\\\r" return '\\r'; + <ESC>"\\\\f" return '\\f'; + <ESC>"\\\\0" return '\\0'; + +.fi +Start condition scopes may be nested. +.PP +Three routines are available for manipulating stacks of start conditions: +.TP +.B void yy_push_state(int new_state) +pushes the current start condition onto the top of the start condition +stack and switches to +.I new_state +as though you had used +.B BEGIN new_state +(recall that start condition names are also integers). +.TP +.B void yy_pop_state() +pops the top of the stack and switches to it via +.B BEGIN. +.TP +.B int yy_top_state() +returns the top of the stack without altering the stack's contents. +.PP +The start condition stack grows dynamically and so has no built-in +size limitation. If memory is exhausted, program execution aborts. +.PP +To use start condition stacks, your scanner must include a +.B %option stack +directive (see Options below). +.SH MULTIPLE INPUT BUFFERS +Some scanners (such as those which support "include" files) +require reading from several input streams. As +.I flex +scanners do a large amount of buffering, one cannot control +where the next input will be read from by simply writing a +.B YY_INPUT +which is sensitive to the scanning context. +.B YY_INPUT +is only called when the scanner reaches the end of its buffer, which +may be a long time after scanning a statement such as an "include" +which requires switching the input source. +.PP +To negotiate these sorts of problems, +.I flex +provides a mechanism for creating and switching between multiple +input buffers. An input buffer is created by using: +.nf + + YY_BUFFER_STATE yy_create_buffer( FILE *file, int size ) + +.fi +which takes a +.I FILE +pointer and a size and creates a buffer associated with the given +file and large enough to hold +.I size +characters (when in doubt, use +.B YY_BUF_SIZE +for the size). It returns a +.B YY_BUFFER_STATE +handle, which may then be passed to other routines (see below). The +.B YY_BUFFER_STATE +type is a pointer to an opaque +.B struct yy_buffer_state +structure, so you may safely initialize YY_BUFFER_STATE variables to +.B ((YY_BUFFER_STATE) 0) +if you wish, and also refer to the opaque structure in order to +correctly declare input buffers in source files other than that +of your scanner. Note that the +.I FILE +pointer in the call to +.B yy_create_buffer +is only used as the value of +.I yyin +seen by +.B YY_INPUT; +if you redefine +.B YY_INPUT +so it no longer uses +.I yyin, +then you can safely pass a nil +.I FILE +pointer to +.B yy_create_buffer. +You select a particular buffer to scan from using: +.nf + + void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) + +.fi +switches the scanner's input buffer so subsequent tokens will +come from +.I new_buffer. +Note that +.B yy_switch_to_buffer() +may be used by yywrap() to set things up for continued scanning, instead +of opening a new file and pointing +.I yyin +at it. Note also that switching input sources via either +.B yy_switch_to_buffer() +or +.B yywrap() +does +.I not +change the start condition. +.nf + + void yy_delete_buffer( YY_BUFFER_STATE buffer ) + +.fi +is used to reclaim the storage associated with a buffer. ( +.B buffer +can be nil, in which case the routine does nothing.) +You can also clear the current contents of a buffer using: +.nf + + void yy_flush_buffer( YY_BUFFER_STATE buffer ) + +.fi +This function discards the buffer's contents, +so the next time the scanner attempts to match a token from the +buffer, it will first fill the buffer anew using +.B YY_INPUT. +.PP +.B yy_new_buffer() +is an alias for +.B yy_create_buffer(), +provided for compatibility with the C++ use of +.I new +and +.I delete +for creating and destroying dynamic objects. +.PP +Finally, the +.B YY_CURRENT_BUFFER +macro returns a +.B YY_BUFFER_STATE +handle to the current buffer. +.PP +Here is an example of using these features for writing a scanner +which expands include files (the +.B <<EOF>> +feature is discussed below): +.nf + + /* the "incl" state is used for picking up the name + * of an include file + */ + %x incl + + %{ + #define MAX_INCLUDE_DEPTH 10 + YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH]; + int include_stack_ptr = 0; + %} + + %% + include BEGIN(incl); + + [a-z]+ ECHO; + [^a-z\\n]*\\n? ECHO; + + <incl>[ \\t]* /* eat the whitespace */ + <incl>[^ \\t\\n]+ { /* got the include file name */ + if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) + { + fprintf( stderr, "Includes nested too deeply" ); + exit( 1 ); + } + + include_stack[include_stack_ptr++] = + YY_CURRENT_BUFFER; + + yyin = fopen( yytext, "r" ); + + if ( ! yyin ) + error( ... ); + + yy_switch_to_buffer( + yy_create_buffer( yyin, YY_BUF_SIZE ) ); + + BEGIN(INITIAL); + } + + <<EOF>> { + if ( --include_stack_ptr < 0 ) + { + yyterminate(); + } + + else + { + yy_delete_buffer( YY_CURRENT_BUFFER ); + yy_switch_to_buffer( + include_stack[include_stack_ptr] ); + } + } + +.fi +Three routines are available for setting up input buffers for +scanning in-memory strings instead of files. All of them create +a new input buffer for scanning the string, and return a corresponding +.B YY_BUFFER_STATE +handle (which you should delete with +.B yy_delete_buffer() +when done with it). They also switch to the new buffer using +.B yy_switch_to_buffer(), +so the next call to +.B yylex() +will start scanning the string. +.TP +.B yy_scan_string(const char *str) +scans a NUL-terminated string. +.TP +.B yy_scan_bytes(const char *bytes, int len) +scans +.I len +bytes (including possibly NUL's) +starting at location +.I bytes. +.PP +Note that both of these functions create and scan a +.I copy +of the string or bytes. (This may be desirable, since +.B yylex() +modifies the contents of the buffer it is scanning.) You can avoid the +copy by using: +.TP +.B yy_scan_buffer(char *base, yy_size_t size) +which scans in place the buffer starting at +.I base, +consisting of +.I size +bytes, the last two bytes of which +.I must +be +.B YY_END_OF_BUFFER_CHAR +(ASCII NUL). +These last two bytes are not scanned; thus, scanning +consists of +.B base[0] +through +.B base[size-2], +inclusive. +.IP +If you fail to set up +.I base +in this manner (i.e., forget the final two +.B YY_END_OF_BUFFER_CHAR +bytes), then +.B yy_scan_buffer() +returns a nil pointer instead of creating a new input buffer. +.IP +The type +.B yy_size_t +is an integral type to which you can cast an integer expression +reflecting the size of the buffer. +.SH END-OF-FILE RULES +The special rule "<<EOF>>" indicates +actions which are to be taken when an end-of-file is +encountered and yywrap() returns non-zero (i.e., indicates +no further files to process). The action must finish +by doing one of four things: +.IP - +assigning +.I yyin +to a new input file (in previous versions of flex, after doing the +assignment you had to call the special action +.B YY_NEW_FILE; +this is no longer necessary); +.IP - +executing a +.I return +statement; +.IP - +executing the special +.B yyterminate() +action; +.IP - +or, switching to a new buffer using +.B yy_switch_to_buffer() +as shown in the example above. +.PP +<<EOF>> rules may not be used with other +patterns; they may only be qualified with a list of start +conditions. If an unqualified <<EOF>> rule is given, it +applies to +.I all +start conditions which do not already have <<EOF>> actions. To +specify an <<EOF>> rule for only the initial start condition, use +.nf + + <INITIAL><<EOF>> + +.fi +.PP +These rules are useful for catching things like unclosed comments. +An example: +.nf + + %x quote + %% + + ...other rules for dealing with quotes... + + <quote><<EOF>> { + error( "unterminated quote" ); + yyterminate(); + } + <<EOF>> { + if ( *++filelist ) + yyin = fopen( *filelist, "r" ); + else + yyterminate(); + } + +.fi +.SH MISCELLANEOUS MACROS +The macro +.B YY_USER_ACTION +can be defined to provide an action +which is always executed prior to the matched rule's action. For example, +it could be #define'd to call a routine to convert yytext to lower-case. +When +.B YY_USER_ACTION +is invoked, the variable +.I yy_act +gives the number of the matched rule (rules are numbered starting with 1). +Suppose you want to profile how often each of your rules is matched. The +following would do the trick: +.nf + + #define YY_USER_ACTION ++ctr[yy_act] + +.fi +where +.I ctr +is an array to hold the counts for the different rules. Note that +the macro +.B YY_NUM_RULES +gives the total number of rules (including the default rule, even if +you use +.B \-s), +so a correct declaration for +.I ctr +is: +.nf + + int ctr[YY_NUM_RULES]; + +.fi +.PP +The macro +.B YY_USER_INIT +may be defined to provide an action which is always executed before +the first scan (and before the scanner's internal initializations are done). +For example, it could be used to call a routine to read +in a data table or open a logging file. +.PP +The macro +.B yy_set_interactive(is_interactive) +can be used to control whether the current buffer is considered +.I interactive. +An interactive buffer is processed more slowly, +but must be used when the scanner's input source is indeed +interactive to avoid problems due to waiting to fill buffers +(see the discussion of the +.B \-I +flag below). A non-zero value +in the macro invocation marks the buffer as interactive, a zero +value as non-interactive. Note that use of this macro overrides +.B %option interactive , +.B %option always-interactive +or +.B %option never-interactive +(see Options below). +.B yy_set_interactive() +must be invoked prior to beginning to scan the buffer that is +(or is not) to be considered interactive. +.PP +The macro +.B yy_set_bol(at_bol) +can be used to control whether the current buffer's scanning +context for the next token match is done as though at the +beginning of a line. A non-zero macro argument makes rules anchored with +\&'^' active, while a zero argument makes '^' rules inactive. +.PP +The macro +.B YY_AT_BOL() +returns true if the next token scanned from the current buffer +will have '^' rules active, false otherwise. +.PP +In the generated scanner, the actions are all gathered in one large +switch statement and separated using +.B YY_BREAK, +which may be redefined. By default, it is simply a "break", to separate +each rule's action from the following rule's. +Redefining +.B YY_BREAK +allows, for example, C++ users to +#define YY_BREAK to do nothing (while being very careful that every +rule ends with a "break" or a "return"!) to avoid suffering from +unreachable statement warnings where because a rule's action ends with +"return", the +.B YY_BREAK +is inaccessible. +.SH VALUES AVAILABLE TO THE USER +This section summarizes the various values available to the user +in the rule actions. +.IP - +.B char *yytext +holds the text of the current token. It may be modified but not lengthened +(you cannot append characters to the end). +.IP +If the special directive +.B %array +appears in the first section of the scanner description, then +.B yytext +is instead declared +.B char yytext[YYLMAX], +where +.B YYLMAX +is a macro definition that you can redefine in the first section +if you don't like the default value (generally 8KB). Using +.B %array +results in somewhat slower scanners, but the value of +.B yytext +becomes immune to calls to +.I input() +and +.I unput(), +which potentially destroy its value when +.B yytext +is a character pointer. The opposite of +.B %array +is +.B %pointer, +which is the default. +.IP +You cannot use +.B %array +when generating C++ scanner classes +(the +.B \-+ +flag). +.IP - +.B int yyleng +holds the length of the current token. +.IP - +.B FILE *yyin +is the file which by default +.I flex +reads from. It may be redefined but doing so only makes sense before +scanning begins or after an EOF has been encountered. Changing it in +the midst of scanning will have unexpected results since +.I flex +buffers its input; use +.B yyrestart() +instead. +Once scanning terminates because an end-of-file +has been seen, you can assign +.I yyin +at the new input file and then call the scanner again to continue scanning. +.IP - +.B void yyrestart( FILE *new_file ) +may be called to point +.I yyin +at the new input file. The switch-over to the new file is immediate +(any previously buffered-up input is lost). Note that calling +.B yyrestart() +with +.I yyin +as an argument thus throws away the current input buffer and continues +scanning the same input file. +.IP - +.B FILE *yyout +is the file to which +.B ECHO +actions are done. It can be reassigned by the user. +.IP - +.B YY_CURRENT_BUFFER +returns a +.B YY_BUFFER_STATE +handle to the current buffer. +.IP - +.B YY_START +returns an integer value corresponding to the current start +condition. You can subsequently use this value with +.B BEGIN +to return to that start condition. +.SH INTERFACING WITH YACC +One of the main uses of +.I flex +is as a companion to the +.I yacc +parser-generator. +.I yacc +parsers expect to call a routine named +.B yylex() +to find the next input token. The routine is supposed to +return the type of the next token as well as putting any associated +value in the global +.B yylval. +To use +.I flex +with +.I yacc, +one specifies the +.B \-d +option to +.I yacc +to instruct it to generate the file +.B y.tab.h +containing definitions of all the +.B %tokens +appearing in the +.I yacc +input. This file is then included in the +.I flex +scanner. For example, if one of the tokens is "TOK_NUMBER", +part of the scanner might look like: +.nf + + %{ + #include "y.tab.h" + %} + + %% + + [0-9]+ yylval = atoi( yytext ); return TOK_NUMBER; + +.fi +.SH OPTIONS +.I flex +has the following options: +.TP +.B \-b +Generate backing-up information to +.I lex.backup. +This is a list of scanner states which require backing up +and the input characters on which they do so. By adding rules one +can remove backing-up states. If +.I all +backing-up states are eliminated and +.B \-Cf +or +.B \-CF +is used, the generated scanner will run faster (see the +.B \-p +flag). Only users who wish to squeeze every last cycle out of their +scanners need worry about this option. (See the section on Performance +Considerations below.) +.TP +.B \-c +is a do-nothing, deprecated option included for POSIX compliance. +.TP +.B \-d +makes the generated scanner run in +.I debug +mode. Whenever a pattern is recognized and the global +.B yy_flex_debug +is non-zero (which is the default), +the scanner will write to +.I stderr +a line of the form: +.nf + + --accepting rule at line 53 ("the matched text") + +.fi +The line number refers to the location of the rule in the file +defining the scanner (i.e., the file that was fed to flex). Messages +are also generated when the scanner backs up, accepts the +default rule, reaches the end of its input buffer (or encounters +a NUL; at this point, the two look the same as far as the scanner's concerned), +or reaches an end-of-file. +.TP +.B \-f +specifies +.I fast scanner. +No table compression is done and stdio is bypassed. +The result is large but fast. This option is equivalent to +.B \-Cfr +(see below). +.TP +.B \-h +generates a "help" summary of +.I flex's +options to +.I stdout +and then exits. +.B \-? +and +.B \-\-help +are synonyms for +.B \-h. +.TP +.B \-i +instructs +.I flex +to generate a +.I case-insensitive +scanner. The case of letters given in the +.I flex +input patterns will +be ignored, and tokens in the input will be matched regardless of case. The +matched text given in +.I yytext +will have the preserved case (i.e., it will not be folded). +.TP +.B \-l +turns on maximum compatibility with the original AT&T +.I lex +implementation. Note that this does not mean +.I full +compatibility. Use of this option costs a considerable amount of +performance, and it cannot be used with the +.B \-+, -f, -F, -Cf, +or +.B -CF +options. For details on the compatibilities it provides, see the section +"Incompatibilities With Lex And POSIX" below. This option also results +in the name +.B YY_FLEX_LEX_COMPAT +being #define'd in the generated scanner. +.TP +.B \-n +is another do-nothing, deprecated option included only for +POSIX compliance. +.TP +.B \-p +generates a performance report to stderr. The report +consists of comments regarding features of the +.I flex +input file which will cause a serious loss of performance in the resulting +scanner. If you give the flag twice, you will also get comments regarding +features that lead to minor performance losses. +.IP +Note that the use of +.B REJECT, +.B %option yylineno, +and variable trailing context (see the Deficiencies / Bugs section below) +entails a substantial performance penalty; use of +.I yymore(), +the +.B ^ +operator, +and the +.B \-I +flag entail minor performance penalties. +.TP +.B \-s +causes the +.I default rule +(that unmatched scanner input is echoed to +.I stdout) +to be suppressed. If the scanner encounters input that does not +match any of its rules, it aborts with an error. This option is +useful for finding holes in a scanner's rule set. +.TP +.B \-t +instructs +.I flex +to write the scanner it generates to standard output instead +of +.B lex.yy.c. +.TP +.B \-v +specifies that +.I flex +should write to +.I stderr +a summary of statistics regarding the scanner it generates. +Most of the statistics are meaningless to the casual +.I flex +user, but the first line identifies the version of +.I flex +(same as reported by +.B \-V), +and the next line the flags used when generating the scanner, including +those that are on by default. +.TP +.B \-w +suppresses warning messages. +.TP +.B \-B +instructs +.I flex +to generate a +.I batch +scanner, the opposite of +.I interactive +scanners generated by +.B \-I +(see below). In general, you use +.B \-B +when you are +.I certain +that your scanner will never be used interactively, and you want to +squeeze a +.I little +more performance out of it. If your goal is instead to squeeze out a +.I lot +more performance, you should be using the +.B \-Cf +or +.B \-CF +options (discussed below), which turn on +.B \-B +automatically anyway. +.TP +.B \-F +specifies that the +.ul +fast +scanner table representation should be used (and stdio +bypassed). This representation is +about as fast as the full table representation +.B (-f), +and for some sets of patterns will be considerably smaller (and for +others, larger). In general, if the pattern set contains both "keywords" +and a catch-all, "identifier" rule, such as in the set: +.nf + + "case" return TOK_CASE; + "switch" return TOK_SWITCH; + ... + "default" return TOK_DEFAULT; + [a-z]+ return TOK_ID; + +.fi +then you're better off using the full table representation. If only +the "identifier" rule is present and you then use a hash table or some such +to detect the keywords, you're better off using +.B -F. +.IP +This option is equivalent to +.B \-CFr +(see below). It cannot be used with +.B \-+. +.TP +.B \-I +instructs +.I flex +to generate an +.I interactive +scanner. An interactive scanner is one that only looks ahead to decide +what token has been matched if it absolutely must. It turns out that +always looking one extra character ahead, even if the scanner has already +seen enough text to disambiguate the current token, is a bit faster than +only looking ahead when necessary. But scanners that always look ahead +give dreadful interactive performance; for example, when a user types +a newline, it is not recognized as a newline token until they enter +.I another +token, which often means typing in another whole line. +.IP +.I Flex +scanners default to +.I interactive +unless you use the +.B \-Cf +or +.B \-CF +table-compression options (see below). That's because if you're looking +for high-performance you should be using one of these options, so if you +didn't, +.I flex +assumes you'd rather trade off a bit of run-time performance for intuitive +interactive behavior. Note also that you +.I cannot +use +.B \-I +in conjunction with +.B \-Cf +or +.B \-CF. +Thus, this option is not really needed; it is on by default for all those +cases in which it is allowed. +.IP +Note that if +.B isatty() +returns false for the scanner input, flex will revert to batch mode, even if +.B \-I +was specified. To force interactive mode no matter what, use +.B %option always-interactive +(see Options below). +.IP +You can force a scanner to +.I not +be interactive by using +.B \-B +(see above). +.TP +.B \-L +instructs +.I flex +not to generate +.B #line +directives. Without this option, +.I flex +peppers the generated scanner +with #line directives so error messages in the actions will be correctly +located with respect to either the original +.I flex +input file (if the errors are due to code in the input file), or +.B lex.yy.c +(if the errors are +.I flex's +fault -- you should report these sorts of errors to the email address +given below). +.TP +.B \-T +makes +.I flex +run in +.I trace +mode. It will generate a lot of messages to +.I stderr +concerning +the form of the input and the resultant non-deterministic and deterministic +finite automata. This option is mostly for use in maintaining +.I flex. +.TP +.B \-V +prints the version number to +.I stdout +and exits. +.B \-\-version +is a synonym for +.B \-V. +.TP +.B \-7 +instructs +.I flex +to generate a 7-bit scanner, i.e., one which can only recognize 7-bit +characters in its input. The advantage of using +.B \-7 +is that the scanner's tables can be up to half the size of those generated +using the +.B \-8 +option (see below). The disadvantage is that such scanners often hang +or crash if their input contains an 8-bit character. +.IP +Note, however, that unless you generate your scanner using the +.B \-Cf +or +.B \-CF +table compression options, use of +.B \-7 +will save only a small amount of table space, and make your scanner +considerably less portable. +.I Flex's +default behavior is to generate an 8-bit scanner unless you use the +.B \-Cf +or +.B \-CF, +in which case +.I flex +defaults to generating 7-bit scanners unless your site was always +configured to generate 8-bit scanners (as will often be the case +with non-USA sites). You can tell whether flex generated a 7-bit +or an 8-bit scanner by inspecting the flag summary in the +.B \-v +output as described above. +.IP +Note that if you use +.B \-Cfe +or +.B \-CFe +(those table compression options, but also using equivalence classes as +discussed see below), flex still defaults to generating an 8-bit +scanner, since usually with these compression options full 8-bit tables +are not much more expensive than 7-bit tables. +.TP +.B \-8 +instructs +.I flex +to generate an 8-bit scanner, i.e., one which can recognize 8-bit +characters. This flag is only needed for scanners generated using +.B \-Cf +or +.B \-CF, +as otherwise flex defaults to generating an 8-bit scanner anyway. +.IP +See the discussion of +.B \-7 +above for flex's default behavior and the tradeoffs between 7-bit +and 8-bit scanners. +.TP +.B \-+ +specifies that you want flex to generate a C++ +scanner class. See the section on Generating C++ Scanners below for +details. +.TP +.B \-C[aefFmr] +controls the degree of table compression and, more generally, trade-offs +between small scanners and fast scanners. +.IP +.B \-Ca +("align") instructs flex to trade off larger tables in the +generated scanner for faster performance because the elements of +the tables are better aligned for memory access and computation. On some +RISC architectures, fetching and manipulating longwords is more efficient +than with smaller-sized units such as shortwords. This option can +double the size of the tables used by your scanner. +.IP +.B \-Ce +directs +.I flex +to construct +.I equivalence classes, +i.e., sets of characters +which have identical lexical properties (for example, if the only +appearance of digits in the +.I flex +input is in the character class +"[0-9]" then the digits '0', '1', ..., '9' will all be put +in the same equivalence class). Equivalence classes usually give +dramatic reductions in the final table/object file sizes (typically +a factor of 2-5) and are pretty cheap performance-wise (one array +look-up per character scanned). +.IP +.B \-Cf +specifies that the +.I full +scanner tables should be generated - +.I flex +should not compress the +tables by taking advantages of similar transition functions for +different states. +.IP +.B \-CF +specifies that the alternate fast scanner representation (described +above under the +.B \-F +flag) +should be used. This option cannot be used with +.B \-+. +.IP +.B \-Cm +directs +.I flex +to construct +.I meta-equivalence classes, +which are sets of equivalence classes (or characters, if equivalence +classes are not being used) that are commonly used together. Meta-equivalence +classes are often a big win when using compressed tables, but they +have a moderate performance impact (one or two "if" tests and one +array look-up per character scanned). +.IP +.B \-Cr +causes the generated scanner to +.I bypass +use of the standard I/O library (stdio) for input. Instead of calling +.B fread() +or +.B getc(), +the scanner will use the +.B read() +system call, resulting in a performance gain which varies from system +to system, but in general is probably negligible unless you are also using +.B \-Cf +or +.B \-CF. +Using +.B \-Cr +can cause strange behavior if, for example, you read from +.I yyin +using stdio prior to calling the scanner (because the scanner will miss +whatever text your previous reads left in the stdio input buffer). +.IP +.B \-Cr +has no effect if you define +.B YY_INPUT +(see The Generated Scanner above). +.IP +A lone +.B \-C +specifies that the scanner tables should be compressed but neither +equivalence classes nor meta-equivalence classes should be used. +.IP +The options +.B \-Cf +or +.B \-CF +and +.B \-Cm +do not make sense together - there is no opportunity for meta-equivalence +classes if the table is not being compressed. Otherwise the options +may be freely mixed, and are cumulative. +.IP +The default setting is +.B \-Cem, +which specifies that +.I flex +should generate equivalence classes +and meta-equivalence classes. This setting provides the highest +degree of table compression. You can trade off +faster-executing scanners at the cost of larger tables with +the following generally being true: +.nf + + slowest & smallest + -Cem + -Cm + -Ce + -C + -C{f,F}e + -C{f,F} + -C{f,F}a + fastest & largest + +.fi +Note that scanners with the smallest tables are usually generated and +compiled the quickest, so +during development you will usually want to use the default, maximal +compression. +.IP +.B \-Cfe +is often a good compromise between speed and size for production +scanners. +.TP +.B \-ooutput +directs flex to write the scanner to the file +.B output +instead of +.B lex.yy.c. +If you combine +.B \-o +with the +.B \-t +option, then the scanner is written to +.I stdout +but its +.B #line +directives (see the +.B \\-L +option above) refer to the file +.B output. +.TP +.B \-Pprefix +changes the default +.I "yy" +prefix used by +.I flex +for all globally-visible variable and function names to instead be +.I prefix. +For example, +.B \-Pfoo +changes the name of +.B yytext +to +.B footext. +It also changes the name of the default output file from +.B lex.yy.c +to +.B lex.foo.c. +Here are all of the names affected: +.nf + + yy_create_buffer + yy_delete_buffer + yy_flex_debug + yy_init_buffer + yy_flush_buffer + yy_load_buffer_state + yy_switch_to_buffer + yyin + yyleng + yylex + yylineno + yyout + yyrestart + yytext + yywrap + +.fi +(If you are using a C++ scanner, then only +.B yywrap +and +.B yyFlexLexer +are affected.) +Within your scanner itself, you can still refer to the global variables +and functions using either version of their name; but externally, they +have the modified name. +.IP +This option lets you easily link together multiple +.I flex +programs into the same executable. Note, though, that using this +option also renames +.B yywrap(), +so you now +.I must +either +provide your own (appropriately-named) version of the routine for your +scanner, or use +.B %option noyywrap, +as linking with +.B \-ll +no longer provides one for you by default. +.TP +.B \-Sskeleton_file +overrides the default skeleton file from which +.I flex +constructs its scanners. You'll never need this option unless you are doing +.I flex +maintenance or development. +.PP +.I flex +also provides a mechanism for controlling options within the +scanner specification itself, rather than from the flex command-line. +This is done by including +.B %option +directives in the first section of the scanner specification. +You can specify multiple options with a single +.B %option +directive, and multiple directives in the first section of your flex input +file. +.PP +Most options are given simply as names, optionally preceded by the +word "no" (with no intervening whitespace) to negate their meaning. +A number are equivalent to flex flags or their negation: +.nf + + 7bit -7 option + 8bit -8 option + align -Ca option + backup -b option + batch -B option + c++ -+ option + + caseful or + case-sensitive opposite of -i (default) + + case-insensitive or + caseless -i option + + debug -d option + default opposite of -s option + ecs -Ce option + fast -F option + full -f option + interactive -I option + lex-compat -l option + meta-ecs -Cm option + perf-report -p option + read -Cr option + stdout -t option + verbose -v option + warn opposite of -w option + (use "%option nowarn" for -w) + + array equivalent to "%array" + pointer equivalent to "%pointer" (default) + +.fi +Some +.B %option's +provide features otherwise not available: +.TP +.B always-interactive +instructs flex to generate a scanner which always considers its input +"interactive". Normally, on each new input file the scanner calls +.B isatty() +in an attempt to determine whether +the scanner's input source is interactive and thus should be read a +character at a time. When this option is used, however, then no +such call is made. +.TP +.B main +directs flex to provide a default +.B main() +program for the scanner, which simply calls +.B yylex(). +This option implies +.B noyywrap +(see below). +.TP +.B never-interactive +instructs flex to generate a scanner which never considers its input +"interactive" (again, no call made to +.B isatty()). +This is the opposite of +.B always-interactive. +.TP +.B stack +enables the use of start condition stacks (see Start Conditions above). +.TP +.B stdinit +if set (i.e., +.B %option stdinit) +initializes +.I yyin +and +.I yyout +to +.I stdin +and +.I stdout, +instead of the default of +.I nil. +Some existing +.I lex +programs depend on this behavior, even though it is not compliant with +ANSI C, which does not require +.I stdin +and +.I stdout +to be compile-time constant. +.TP +.B yylineno +directs +.I flex +to generate a scanner that maintains the number of the current line +read from its input in the global variable +.B yylineno. +This option is implied by +.B %option lex-compat. +.TP +.B yywrap +if unset (i.e., +.B %option noyywrap), +makes the scanner not call +.B yywrap() +upon an end-of-file, but simply assume that there are no more +files to scan (until the user points +.I yyin +at a new file and calls +.B yylex() +again). +.PP +.I flex +scans your rule actions to determine whether you use the +.B REJECT +or +.B yymore() +features. The +.B reject +and +.B yymore +options are available to override its decision as to whether you use the +options, either by setting them (e.g., +.B %option reject) +to indicate the feature is indeed used, or +unsetting them to indicate it actually is not used +(e.g., +.B %option noyymore). +.PP +Three options take string-delimited values, offset with '=': +.nf + + %option outfile="ABC" + +.fi +is equivalent to +.B -oABC, +and +.nf + + %option prefix="XYZ" + +.fi +is equivalent to +.B -PXYZ. +Finally, +.nf + + %option yyclass="foo" + +.fi +only applies when generating a C++ scanner ( +.B \-+ +option). It informs +.I flex +that you have derived +.B foo +as a subclass of +.B yyFlexLexer, +so +.I flex +will place your actions in the member function +.B foo::yylex() +instead of +.B yyFlexLexer::yylex(). +It also generates a +.B yyFlexLexer::yylex() +member function that emits a run-time error (by invoking +.B yyFlexLexer::LexerError()) +if called. +See Generating C++ Scanners, below, for additional information. +.PP +A number of options are available for lint purists who want to suppress +the appearance of unneeded routines in the generated scanner. Each of the +following, if unset +(e.g., +.B %option nounput +), results in the corresponding routine not appearing in +the generated scanner: +.nf + + input, unput + yy_push_state, yy_pop_state, yy_top_state + yy_scan_buffer, yy_scan_bytes, yy_scan_string + +.fi +(though +.B yy_push_state() +and friends won't appear anyway unless you use +.B %option stack). +.SH PERFORMANCE CONSIDERATIONS +The main design goal of +.I flex +is that it generate high-performance scanners. It has been optimized +for dealing well with large sets of rules. Aside from the effects on +scanner speed of the table compression +.B \-C +options outlined above, +there are a number of options/actions which degrade performance. These +are, from most expensive to least: +.nf + + REJECT + %option yylineno + arbitrary trailing context + + pattern sets that require backing up + %array + %option interactive + %option always-interactive + + '^' beginning-of-line operator + yymore() + +.fi +with the first three all being quite expensive and the last two +being quite cheap. Note also that +.B unput() +is implemented as a routine call that potentially does quite a bit of +work, while +.B yyless() +is a quite-cheap macro; so if just putting back some excess text you +scanned, use +.B yyless(). +.PP +.B REJECT +should be avoided at all costs when performance is important. +It is a particularly expensive option. +.PP +Getting rid of backing up is messy and often may be an enormous +amount of work for a complicated scanner. In principal, one begins +by using the +.B \-b +flag to generate a +.I lex.backup +file. For example, on the input +.nf + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + +.fi +the file looks like: +.nf + + State #6 is non-accepting - + associated rule line numbers: + 2 3 + out-transitions: [ o ] + jam-transitions: EOF [ \\001-n p-\\177 ] + + State #8 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ a ] + jam-transitions: EOF [ \\001-` b-\\177 ] + + State #9 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ r ] + jam-transitions: EOF [ \\001-q s-\\177 ] + + Compressed tables always back up. + +.fi +The first few lines tell us that there's a scanner state in +which it can make a transition on an 'o' but not on any other +character, and that in that state the currently scanned text does not match +any rule. The state occurs when trying to match the rules found +at lines 2 and 3 in the input file. +If the scanner is in that state and then reads +something other than an 'o', it will have to back up to find +a rule which is matched. With +a bit of headscratching one can see that this must be the +state it's in when it has seen "fo". When this has happened, +if anything other than another 'o' is seen, the scanner will +have to back up to simply match the 'f' (by the default rule). +.PP +The comment regarding State #8 indicates there's a problem +when "foob" has been scanned. Indeed, on any character other +than an 'a', the scanner will have to back up to accept "foo". +Similarly, the comment for State #9 concerns when "fooba" has +been scanned and an 'r' does not follow. +.PP +The final comment reminds us that there's no point going to +all the trouble of removing backing up from the rules unless +we're using +.B \-Cf +or +.B \-CF, +since there's no performance gain doing so with compressed scanners. +.PP +The way to remove the backing up is to add "error" rules: +.nf + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + fooba | + foob | + fo { + /* false alarm, not really a keyword */ + return TOK_ID; + } + +.fi +.PP +Eliminating backing up among a list of keywords can also be +done using a "catch-all" rule: +.nf + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + [a-z]+ return TOK_ID; + +.fi +This is usually the best solution when appropriate. +.PP +Backing up messages tend to cascade. +With a complicated set of rules it's not uncommon to get hundreds +of messages. If one can decipher them, though, it often +only takes a dozen or so rules to eliminate the backing up (though +it's easy to make a mistake and have an error rule accidentally match +a valid token. A possible future +.I flex +feature will be to automatically add rules to eliminate backing up). +.PP +It's important to keep in mind that you gain the benefits of eliminating +backing up only if you eliminate +.I every +instance of backing up. Leaving just one means you gain nothing. +.PP +.I Variable +trailing context (where both the leading and trailing parts do not have +a fixed length) entails almost the same performance loss as +.B REJECT +(i.e., substantial). So when possible a rule like: +.nf + + %% + mouse|rat/(cat|dog) run(); + +.fi +is better written: +.nf + + %% + mouse/cat|dog run(); + rat/cat|dog run(); + +.fi +or as +.nf + + %% + mouse|rat/cat run(); + mouse|rat/dog run(); + +.fi +Note that here the special '|' action does +.I not +provide any savings, and can even make things worse (see +Deficiencies / Bugs below). +.LP +Another area where the user can increase a scanner's performance +(and one that's easier to implement) arises from the fact that +the longer the tokens matched, the faster the scanner will run. +This is because with long tokens the processing of most input +characters takes place in the (short) inner scanning loop, and +does not often have to go through the additional work of setting up +the scanning environment (e.g., +.B yytext) +for the action. Recall the scanner for C comments: +.nf + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\\n]* + <comment>"*"+[^*/\\n]* + <comment>\\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + +.fi +This could be sped up by writing it as: +.nf + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\\n]* + <comment>[^*\\n]*\\n ++line_num; + <comment>"*"+[^*/\\n]* + <comment>"*"+[^*/\\n]*\\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + +.fi +Now instead of each newline requiring the processing of another +action, recognizing the newlines is "distributed" over the other rules +to keep the matched text as long as possible. Note that +.I adding +rules does +.I not +slow down the scanner! The speed of the scanner is independent +of the number of rules or (modulo the considerations given at the +beginning of this section) how complicated the rules are with +regard to operators such as '*' and '|'. +.PP +A final example in speeding up a scanner: suppose you want to scan +through a file containing identifiers and keywords, one per line +and with no other extraneous characters, and recognize all the +keywords. A natural first approach is: +.nf + + %% + asm | + auto | + break | + ... etc ... + volatile | + while /* it's a keyword */ + + .|\\n /* it's not a keyword */ + +.fi +To eliminate the back-tracking, introduce a catch-all rule: +.nf + + %% + asm | + auto | + break | + ... etc ... + volatile | + while /* it's a keyword */ + + [a-z]+ | + .|\\n /* it's not a keyword */ + +.fi +Now, if it's guaranteed that there's exactly one word per line, +then we can reduce the total number of matches by a half by +merging in the recognition of newlines with that of the other +tokens: +.nf + + %% + asm\\n | + auto\\n | + break\\n | + ... etc ... + volatile\\n | + while\\n /* it's a keyword */ + + [a-z]+\\n | + .|\\n /* it's not a keyword */ + +.fi +One has to be careful here, as we have now reintroduced backing up +into the scanner. In particular, while +.I we +know that there will never be any characters in the input stream +other than letters or newlines, +.I flex +can't figure this out, and it will plan for possibly needing to back up +when it has scanned a token like "auto" and then the next character +is something other than a newline or a letter. Previously it would +then just match the "auto" rule and be done, but now it has no "auto" +rule, only an "auto\\n" rule. To eliminate the possibility of backing up, +we could either duplicate all rules but without final newlines, or, +since we never expect to encounter such an input and therefore don't +how it's classified, we can introduce one more catch-all rule, this +one which doesn't include a newline: +.nf + + %% + asm\\n | + auto\\n | + break\\n | + ... etc ... + volatile\\n | + while\\n /* it's a keyword */ + + [a-z]+\\n | + [a-z]+ | + .|\\n /* it's not a keyword */ + +.fi +Compiled with +.B \-Cf, +this is about as fast as one can get a +.I flex +scanner to go for this particular problem. +.PP +A final note: +.I flex +is slow when matching NUL's, particularly when a token contains +multiple NUL's. +It's best to write rules which match +.I short +amounts of text if it's anticipated that the text will often include NUL's. +.PP +Another final note regarding performance: as mentioned above in the section +How the Input is Matched, dynamically resizing +.B yytext +to accommodate huge tokens is a slow process because it presently requires that +the (huge) token be rescanned from the beginning. Thus if performance is +vital, you should attempt to match "large" quantities of text but not +"huge" quantities, where the cutoff between the two is at about 8K +characters/token. +.SH GENERATING C++ SCANNERS +.I flex +provides two different ways to generate scanners for use with C++. The +first way is to simply compile a scanner generated by +.I flex +using a C++ compiler instead of a C compiler. You should not encounter +any compilations errors (please report any you find to the email address +given in the Author section below). You can then use C++ code in your +rule actions instead of C code. Note that the default input source for +your scanner remains +.I yyin, +and default echoing is still done to +.I yyout. +Both of these remain +.I FILE * +variables and not C++ +.I streams. +.PP +You can also use +.I flex +to generate a C++ scanner class, using the +.B \-+ +option (or, equivalently, +.B %option c++), +which is automatically specified if the name of the flex +executable ends in a '+', such as +.I flex++. +When using this option, flex defaults to generating the scanner to the file +.B lex.yy.cc +instead of +.B lex.yy.c. +The generated scanner includes the header file +.I FlexLexer.h, +which defines the interface to two C++ classes. +.PP +The first class, +.B FlexLexer, +provides an abstract base class defining the general scanner class +interface. It provides the following member functions: +.TP +.B const char* YYText() +returns the text of the most recently matched token, the equivalent of +.B yytext. +.TP +.B int YYLeng() +returns the length of the most recently matched token, the equivalent of +.B yyleng. +.TP +.B int lineno() const +returns the current input line number +(see +.B %option yylineno), +or +.B 1 +if +.B %option yylineno +was not used. +.TP +.B void set_debug( int flag ) +sets the debugging flag for the scanner, equivalent to assigning to +.B yy_flex_debug +(see the Options section above). Note that you must build the scanner +using +.B %option debug +to include debugging information in it. +.TP +.B int debug() const +returns the current setting of the debugging flag. +.PP +Also provided are member functions equivalent to +.B yy_switch_to_buffer(), +.B yy_create_buffer() +(though the first argument is an +.B istream* +object pointer and not a +.B FILE*), +.B yy_flush_buffer(), +.B yy_delete_buffer(), +and +.B yyrestart() +(again, the first argument is a +.B istream* +object pointer). +.PP +The second class defined in +.I FlexLexer.h +is +.B yyFlexLexer, +which is derived from +.B FlexLexer. +It defines the following additional member functions: +.TP +.B +yyFlexLexer( istream* arg_yyin = 0, ostream* arg_yyout = 0 ) +constructs a +.B yyFlexLexer +object using the given streams for input and output. If not specified, +the streams default to +.B cin +and +.B cout, +respectively. +.TP +.B virtual int yylex() +performs the same role is +.B yylex() +does for ordinary flex scanners: it scans the input stream, consuming +tokens, until a rule's action returns a value. If you derive a subclass +.B S +from +.B yyFlexLexer +and want to access the member functions and variables of +.B S +inside +.B yylex(), +then you need to use +.B %option yyclass="S" +to inform +.I flex +that you will be using that subclass instead of +.B yyFlexLexer. +In this case, rather than generating +.B yyFlexLexer::yylex(), +.I flex +generates +.B S::yylex() +(and also generates a dummy +.B yyFlexLexer::yylex() +that calls +.B yyFlexLexer::LexerError() +if called). +.TP +.B +virtual void switch_streams(istream* new_in = 0, +.B +ostream* new_out = 0) +reassigns +.B yyin +to +.B new_in +(if non-nil) +and +.B yyout +to +.B new_out +(ditto), deleting the previous input buffer if +.B yyin +is reassigned. +.TP +.B +int yylex( istream* new_in, ostream* new_out = 0 ) +first switches the input streams via +.B switch_streams( new_in, new_out ) +and then returns the value of +.B yylex(). +.PP +In addition, +.B yyFlexLexer +defines the following protected virtual functions which you can redefine +in derived classes to tailor the scanner: +.TP +.B +virtual int LexerInput( char* buf, int max_size ) +reads up to +.B max_size +characters into +.B buf +and returns the number of characters read. To indicate end-of-input, +return 0 characters. Note that "interactive" scanners (see the +.B \-B +and +.B \-I +flags) define the macro +.B YY_INTERACTIVE. +If you redefine +.B LexerInput() +and need to take different actions depending on whether or not +the scanner might be scanning an interactive input source, you can +test for the presence of this name via +.B #ifdef. +.TP +.B +virtual void LexerOutput( const char* buf, int size ) +writes out +.B size +characters from the buffer +.B buf, +which, while NUL-terminated, may also contain "internal" NUL's if +the scanner's rules can match text with NUL's in them. +.TP +.B +virtual void LexerError( const char* msg ) +reports a fatal error message. The default version of this function +writes the message to the stream +.B cerr +and exits. +.PP +Note that a +.B yyFlexLexer +object contains its +.I entire +scanning state. Thus you can use such objects to create reentrant +scanners. You can instantiate multiple instances of the same +.B yyFlexLexer +class, and you can also combine multiple C++ scanner classes together +in the same program using the +.B \-P +option discussed above. +.PP +Finally, note that the +.B %array +feature is not available to C++ scanner classes; you must use +.B %pointer +(the default). +.PP +Here is an example of a simple C++ scanner: +.nf + + // An example of using the flex C++ scanner class. + + %{ + int mylineno = 0; + %} + + string \\"[^\\n"]+\\" + + ws [ \\t]+ + + alpha [A-Za-z] + dig [0-9] + name ({alpha}|{dig}|\\$)({alpha}|{dig}|[_.\\-/$])* + num1 [-+]?{dig}+\\.?([eE][-+]?{dig}+)? + num2 [-+]?{dig}*\\.{dig}+([eE][-+]?{dig}+)? + number {num1}|{num2} + + %% + + {ws} /* skip blanks and tabs */ + + "/*" { + int c; + + while((c = yyinput()) != 0) + { + if(c == '\\n') + ++mylineno; + + else if(c == '*') + { + if((c = yyinput()) == '/') + break; + else + unput(c); + } + } + } + + {number} cout << "number " << YYText() << '\\n'; + + \\n mylineno++; + + {name} cout << "name " << YYText() << '\\n'; + + {string} cout << "string " << YYText() << '\\n'; + + %% + + int main( int /* argc */, char** /* argv */ ) + { + FlexLexer* lexer = new yyFlexLexer; + while(lexer->yylex() != 0) + ; + return 0; + } +.fi +If you want to create multiple (different) lexer classes, you use the +.B \-P +flag (or the +.B prefix= +option) to rename each +.B yyFlexLexer +to some other +.B xxFlexLexer. +You then can include +.B <FlexLexer.h> +in your other sources once per lexer class, first renaming +.B yyFlexLexer +as follows: +.nf + + #undef yyFlexLexer + #define yyFlexLexer xxFlexLexer + #include <FlexLexer.h> + + #undef yyFlexLexer + #define yyFlexLexer zzFlexLexer + #include <FlexLexer.h> + +.fi +if, for example, you used +.B %option prefix="xx" +for one of your scanners and +.B %option prefix="zz" +for the other. +.PP +IMPORTANT: the present form of the scanning class is +.I experimental +and may change considerably between major releases. +.SH INCOMPATIBILITIES WITH LEX AND POSIX +.I flex +is a rewrite of the AT&T Unix +.I lex +tool (the two implementations do not share any code, though), +with some extensions and incompatibilities, both of which +are of concern to those who wish to write scanners acceptable +to either implementation. Flex is fully compliant with the POSIX +.I lex +specification, except that when using +.B %pointer +(the default), a call to +.B unput() +destroys the contents of +.B yytext, +which is counter to the POSIX specification. +.PP +In this section we discuss all of the known areas of incompatibility +between flex, AT&T lex, and the POSIX specification. +.PP +.I flex's +.B \-l +option turns on maximum compatibility with the original AT&T +.I lex +implementation, at the cost of a major loss in the generated scanner's +performance. We note below which incompatibilities can be overcome +using the +.B \-l +option. +.PP +.I flex +is fully compatible with +.I lex +with the following exceptions: +.IP - +The undocumented +.I lex +scanner internal variable +.B yylineno +is not supported unless +.B \-l +or +.B %option yylineno +is used. +.IP +.B yylineno +should be maintained on a per-buffer basis, rather than a per-scanner +(single global variable) basis. +.IP +.B yylineno +is not part of the POSIX specification. +.IP - +The +.B input() +routine is not redefinable, though it may be called to read characters +following whatever has been matched by a rule. If +.B input() +encounters an end-of-file the normal +.B yywrap() +processing is done. A ``real'' end-of-file is returned by +.B input() +as +.I EOF. +.IP +Input is instead controlled by defining the +.B YY_INPUT +macro. +.IP +The +.I flex +restriction that +.B input() +cannot be redefined is in accordance with the POSIX specification, +which simply does not specify any way of controlling the +scanner's input other than by making an initial assignment to +.I yyin. +.IP - +The +.B unput() +routine is not redefinable. This restriction is in accordance with POSIX. +.IP - +.I flex +scanners are not as reentrant as +.I lex +scanners. In particular, if you have an interactive scanner and +an interrupt handler which long-jumps out of the scanner, and +the scanner is subsequently called again, you may get the following +message: +.nf + + fatal flex scanner internal error--end of buffer missed + +.fi +To reenter the scanner, first use +.nf + + yyrestart( yyin ); + +.fi +Note that this call will throw away any buffered input; usually this +isn't a problem with an interactive scanner. +.IP +Also note that flex C++ scanner classes +.I are +reentrant, so if using C++ is an option for you, you should use +them instead. See "Generating C++ Scanners" above for details. +.IP - +.B output() +is not supported. +Output from the +.B ECHO +macro is done to the file-pointer +.I yyout +(default +.I stdout). +.IP +.B output() +is not part of the POSIX specification. +.IP - +.I lex +does not support exclusive start conditions (%x), though they +are in the POSIX specification. +.IP - +When definitions are expanded, +.I flex +encloses them in parentheses. +With lex, the following: +.nf + + NAME [A-Z][A-Z0-9]* + %% + foo{NAME}? printf( "Found it\\n" ); + %% + +.fi +will not match the string "foo" because when the macro +is expanded the rule is equivalent to "foo[A-Z][A-Z0-9]*?" +and the precedence is such that the '?' is associated with +"[A-Z0-9]*". With +.I flex, +the rule will be expanded to +"foo([A-Z][A-Z0-9]*)?" and so the string "foo" will match. +.IP +Note that if the definition begins with +.B ^ +or ends with +.B $ +then it is +.I not +expanded with parentheses, to allow these operators to appear in +definitions without losing their special meanings. But the +.B <s>, /, +and +.B <<EOF>> +operators cannot be used in a +.I flex +definition. +.IP +Using +.B \-l +results in the +.I lex +behavior of no parentheses around the definition. +.IP +The POSIX specification is that the definition be enclosed in parentheses. +.IP - +Some implementations of +.I lex +allow a rule's action to begin on a separate line, if the rule's pattern +has trailing whitespace: +.nf + + %% + foo|bar<space here> + { foobar_action(); } + +.fi +.I flex +does not support this feature. +.IP - +The +.I lex +.B %r +(generate a Ratfor scanner) option is not supported. It is not part +of the POSIX specification. +.IP - +After a call to +.B unput(), +.I yytext +is undefined until the next token is matched, unless the scanner +was built using +.B %array. +This is not the case with +.I lex +or the POSIX specification. The +.B \-l +option does away with this incompatibility. +.IP - +The precedence of the +.B {} +(numeric range) operator is different. +.I lex +interprets "abc{1,3}" as "match one, two, or +three occurrences of 'abc'", whereas +.I flex +interprets it as "match 'ab' +followed by one, two, or three occurrences of 'c'". The latter is +in agreement with the POSIX specification. +.IP - +The precedence of the +.B ^ +operator is different. +.I lex +interprets "^foo|bar" as "match either 'foo' at the beginning of a line, +or 'bar' anywhere", whereas +.I flex +interprets it as "match either 'foo' or 'bar' if they come at the beginning +of a line". The latter is in agreement with the POSIX specification. +.IP - +The special table-size declarations such as +.B %a +supported by +.I lex +are not required by +.I flex +scanners; +.I flex +ignores them. +.IP - +The name +.B FLEX_SCANNER +is #define'd so scanners may be written for use with either +.I flex +or +.I lex. +Scanners also include +.B YY_FLEX_MAJOR_VERSION +and +.B YY_FLEX_MINOR_VERSION +indicating which version of +.I flex +generated the scanner +(for example, for the 2.5 release, these defines would be 2 and 5 +respectively). +.PP +The following +.I flex +features are not included in +.I lex +or the POSIX specification: +.nf + + C++ scanners + %option + start condition scopes + start condition stacks + interactive/non-interactive scanners + yy_scan_string() and friends + yyterminate() + yy_set_interactive() + yy_set_bol() + YY_AT_BOL() + <<EOF>> + <*> + YY_DECL + YY_START + YY_USER_ACTION + YY_USER_INIT + #line directives + %{}'s around actions + multiple actions on a line + +.fi +plus almost all of the flex flags. +The last feature in the list refers to the fact that with +.I flex +you can put multiple actions on the same line, separated with +semi-colons, while with +.I lex, +the following +.nf + + foo handle_foo(); ++num_foos_seen; + +.fi +is (rather surprisingly) truncated to +.nf + + foo handle_foo(); + +.fi +.I flex +does not truncate the action. Actions that are not enclosed in +braces are simply terminated at the end of the line. +.SH DIAGNOSTICS +.I warning, rule cannot be matched +indicates that the given rule +cannot be matched because it follows other rules that will +always match the same text as it. For +example, in the following "foo" cannot be matched because it comes after +an identifier "catch-all" rule: +.nf + + [a-z]+ got_identifier(); + foo got_foo(); + +.fi +Using +.B REJECT +in a scanner suppresses this warning. +.PP +.I warning, +.B \-s +.I +option given but default rule can be matched +means that it is possible (perhaps only in a particular start condition) +that the default rule (match any single character) is the only one +that will match a particular input. Since +.B \-s +was given, presumably this is not intended. +.PP +.I reject_used_but_not_detected undefined +or +.I yymore_used_but_not_detected undefined - +These errors can occur at compile time. They indicate that the +scanner uses +.B REJECT +or +.B yymore() +but that +.I flex +failed to notice the fact, meaning that +.I flex +scanned the first two sections looking for occurrences of these actions +and failed to find any, but somehow you snuck some in (via a #include +file, for example). Use +.B %option reject +or +.B %option yymore +to indicate to flex that you really do use these features. +.PP +.I flex scanner jammed - +a scanner compiled with +.B \-s +has encountered an input string which wasn't matched by +any of its rules. This error can also occur due to internal problems. +.PP +.I token too large, exceeds YYLMAX - +your scanner uses +.B %array +and one of its rules matched a string longer than the +.B YYLMAX +constant (8K bytes by default). You can increase the value by +#define'ing +.B YYLMAX +in the definitions section of your +.I flex +input. +.PP +.I scanner requires \-8 flag to +.I use the character 'x' - +Your scanner specification includes recognizing the 8-bit character +.I 'x' +and you did not specify the \-8 flag, and your scanner defaulted to 7-bit +because you used the +.B \-Cf +or +.B \-CF +table compression options. See the discussion of the +.B \-7 +flag for details. +.PP +.I flex scanner push-back overflow - +you used +.B unput() +to push back so much text that the scanner's buffer could not hold +both the pushed-back text and the current token in +.B yytext. +Ideally the scanner should dynamically resize the buffer in this case, but at +present it does not. +.PP +.I +input buffer overflow, can't enlarge buffer because scanner uses REJECT - +the scanner was working on matching an extremely large token and needed +to expand the input buffer. This doesn't work with scanners that use +.B +REJECT. +.PP +.I +fatal flex scanner internal error--end of buffer missed - +This can occur in a scanner which is reentered after a long-jump +has jumped out (or over) the scanner's activation frame. Before +reentering the scanner, use: +.nf + + yyrestart( yyin ); + +.fi +or, as noted above, switch to using the C++ scanner class. +.PP +.I too many start conditions in <> construct! - +you listed more start conditions in a <> construct than exist (so +you must have listed at least one of them twice). +.SH FILES +.TP +.B \-ll +library with which scanners must be linked. +.TP +.I lex.yy.c +generated scanner (called +.I lexyy.c +on some systems). +.TP +.I lex.yy.cc +generated C++ scanner class, when using +.B -+. +.TP +.I <FlexLexer.h> +header file defining the C++ scanner base class, +.B FlexLexer, +and its derived class, +.B yyFlexLexer. +.TP +.I flex.skl +skeleton scanner. This file is only used when building flex, not when +flex executes. +.TP +.I lex.backup +backing-up information for +.B \-b +flag (called +.I lex.bck +on some systems). +.SH DEFICIENCIES / BUGS +Some trailing context +patterns cannot be properly matched and generate +warning messages ("dangerous trailing context"). These are +patterns where the ending of the +first part of the rule matches the beginning of the second +part, such as "zx*/xy*", where the 'x*' matches the 'x' at +the beginning of the trailing context. (Note that the POSIX draft +states that the text matched by such patterns is undefined.) +.PP +For some trailing context rules, parts which are actually fixed-length are +not recognized as such, leading to the above mentioned performance loss. +In particular, parts using '|' or {n} (such as "foo{3}") are always +considered variable-length. +.PP +Combining trailing context with the special '|' action can result in +.I fixed +trailing context being turned into the more expensive +.I variable +trailing context. For example, in the following: +.nf + + %% + abc | + xyz/def + +.fi +.PP +Use of +.B unput() +invalidates yytext and yyleng, unless the +.B %array +directive +or the +.B \-l +option has been used. +.PP +Pattern-matching of NUL's is substantially slower than matching other +characters. +.PP +Dynamic resizing of the input buffer is slow, as it entails rescanning +all the text matched so far by the current (generally huge) token. +.PP +Due to both buffering of input and read-ahead, you cannot intermix +calls to <stdio.h> routines, such as, for example, +.B getchar(), +with +.I flex +rules and expect it to work. Call +.B input() +instead. +.PP +The total table entries listed by the +.B \-v +flag excludes the number of table entries needed to determine +what rule has been matched. The number of entries is equal +to the number of DFA states if the scanner does not use +.B REJECT, +and somewhat greater than the number of states if it does. +.PP +.B REJECT +cannot be used with the +.B \-f +or +.B \-F +options. +.PP +The +.I flex +internal algorithms need documentation. +.SH SEE ALSO +lex(1), yacc(1), sed(1), awk(1). +.PP +John Levine, Tony Mason, and Doug Brown, +.I Lex & Yacc, +O'Reilly and Associates. Be sure to get the 2nd edition. +.PP +M. E. Lesk and E. Schmidt, +.I LEX \- Lexical Analyzer Generator +.PP +Alfred Aho, Ravi Sethi and Jeffrey Ullman, +.I Compilers: Principles, Techniques and Tools, +Addison-Wesley (1986). Describes the pattern-matching techniques used by +.I flex +(deterministic finite automata). +.SH AUTHOR +Vern Paxson, with the help of many ideas and much inspiration from +Van Jacobson. Original version by Jef Poskanzer. The fast table +representation is a partial implementation of a design done by Van +Jacobson. The implementation was done by Kevin Gong and Vern Paxson. +.PP +Thanks to the many +.I flex +beta-testers, feedbackers, and contributors, especially Francois Pinard, +Casey Leedom, +Robert Abramovitz, +Stan Adermann, Terry Allen, David Barker-Plummer, John Basrai, +Neal Becker, Nelson H.F. Beebe, benson@odi.com, +Karl Berry, Peter A. Bigot, Simon Blanchard, +Keith Bostic, Frederic Brehm, Ian Brockbank, Kin Cho, Nick Christopher, +Brian Clapper, J.T. Conklin, +Jason Coughlin, Bill Cox, Nick Cropper, Dave Curtis, Scott David +Daniels, Chris G. Demetriou, Theo Deraadt, +Mike Donahue, Chuck Doucette, Tom Epperly, Leo Eskin, +Chris Faylor, Chris Flatters, Jon Forrest, Jeffrey Friedl, +Joe Gayda, Kaveh R. Ghazi, Wolfgang Glunz, +Eric Goldman, Christopher M. Gould, Ulrich Grepel, Peer Griebel, +Jan Hajic, Charles Hemphill, NORO Hideo, +Jarkko Hietaniemi, Scott Hofmann, +Jeff Honig, Dana Hudes, Eric Hughes, John Interrante, +Ceriel Jacobs, Michal Jaegermann, Sakari Jalovaara, Jeffrey R. Jones, +Henry Juengst, Klaus Kaempf, Jonathan I. Kamens, Terrence O Kane, +Amir Katz, ken@ken.hilco.com, Kevin B. Kenny, +Steve Kirsch, Winfried Koenig, Marq Kole, Ronald Lamprecht, +Greg Lee, Rohan Lenard, Craig Leres, John Levine, Steve Liddle, +David Loffredo, Mike Long, +Mohamed el Lozy, Brian Madsen, Malte, Joe Marshall, +Bengt Martensson, Chris Metcalf, +Luke Mewburn, Jim Meyering, R. Alexander Milowski, Erik Naggum, +G.T. Nicol, Landon Noll, James Nordby, Marc Nozell, +Richard Ohnemus, Karsten Pahnke, +Sven Panne, Roland Pesch, Walter Pelissero, Gaumond +Pierre, Esmond Pitt, Jef Poskanzer, Joe Rahmeh, Jarmo Raiha, +Frederic Raimbault, Pat Rankin, Rick Richardson, +Kevin Rodgers, Kai Uwe Rommel, Jim Roskind, Alberto Santini, +Andreas Scherer, Darrell Schiebel, Raf Schietekat, +Doug Schmidt, Philippe Schnoebelen, Andreas Schwab, +Larry Schwimmer, Alex Siegel, Eckehard Stolz, Jan-Erik Strvmquist, +Mike Stump, Paul Stuart, Dave Tallman, Ian Lance Taylor, +Chris Thewalt, Richard M. Timoney, Jodi Tsai, +Paul Tuinenga, Gary Weik, Frank Whaley, Gerhard Wilhelms, Kent Williams, Ken +Yap, Ron Zellar, Nathan Zelle, David Zuhn, +and those whose names have slipped my marginal +mail-archiving skills but whose contributions are appreciated all the +same. +.PP +Thanks to Keith Bostic, Jon Forrest, Noah Friedman, +John Gilmore, Craig Leres, John Levine, Bob Mulcahy, G.T. +Nicol, Francois Pinard, Rich Salz, and Richard Stallman for help with various +distribution headaches. +.PP +Thanks to Esmond Pitt and Earle Horton for 8-bit character support; to +Benson Margulies and Fred Burke for C++ support; to Kent Williams and Tom +Epperly for C++ class support; to Ove Ewerlid for support of NUL's; and to +Eric Hughes for support of multiple buffers. +.PP +This work was primarily done when I was with the Real Time Systems Group +at the Lawrence Berkeley Laboratory in Berkeley, CA. Many thanks to all there +for the support I received. +.PP +Send comments to vern@ee.lbl.gov. diff --git a/usr.bin/lex/lib/Makefile b/usr.bin/lex/lib/Makefile new file mode 100644 index 0000000..b43d803 --- /dev/null +++ b/usr.bin/lex/lib/Makefile @@ -0,0 +1,22 @@ +# $FreeBSD$ + +.include <bsd.own.mk> + +LIB= ln +SRCS= libmain.c libyywrap.c +NO_PIC= + +WARNS?= 2 + +.if ${MK_INSTALLLIB} != "no" +LINKS= ${LIBDIR}/libln.a ${LIBDIR}/libl.a +LINKS+= ${LIBDIR}/libln.a ${LIBDIR}/libfl.a +.endif + +.if ${MK_PROFILE} != "no" +LINKS+= ${LIBDIR}/libln_p.a ${LIBDIR}/libl_p.a +LINKS+= ${LIBDIR}/libln_p.a ${LIBDIR}/libfl_p.a +.endif + +.include <bsd.lib.mk> + diff --git a/usr.bin/lex/lib/libmain.c b/usr.bin/lex/lib/libmain.c new file mode 100644 index 0000000..426eb98 --- /dev/null +++ b/usr.bin/lex/lib/libmain.c @@ -0,0 +1,16 @@ +/* libmain - flex run-time support library "main" function */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/libmain.c,v 1.4 95/09/27 12:47:55 vern Exp $ + * $FreeBSD$ */ + +extern int yylex(); + +int main( argc, argv ) +int argc; +char *argv[]; + { + while ( yylex() != 0 ) + ; + + return 0; + } diff --git a/usr.bin/lex/lib/libyywrap.c b/usr.bin/lex/lib/libyywrap.c new file mode 100644 index 0000000..75c669c --- /dev/null +++ b/usr.bin/lex/lib/libyywrap.c @@ -0,0 +1,9 @@ +/* libyywrap - flex run-time support library "yywrap" function */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/libyywrap.c,v 1.1 93/10/02 15:23:09 vern Exp $ + * $FreeBSD$ */ + +int yywrap() + { + return 1; + } diff --git a/usr.bin/lex/main.c b/usr.bin/lex/main.c new file mode 100644 index 0000000..a037ce9 --- /dev/null +++ b/usr.bin/lex/main.c @@ -0,0 +1,1179 @@ +/* flex - tool to generate fast lexical analyzers */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +#ifndef lint +static const char copyright[] = +"@(#) Copyright (c) 1990 The Regents of the University of California.\n\ + All rights reserved.\n"; +#endif /* not lint */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/main.c,v 2.64 96/05/25 20:42:42 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + + +#include "flexdef.h" +#include "version.h" + +static char flex_version[] = FLEX_VERSION; + + +/* declare functions that have forward references */ + +void flexinit PROTO((int, char**)); +void readin PROTO((void)); +void set_up_initial_allocations PROTO((void)); + +#ifdef NEED_ARGV_FIXUP +extern void argv_fixup PROTO((int *, char ***)); +#endif + + +/* these globals are all defined and commented in flexdef.h */ +int printstats, syntaxerror, eofseen, ddebug, trace, nowarn, spprdflt; +int interactive, caseins, lex_compat, do_yylineno, useecs, fulltbl, usemecs; +int fullspd, gen_line_dirs, performance_report, backing_up_report; +int C_plus_plus, long_align, use_read, yytext_is_array, do_yywrap, csize; +int yymore_used, reject, real_reject, continued_action, in_rule; +int yymore_really_used, reject_really_used; +int datapos, dataline, linenum, out_linenum; +FILE *skelfile = NULL; +int skel_ind = 0; +char *action_array; +int action_size, defs1_offset, prolog_offset, action_offset, action_index; +char *infilename = NULL, *outfilename = NULL; +int did_outfilename; +char *prefix, *yyclass; +int do_stdinit, use_stdout; +int onestate[ONE_STACK_SIZE], onesym[ONE_STACK_SIZE]; +int onenext[ONE_STACK_SIZE], onedef[ONE_STACK_SIZE], onesp; +int current_mns, current_max_rules; +int num_rules, num_eof_rules, default_rule, lastnfa; +int *firstst, *lastst, *finalst, *transchar, *trans1, *trans2; +int *accptnum, *assoc_rule, *state_type; +int *rule_type, *rule_linenum, *rule_useful; +int current_state_type; +int variable_trailing_context_rules; +int numtemps, numprots, protprev[MSP], protnext[MSP], prottbl[MSP]; +int protcomst[MSP], firstprot, lastprot, protsave[PROT_SAVE_SIZE]; +int numecs, nextecm[CSIZE + 1], ecgroup[CSIZE + 1], nummecs, tecfwd[CSIZE + 1]; +int tecbck[CSIZE + 1]; +int lastsc, *scset, *scbol, *scxclu, *sceof; +int current_max_scs; +char **scname; +int current_max_dfa_size, current_max_xpairs; +int current_max_template_xpairs, current_max_dfas; +int lastdfa, *nxt, *chk, *tnxt; +int *base, *def, *nultrans, NUL_ec, tblend, firstfree, **dss, *dfasiz; +union dfaacc_union *dfaacc; +int *accsiz, *dhash, numas; +int numsnpairs, jambase, jamstate; +int lastccl, *cclmap, *ccllen, *cclng, cclreuse; +int current_maxccls, current_max_ccl_tbl_size; +Char *ccltbl; +char nmstr[MAXLINE]; +int sectnum, nummt, hshcol, dfaeql, numeps, eps2, num_reallocs; +int tmpuses, totnst, peakpairs, numuniq, numdup, hshsave; +int num_backing_up, bol_needed; +FILE *backing_up_file; +int end_of_buffer_state; +char **input_files; +int num_input_files; + +/* Make sure program_name is initialized so we don't crash if writing + * out an error message before getting the program name from argv[0]. + */ +char *program_name = "flex"; + +#ifndef SHORT_FILE_NAMES +static char *outfile_template = "lex.%s.%s"; +static char *backing_name = "lex.backup"; +#else +static char *outfile_template = "lex%s.%s"; +static char *backing_name = "lex.bck"; +#endif + +#ifdef THINK_C +#include <console.h> +#endif + +#ifdef MS_DOS +extern unsigned _stklen = 16384; +#endif + +static char outfile_path[MAXLINE]; +static int outfile_created = 0; +static char *skelname = NULL; + + +int main( argc, argv ) +int argc; +char **argv; + { + int i; + +#ifdef THINK_C + argc = ccommand( &argv ); +#endif +#ifdef NEED_ARGV_FIXUP + argv_fixup( &argc, &argv ); +#endif + + flexinit( argc, argv ); + + readin(); + + ntod(); + + for ( i = 1; i <= num_rules; ++i ) + if ( ! rule_useful[i] && i != default_rule ) + line_warning( _( "rule cannot be matched" ), + rule_linenum[i] ); + + if ( spprdflt && ! reject && rule_useful[default_rule] ) + line_warning( + _( "-s option given but default rule can be matched" ), + rule_linenum[default_rule] ); + + /* Generate the C state transition tables from the DFA. */ + make_tables(); + + /* Note, flexend does not return. It exits with its argument + * as status. + */ + flexend( 0 ); + + return 0; /* keep compilers/lint happy */ + } + + +/* check_options - check user-specified options */ + +void check_options() + { + int i; + + if ( lex_compat ) + { + if ( C_plus_plus ) + flexerror( _( "Can't use -+ with -l option" ) ); + + if ( fulltbl || fullspd ) + flexerror( _( "Can't use -f or -F with -l option" ) ); + + /* Don't rely on detecting use of yymore() and REJECT, + * just assume they'll be used. + */ + yymore_really_used = reject_really_used = true; + + yytext_is_array = true; + do_yylineno = true; + use_read = false; + } + + if ( do_yylineno ) + /* This should really be "maintain_backup_tables = true" */ + reject_really_used = true; + + if ( csize == unspecified ) + { + if ( (fulltbl || fullspd) && ! useecs ) + csize = DEFAULT_CSIZE; + else + csize = CSIZE; + } + + if ( interactive == unspecified ) + { + if ( fulltbl || fullspd ) + interactive = false; + else + interactive = true; + } + + if ( fulltbl || fullspd ) + { + if ( usemecs ) + flexerror( + _( "-Cf/-CF and -Cm don't make sense together" ) ); + + if ( interactive ) + flexerror( _( "-Cf/-CF and -I are incompatible" ) ); + + if ( lex_compat ) + flexerror( + _( "-Cf/-CF are incompatible with lex-compatibility mode" ) ); + + if ( do_yylineno ) + flexerror( + _( "-Cf/-CF and %option yylineno are incompatible" ) ); + + if ( fulltbl && fullspd ) + flexerror( _( "-Cf and -CF are mutually exclusive" ) ); + } + + if ( C_plus_plus && fullspd ) + flexerror( _( "Can't use -+ with -CF option" ) ); + + if ( C_plus_plus && yytext_is_array ) + { + warn( _( "%array incompatible with -+ option" ) ); + yytext_is_array = false; + } + + if ( useecs ) + { /* Set up doubly-linked equivalence classes. */ + + /* We loop all the way up to csize, since ecgroup[csize] is + * the position used for NUL characters. + */ + ecgroup[1] = NIL; + + for ( i = 2; i <= csize; ++i ) + { + ecgroup[i] = i - 1; + nextecm[i - 1] = i; + } + + nextecm[csize] = NIL; + } + + else + { + /* Put everything in its own equivalence class. */ + for ( i = 1; i <= csize; ++i ) + { + ecgroup[i] = i; + nextecm[i] = BAD_SUBSCRIPT; /* to catch errors */ + } + } + + if ( ! use_stdout ) + { + FILE *prev_stdout; + + if ( ! did_outfilename ) + { + char *suffix; + + if ( C_plus_plus ) + suffix = "cc"; + else + suffix = "c"; + + sprintf( outfile_path, outfile_template, + prefix, suffix ); + + outfilename = outfile_path; + } + + prev_stdout = freopen( outfilename, "w", stdout ); + + if ( prev_stdout == NULL ) + lerrsf( _( "could not create %s" ), outfilename ); + + outfile_created = 1; + } + + if ( skelname && (skelfile = fopen( skelname, "r" )) == NULL ) + lerrsf( _( "can't open skeleton file %s" ), skelname ); + + if ( strcmp( prefix, "yy" ) ) + { +#define GEN_PREFIX(name) out_str3( "#define yy%s %s%s\n", name, prefix, name ) + if ( C_plus_plus ) + GEN_PREFIX( "FlexLexer" ); + else + { + GEN_PREFIX( "_create_buffer" ); + GEN_PREFIX( "_delete_buffer" ); + GEN_PREFIX( "_scan_buffer" ); + GEN_PREFIX( "_scan_string" ); + GEN_PREFIX( "_scan_bytes" ); + GEN_PREFIX( "_flex_debug" ); + GEN_PREFIX( "_init_buffer" ); + GEN_PREFIX( "_flush_buffer" ); + GEN_PREFIX( "_load_buffer_state" ); + GEN_PREFIX( "_switch_to_buffer" ); + GEN_PREFIX( "in" ); + GEN_PREFIX( "leng" ); + GEN_PREFIX( "lex" ); + GEN_PREFIX( "out" ); + GEN_PREFIX( "restart" ); + GEN_PREFIX( "text" ); + + if ( do_yylineno ) + GEN_PREFIX( "lineno" ); + } + + if ( do_yywrap ) + GEN_PREFIX( "wrap" ); + + outn( "" ); + } + + if ( did_outfilename ) + line_directive_out( stdout, 0 ); + + skelout(); + } + + +/* flexend - terminate flex + * + * note + * This routine does not return. + */ + +void flexend( exit_status ) +int exit_status; + + { + int tblsiz; + int unlink(); + + if ( skelfile != NULL ) + { + if ( ferror( skelfile ) ) + lerrsf( _( "input error reading skeleton file %s" ), + skelname ); + + else if ( fclose( skelfile ) ) + lerrsf( _( "error closing skeleton file %s" ), + skelname ); + } + + if ( exit_status != 0 && outfile_created ) + { + if ( ferror( stdout ) ) + lerrsf( _( "error writing output file %s" ), + outfilename ); + + else if ( fclose( stdout ) ) + lerrsf( _( "error closing output file %s" ), + outfilename ); + + else if ( unlink( outfilename ) ) + lerrsf( _( "error deleting output file %s" ), + outfilename ); + } + + if ( backing_up_report && backing_up_file ) + { + if ( num_backing_up == 0 ) + fprintf( backing_up_file, _( "No backing up.\n" ) ); + else if ( fullspd || fulltbl ) + fprintf( backing_up_file, + _( "%d backing up (non-accepting) states.\n" ), + num_backing_up ); + else + fprintf( backing_up_file, + _( "Compressed tables always back up.\n" ) ); + + if ( ferror( backing_up_file ) ) + lerrsf( _( "error writing backup file %s" ), + backing_name ); + + else if ( fclose( backing_up_file ) ) + lerrsf( _( "error closing backup file %s" ), + backing_name ); + } + + if ( printstats ) + { + fprintf( stderr, _( "%s version %s usage statistics:\n" ), + program_name, flex_version ); + + fprintf( stderr, _( " scanner options: -" ) ); + + if ( C_plus_plus ) + putc( '+', stderr ); + if ( backing_up_report ) + putc( 'b', stderr ); + if ( ddebug ) + putc( 'd', stderr ); + if ( caseins ) + putc( 'i', stderr ); + if ( lex_compat ) + putc( 'l', stderr ); + if ( performance_report > 0 ) + putc( 'p', stderr ); + if ( performance_report > 1 ) + putc( 'p', stderr ); + if ( spprdflt ) + putc( 's', stderr ); + if ( use_stdout ) + putc( 't', stderr ); + if ( printstats ) + putc( 'v', stderr ); /* always true! */ + if ( nowarn ) + putc( 'w', stderr ); + if ( interactive == false ) + putc( 'B', stderr ); + if ( interactive == true ) + putc( 'I', stderr ); + if ( ! gen_line_dirs ) + putc( 'L', stderr ); + if ( trace ) + putc( 'T', stderr ); + + if ( csize == unspecified ) + /* We encountered an error fairly early on, so csize + * never got specified. Define it now, to prevent + * bogus table sizes being written out below. + */ + csize = 256; + + if ( csize == 128 ) + putc( '7', stderr ); + else + putc( '8', stderr ); + + fprintf( stderr, " -C" ); + + if ( long_align ) + putc( 'a', stderr ); + if ( fulltbl ) + putc( 'f', stderr ); + if ( fullspd ) + putc( 'F', stderr ); + if ( useecs ) + putc( 'e', stderr ); + if ( usemecs ) + putc( 'm', stderr ); + if ( use_read ) + putc( 'r', stderr ); + + if ( did_outfilename ) + fprintf( stderr, " -o%s", outfilename ); + + if ( skelname ) + fprintf( stderr, " -S%s", skelname ); + + if ( strcmp( prefix, "yy" ) ) + fprintf( stderr, " -P%s", prefix ); + + putc( '\n', stderr ); + + fprintf( stderr, _( " %d/%d NFA states\n" ), + lastnfa, current_mns ); + fprintf( stderr, _( " %d/%d DFA states (%d words)\n" ), + lastdfa, current_max_dfas, totnst ); + fprintf( stderr, _( " %d rules\n" ), + num_rules + num_eof_rules - 1 /* - 1 for def. rule */ ); + + if ( num_backing_up == 0 ) + fprintf( stderr, _( " No backing up\n" ) ); + else if ( fullspd || fulltbl ) + fprintf( stderr, + _( " %d backing-up (non-accepting) states\n" ), + num_backing_up ); + else + fprintf( stderr, + _( " Compressed tables always back-up\n" ) ); + + if ( bol_needed ) + fprintf( stderr, + _( " Beginning-of-line patterns used\n" ) ); + + fprintf( stderr, _( " %d/%d start conditions\n" ), lastsc, + current_max_scs ); + fprintf( stderr, + _( " %d epsilon states, %d double epsilon states\n" ), + numeps, eps2 ); + + if ( lastccl == 0 ) + fprintf( stderr, _( " no character classes\n" ) ); + else + fprintf( stderr, +_( " %d/%d character classes needed %d/%d words of storage, %d reused\n" ), + lastccl, current_maxccls, + cclmap[lastccl] + ccllen[lastccl], + current_max_ccl_tbl_size, cclreuse ); + + fprintf( stderr, _( " %d state/nextstate pairs created\n" ), + numsnpairs ); + fprintf( stderr, _( " %d/%d unique/duplicate transitions\n" ), + numuniq, numdup ); + + if ( fulltbl ) + { + tblsiz = lastdfa * numecs; + fprintf( stderr, _( " %d table entries\n" ), tblsiz ); + } + + else + { + tblsiz = 2 * (lastdfa + numtemps) + 2 * tblend; + + fprintf( stderr, + _( " %d/%d base-def entries created\n" ), + lastdfa + numtemps, current_max_dfas ); + fprintf( stderr, + _( " %d/%d (peak %d) nxt-chk entries created\n" ), + tblend, current_max_xpairs, peakpairs ); + fprintf( stderr, + _( " %d/%d (peak %d) template nxt-chk entries created\n" ), + numtemps * nummecs, + current_max_template_xpairs, + numtemps * numecs ); + fprintf( stderr, _( " %d empty table entries\n" ), + nummt ); + fprintf( stderr, _( " %d protos created\n" ), + numprots ); + fprintf( stderr, + _( " %d templates created, %d uses\n" ), + numtemps, tmpuses ); + } + + if ( useecs ) + { + tblsiz = tblsiz + csize; + fprintf( stderr, + _( " %d/%d equivalence classes created\n" ), + numecs, csize ); + } + + if ( usemecs ) + { + tblsiz = tblsiz + numecs; + fprintf( stderr, + _( " %d/%d meta-equivalence classes created\n" ), + nummecs, csize ); + } + + fprintf( stderr, + _( " %d (%d saved) hash collisions, %d DFAs equal\n" ), + hshcol, hshsave, dfaeql ); + fprintf( stderr, _( " %d sets of reallocations needed\n" ), + num_reallocs ); + fprintf( stderr, _( " %d total table entries needed\n" ), + tblsiz ); + } + + exit( exit_status ); + } + + +/* flexinit - initialize flex */ + +void flexinit( argc, argv ) +int argc; +char **argv; + { + int i, sawcmpflag; + char *arg; + + printstats = syntaxerror = trace = spprdflt = caseins = false; + lex_compat = C_plus_plus = backing_up_report = ddebug = fulltbl = false; + fullspd = long_align = nowarn = yymore_used = continued_action = false; + do_yylineno = yytext_is_array = in_rule = reject = do_stdinit = false; + yymore_really_used = reject_really_used = unspecified; + interactive = csize = unspecified; + do_yywrap = gen_line_dirs = usemecs = useecs = true; + performance_report = 0; + did_outfilename = 0; + prefix = "yy"; + yyclass = 0; + use_read = use_stdout = false; + + sawcmpflag = false; + + /* Initialize dynamic array for holding the rule actions. */ + action_size = 2048; /* default size of action array in bytes */ + action_array = allocate_character_array( action_size ); + defs1_offset = prolog_offset = action_offset = action_index = 0; + action_array[0] = '\0'; + + program_name = argv[0]; + + if ( program_name[0] != '\0' && + program_name[strlen( program_name ) - 1] == '+' ) + C_plus_plus = true; + + /* read flags */ + for ( --argc, ++argv; argc ; --argc, ++argv ) + { + arg = argv[0]; + + if ( arg[0] != '-' || arg[1] == '\0' ) + break; + + if ( arg[1] == '-' ) + { /* --option */ + if ( ! strcmp( arg, "--help" ) ) + arg = "-h"; + + else if ( ! strcmp( arg, "--version" ) ) + arg = "-V"; + + else if ( ! strcmp( arg, "--" ) ) + { /* end of options */ + --argc; + ++argv; + break; + } + } + + for ( i = 1; arg[i] != '\0'; ++i ) + switch ( arg[i] ) + { + case '+': + C_plus_plus = true; + break; + + case 'B': + interactive = false; + break; + + case 'b': + backing_up_report = true; + break; + + case 'c': + break; + + case 'C': + if ( i != 1 ) + flexerror( + _( "-C flag must be given separately" ) ); + + if ( ! sawcmpflag ) + { + useecs = false; + usemecs = false; + fulltbl = false; + sawcmpflag = true; + } + + for ( ++i; arg[i] != '\0'; ++i ) + switch ( arg[i] ) + { + case 'a': + long_align = + true; + break; + + case 'e': + useecs = true; + break; + + case 'F': + fullspd = true; + break; + + case 'f': + fulltbl = true; + break; + + case 'm': + usemecs = true; + break; + + case 'r': + use_read = true; + break; + + default: + lerrif( + _( "unknown -C option '%c'" ), + (int) arg[i] ); + break; + } + + goto get_next_arg; + + case 'd': + ddebug = true; + break; + + case 'f': + useecs = usemecs = false; + use_read = fulltbl = true; + break; + + case 'F': + useecs = usemecs = false; + use_read = fullspd = true; + break; + + case '?': + case 'h': + usage(); + exit( 0 ); + + case 'I': + interactive = true; + break; + + case 'i': + caseins = true; + break; + + case 'l': + lex_compat = true; + break; + + case 'L': + gen_line_dirs = false; + break; + + case 'n': + /* Stupid do-nothing deprecated + * option. + */ + break; + + case 'o': + if ( i != 1 ) + flexerror( + _( "-o flag must be given separately" ) ); + + outfilename = arg + i + 1; + did_outfilename = 1; + goto get_next_arg; + + case 'P': + if ( i != 1 ) + flexerror( + _( "-P flag must be given separately" ) ); + + prefix = arg + i + 1; + goto get_next_arg; + + case 'p': + ++performance_report; + break; + + case 'S': + if ( i != 1 ) + flexerror( + _( "-S flag must be given separately" ) ); + + skelname = arg + i + 1; + goto get_next_arg; + + case 's': + spprdflt = true; + break; + + case 't': + use_stdout = true; + break; + + case 'T': + trace = true; + break; + + case 'v': + printstats = true; + break; + + case 'V': + printf( _( "%s version %s\n" ), + program_name, flex_version ); + exit( 0 ); + + case 'w': + nowarn = true; + break; + + case '7': + csize = 128; + break; + + case '8': + csize = CSIZE; + break; + + default: + fprintf( stderr, + _( "%s: unknown flag '%c'. For usage, try\n\t%s --help\n" ), + program_name, (int) arg[i], + program_name ); + exit( 1 ); + } + + /* Used by -C, -S, -o, and -P flags in lieu of a "continue 2" + * control. + */ + get_next_arg: ; + } + + num_input_files = argc; + input_files = argv; + set_input_file( num_input_files > 0 ? input_files[0] : NULL ); + + lastccl = lastsc = lastdfa = lastnfa = 0; + num_rules = num_eof_rules = default_rule = 0; + numas = numsnpairs = tmpuses = 0; + numecs = numeps = eps2 = num_reallocs = hshcol = dfaeql = totnst = 0; + numuniq = numdup = hshsave = eofseen = datapos = dataline = 0; + num_backing_up = onesp = numprots = 0; + variable_trailing_context_rules = bol_needed = false; + + out_linenum = linenum = sectnum = 1; + firstprot = NIL; + + /* Used in mkprot() so that the first proto goes in slot 1 + * of the proto queue. + */ + lastprot = 1; + + set_up_initial_allocations(); + } + + +/* readin - read in the rules section of the input file(s) */ + +void readin() + { + static char yy_stdinit[] = "FILE *yyin = stdin, *yyout = stdout;"; + static char yy_nostdinit[] = + "FILE *yyin = (FILE *) 0, *yyout = (FILE *) 0;"; + + line_directive_out( (FILE *) 0, 1 ); + + if ( yyparse() ) + { + pinpoint_message( _( "fatal parse error" ) ); + flexend( 1 ); + } + + if ( syntaxerror ) + flexend( 1 ); + + if ( backing_up_report ) + { + backing_up_file = fopen( backing_name, "w" ); + if ( backing_up_file == NULL ) + lerrsf( + _( "could not create backing-up info file %s" ), + backing_name ); + } + + else + backing_up_file = NULL; + + if ( yymore_really_used == true ) + yymore_used = true; + else if ( yymore_really_used == false ) + yymore_used = false; + + if ( reject_really_used == true ) + reject = true; + else if ( reject_really_used == false ) + reject = false; + + if ( performance_report > 0 ) + { + if ( lex_compat ) + { + fprintf( stderr, +_( "-l AT&T lex compatibility option entails a large performance penalty\n" ) ); + fprintf( stderr, +_( " and may be the actual source of other reported performance penalties\n" ) ); + } + + else if ( do_yylineno ) + { + fprintf( stderr, + _( "%%option yylineno entails a large performance penalty\n" ) ); + } + + if ( performance_report > 1 ) + { + if ( interactive ) + fprintf( stderr, + _( "-I (interactive) entails a minor performance penalty\n" ) ); + + if ( yymore_used ) + fprintf( stderr, + _( "yymore() entails a minor performance penalty\n" ) ); + } + + if ( reject ) + fprintf( stderr, + _( "REJECT entails a large performance penalty\n" ) ); + + if ( variable_trailing_context_rules ) + fprintf( stderr, +_( "Variable trailing context rules entail a large performance penalty\n" ) ); + } + + if ( reject ) + real_reject = true; + + if ( variable_trailing_context_rules ) + reject = true; + + if ( (fulltbl || fullspd) && reject ) + { + if ( real_reject ) + flexerror( + _( "REJECT cannot be used with -f or -F" ) ); + else if ( do_yylineno ) + flexerror( + _( "%option yylineno cannot be used with -f or -F" ) ); + else + flexerror( + _( "variable trailing context rules cannot be used with -f or -F" ) ); + } + + if ( reject ) + outn( "\n#define YY_USES_REJECT" ); + + if ( ! do_yywrap ) + { + outn( "\n#define yywrap() 1" ); + outn( "#define YY_SKIP_YYWRAP" ); + } + + if ( ddebug ) + outn( "\n#define FLEX_DEBUG" ); + + if ( csize == 256 ) + outn( "typedef unsigned char YY_CHAR;" ); + else + outn( "typedef char YY_CHAR;" ); + + if ( C_plus_plus ) + { + outn( "#define yytext_ptr yytext" ); + + if ( interactive ) + outn( "#define YY_INTERACTIVE" ); + } + + else + { + if ( do_stdinit ) + { + outn( "#ifdef VMS" ); + outn( "#ifndef __VMS_POSIX" ); + outn( yy_nostdinit ); + outn( "#else" ); + outn( yy_stdinit ); + outn( "#endif" ); + outn( "#else" ); + outn( yy_stdinit ); + outn( "#endif" ); + } + + else + outn( yy_nostdinit ); + } + + if ( fullspd ) + outn( "typedef yyconst struct yy_trans_info *yy_state_type;" ); + else if ( ! C_plus_plus ) + outn( "typedef int yy_state_type;" ); + + if ( ddebug ) + outn( "\n#define FLEX_DEBUG" ); + + if ( lex_compat ) + outn( "#define YY_FLEX_LEX_COMPAT" ); + + if ( do_yylineno && ! C_plus_plus ) + { + outn( "extern int yylineno;" ); + outn( "int yylineno = 1;" ); + } + + if ( C_plus_plus ) + { + outn( "\n#include <FlexLexer.h>" ); + + if ( yyclass ) + { + outn( "int yyFlexLexer::yylex()" ); + outn( "\t{" ); + outn( +"\tLexerError( \"yyFlexLexer::yylex invoked but %option yyclass used\" );" ); + outn( "\treturn 0;" ); + outn( "\t}" ); + + out_str( "\n#define YY_DECL int %s::yylex()\n", + yyclass ); + } + } + + else + { + if ( yytext_is_array ) + outn( "extern char yytext[];\n" ); + + else + { + outn( "extern char *yytext;" ); + outn( "#define yytext_ptr yytext" ); + } + + if ( yyclass ) + flexerror( + _( "%option yyclass only meaningful for C++ scanners" ) ); + } + + if ( useecs ) + numecs = cre8ecs( nextecm, ecgroup, csize ); + else + numecs = csize; + + /* Now map the equivalence class for NUL to its expected place. */ + ecgroup[0] = ecgroup[csize]; + NUL_ec = ABS( ecgroup[0] ); + + if ( useecs ) + ccl2ecl(); + } + + +/* set_up_initial_allocations - allocate memory for internal tables */ + +void set_up_initial_allocations() + { + current_mns = INITIAL_MNS; + firstst = allocate_integer_array( current_mns ); + lastst = allocate_integer_array( current_mns ); + finalst = allocate_integer_array( current_mns ); + transchar = allocate_integer_array( current_mns ); + trans1 = allocate_integer_array( current_mns ); + trans2 = allocate_integer_array( current_mns ); + accptnum = allocate_integer_array( current_mns ); + assoc_rule = allocate_integer_array( current_mns ); + state_type = allocate_integer_array( current_mns ); + + current_max_rules = INITIAL_MAX_RULES; + rule_type = allocate_integer_array( current_max_rules ); + rule_linenum = allocate_integer_array( current_max_rules ); + rule_useful = allocate_integer_array( current_max_rules ); + + current_max_scs = INITIAL_MAX_SCS; + scset = allocate_integer_array( current_max_scs ); + scbol = allocate_integer_array( current_max_scs ); + scxclu = allocate_integer_array( current_max_scs ); + sceof = allocate_integer_array( current_max_scs ); + scname = allocate_char_ptr_array( current_max_scs ); + + current_maxccls = INITIAL_MAX_CCLS; + cclmap = allocate_integer_array( current_maxccls ); + ccllen = allocate_integer_array( current_maxccls ); + cclng = allocate_integer_array( current_maxccls ); + + current_max_ccl_tbl_size = INITIAL_MAX_CCL_TBL_SIZE; + ccltbl = allocate_Character_array( current_max_ccl_tbl_size ); + + current_max_dfa_size = INITIAL_MAX_DFA_SIZE; + + current_max_xpairs = INITIAL_MAX_XPAIRS; + nxt = allocate_integer_array( current_max_xpairs ); + chk = allocate_integer_array( current_max_xpairs ); + + current_max_template_xpairs = INITIAL_MAX_TEMPLATE_XPAIRS; + tnxt = allocate_integer_array( current_max_template_xpairs ); + + current_max_dfas = INITIAL_MAX_DFAS; + base = allocate_integer_array( current_max_dfas ); + def = allocate_integer_array( current_max_dfas ); + dfasiz = allocate_integer_array( current_max_dfas ); + accsiz = allocate_integer_array( current_max_dfas ); + dhash = allocate_integer_array( current_max_dfas ); + dss = allocate_int_ptr_array( current_max_dfas ); + dfaacc = allocate_dfaacc_union( current_max_dfas ); + + nultrans = (int *) 0; + } + + +void usage() + { + FILE *f = stdout; + + fprintf( f, +_( "%s [-bcdfhilnpstvwBFILTV78+? -C[aefFmr] -ooutput -Pprefix -Sskeleton]\n" ), + program_name ); + fprintf( f, _( "\t[--help --version] [file ...]\n" ) ); + + fprintf( f, _( "\t-b generate backing-up information to %s\n" ), + backing_name ); + fprintf( f, _( "\t-c do-nothing POSIX option\n" ) ); + fprintf( f, _( "\t-d turn on debug mode in generated scanner\n" ) ); + fprintf( f, _( "\t-f generate fast, large scanner\n" ) ); + fprintf( f, _( "\t-h produce this help message\n" ) ); + fprintf( f, _( "\t-i generate case-insensitive scanner\n" ) ); + fprintf( f, _( "\t-l maximal compatibility with original lex\n" ) ); + fprintf( f, _( "\t-n do-nothing POSIX option\n" ) ); + fprintf( f, _( "\t-p generate performance report to stderr\n" ) ); + fprintf( f, + _( "\t-s suppress default rule to ECHO unmatched text\n" ) ); + + if ( ! did_outfilename ) + { + sprintf( outfile_path, outfile_template, + prefix, C_plus_plus ? "cc" : "c" ); + outfilename = outfile_path; + } + + fprintf( f, + _( "\t-t write generated scanner on stdout instead of %s\n" ), + outfilename ); + + fprintf( f, + _( "\t-v write summary of scanner statistics to f\n" ) ); + fprintf( f, _( "\t-w do not generate warnings\n" ) ); + fprintf( f, _( "\t-B generate batch scanner (opposite of -I)\n" ) ); + fprintf( f, + _( "\t-F use alternative fast scanner representation\n" ) ); + fprintf( f, + _( "\t-I generate interactive scanner (opposite of -B)\n" ) ); + fprintf( f, _( "\t-L suppress #line directives in scanner\n" ) ); + fprintf( f, _( "\t-T %s should run in trace mode\n" ), program_name ); + fprintf( f, _( "\t-V report %s version\n" ), program_name ); + fprintf( f, _( "\t-7 generate 7-bit scanner\n" ) ); + fprintf( f, _( "\t-8 generate 8-bit scanner\n" ) ); + fprintf( f, _( "\t-+ generate C++ scanner class\n" ) ); + fprintf( f, _( "\t-? produce this help message\n" ) ); + fprintf( f, +_( "\t-C specify degree of table compression (default is -Cem):\n" ) ); + fprintf( f, +_( "\t\t-Ca trade off larger tables for better memory alignment\n" ) ); + fprintf( f, _( "\t\t-Ce construct equivalence classes\n" ) ); + fprintf( f, +_( "\t\t-Cf do not compress scanner tables; use -f representation\n" ) ); + fprintf( f, +_( "\t\t-CF do not compress scanner tables; use -F representation\n" ) ); + fprintf( f, _( "\t\t-Cm construct meta-equivalence classes\n" ) ); + fprintf( f, + _( "\t\t-Cr use read() instead of stdio for scanner input\n" ) ); + fprintf( f, _( "\t-o specify output filename\n" ) ); + fprintf( f, _( "\t-P specify scanner prefix other than \"yy\"\n" ) ); + fprintf( f, _( "\t-S specify skeleton file\n" ) ); + fprintf( f, _( "\t--help produce this help message\n" ) ); + fprintf( f, _( "\t--version report %s version\n" ), program_name ); + } diff --git a/usr.bin/lex/misc.c b/usr.bin/lex/misc.c new file mode 100644 index 0000000..b78c8df --- /dev/null +++ b/usr.bin/lex/misc.c @@ -0,0 +1,888 @@ +/* misc - miscellaneous flex routines */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/misc.c,v 2.47 95/04/28 11:39:39 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + + +void action_define( defname, value ) +char *defname; +int value; + { + char buf[MAXLINE]; + + if ( (int) strlen( defname ) > MAXLINE / 2 ) + { + format_pinpoint_message( _( "name \"%s\" ridiculously long" ), + defname ); + return; + } + + sprintf( buf, "#define %s %d\n", defname, value ); + add_action( buf ); + } + + +void add_action( new_text ) +char *new_text; + { + int len = strlen( new_text ); + + while ( len + action_index >= action_size - 10 /* slop */ ) + { + int new_size = action_size * 2; + + if ( new_size <= 0 ) + /* Increase just a little, to try to avoid overflow + * on 16-bit machines. + */ + action_size += action_size / 8; + else + action_size = new_size; + + action_array = + reallocate_character_array( action_array, action_size ); + } + + strcpy( &action_array[action_index], new_text ); + + action_index += len; + } + + +/* allocate_array - allocate memory for an integer array of the given size */ + +void *allocate_array( size, element_size ) +int size; +size_t element_size; + { + void *mem; + size_t num_bytes = element_size * size; + + mem = flex_alloc( num_bytes ); + if ( ! mem ) + flexfatal( + _( "memory allocation failed in allocate_array()" ) ); + + return mem; + } + + +/* all_lower - true if a string is all lower-case */ + +int all_lower( str ) +char *str; + { + while ( *str ) + { + if ( ! isascii( (Char) *str ) || ! islower( *str ) ) + return 0; + ++str; + } + + return 1; + } + + +/* all_upper - true if a string is all upper-case */ + +int all_upper( str ) +char *str; + { + while ( *str ) + { + if ( ! isascii( (Char) *str ) || ! isupper( *str ) ) + return 0; + ++str; + } + + return 1; + } + + +/* bubble - bubble sort an integer array in increasing order + * + * synopsis + * int v[n], n; + * void bubble( v, n ); + * + * description + * sorts the first n elements of array v and replaces them in + * increasing order. + * + * passed + * v - the array to be sorted + * n - the number of elements of 'v' to be sorted + */ + +void bubble( v, n ) +int v[], n; + { + int i, j, k; + + for ( i = n; i > 1; --i ) + for ( j = 1; j < i; ++j ) + if ( v[j] > v[j + 1] ) /* compare */ + { + k = v[j]; /* exchange */ + v[j] = v[j + 1]; + v[j + 1] = k; + } + } + + +/* check_char - checks a character to make sure it's within the range + * we're expecting. If not, generates fatal error message + * and exits. + */ + +void check_char( c ) +int c; + { + if ( c >= CSIZE ) + lerrsf( _( "bad character '%s' detected in check_char()" ), + readable_form( c ) ); + + if ( c >= csize ) + lerrsf( + _( "scanner requires -8 flag to use the character %s" ), + readable_form( c ) ); + } + + + +/* clower - replace upper-case letter to lower-case */ + +Char clower( c ) +int c; + { + return (Char) ((isascii( c ) && isupper( c )) ? tolower( c ) : c); + } + + +/* copy_string - returns a dynamically allocated copy of a string */ + +char *copy_string( str ) +const char *str; + { + const char *c1; + char *c2; + char *copy; + unsigned int size; + + /* find length */ + for ( c1 = str; *c1; ++c1 ) + ; + + size = (c1 - str + 1) * sizeof( char ); + copy = (char *) flex_alloc( size ); + + if ( copy == NULL ) + flexfatal( _( "dynamic memory failure in copy_string()" ) ); + + for ( c2 = copy; (*c2++ = *str++) != 0; ) + ; + + return copy; + } + + +/* copy_unsigned_string - + * returns a dynamically allocated copy of a (potentially) unsigned string + */ + +Char *copy_unsigned_string( str ) +Char *str; + { + Char *c; + Char *copy; + + /* find length */ + for ( c = str; *c; ++c ) + ; + + copy = allocate_Character_array( c - str + 1 ); + + for ( c = copy; (*c++ = *str++) != 0; ) + ; + + return copy; + } + + +/* cshell - shell sort a character array in increasing order + * + * synopsis + * + * Char v[n]; + * int n, special_case_0; + * cshell( v, n, special_case_0 ); + * + * description + * Does a shell sort of the first n elements of array v. + * If special_case_0 is true, then any element equal to 0 + * is instead assumed to have infinite weight. + * + * passed + * v - array to be sorted + * n - number of elements of v to be sorted + */ + +void cshell( v, n, special_case_0 ) +Char v[]; +int n, special_case_0; + { + int gap, i, j, jg; + Char k; + + for ( gap = n / 2; gap > 0; gap = gap / 2 ) + for ( i = gap; i < n; ++i ) + for ( j = i - gap; j >= 0; j = j - gap ) + { + jg = j + gap; + + if ( special_case_0 ) + { + if ( v[jg] == 0 ) + break; + + else if ( v[j] != 0 && v[j] <= v[jg] ) + break; + } + + else if ( v[j] <= v[jg] ) + break; + + k = v[j]; + v[j] = v[jg]; + v[jg] = k; + } + } + + +/* dataend - finish up a block of data declarations */ + +void dataend() + { + if ( datapos > 0 ) + dataflush(); + + /* add terminator for initialization; { for vi */ + outn( " } ;\n" ); + + dataline = 0; + datapos = 0; + } + + +/* dataflush - flush generated data statements */ + +void dataflush() + { + outc( '\n' ); + + if ( ++dataline >= NUMDATALINES ) + { + /* Put out a blank line so that the table is grouped into + * large blocks that enable the user to find elements easily. + */ + outc( '\n' ); + dataline = 0; + } + + /* Reset the number of characters written on the current line. */ + datapos = 0; + } + + +/* flexerror - report an error message and terminate */ + +void flexerror( msg ) +const char msg[]; + { + fprintf( stderr, "%s: %s\n", program_name, msg ); + flexend( 1 ); + } + + +/* flexfatal - report a fatal error message and terminate */ + +void flexfatal( msg ) +const char msg[]; + { + fprintf( stderr, _( "%s: fatal internal error, %s\n" ), + program_name, msg ); + exit( 1 ); + } + + +/* htoi - convert a hexadecimal digit string to an integer value */ + +int htoi( str ) +Char str[]; + { + unsigned int result; + + (void) sscanf( (char *) str, "%x", &result ); + + return result; + } + + +/* lerrif - report an error message formatted with one integer argument */ + +void lerrif( msg, arg ) +const char msg[]; +int arg; + { + char errmsg[MAXLINE]; + (void) sprintf( errmsg, msg, arg ); + flexerror( errmsg ); + } + + +/* lerrsf - report an error message formatted with one string argument */ + +void lerrsf( msg, arg ) +const char msg[], arg[]; + { + char errmsg[MAXLINE]; + + (void) sprintf( errmsg, msg, arg ); + flexerror( errmsg ); + } + + +/* line_directive_out - spit out a "#line" statement */ + +void line_directive_out( output_file, do_infile ) +FILE *output_file; +int do_infile; + { + char directive[MAXLINE], filename[MAXLINE]; + char *s1, *s2, *s3; + static char line_fmt[] = "#line %d \"%s\"\n"; + + if ( ! gen_line_dirs ) + return; + + if ( (do_infile && ! infilename) || (! do_infile && ! outfilename) ) + /* don't know the filename to use, skip */ + return; + + s1 = do_infile ? infilename : outfilename; + s2 = filename; + s3 = &filename[sizeof( filename ) - 2]; + + while ( s2 < s3 && *s1 ) + { + if ( *s1 == '\\' ) + /* Escape the '\' */ + *s2++ = '\\'; + + *s2++ = *s1++; + } + + *s2 = '\0'; + + if ( do_infile ) + sprintf( directive, line_fmt, linenum, filename ); + else + { + if ( output_file == stdout ) + /* Account for the line directive itself. */ + ++out_linenum; + + sprintf( directive, line_fmt, out_linenum, filename ); + } + + /* If output_file is nil then we should put the directive in + * the accumulated actions. + */ + if ( output_file ) + { + fputs( directive, output_file ); + } + else + add_action( directive ); + } + + +/* mark_defs1 - mark the current position in the action array as + * representing where the user's section 1 definitions end + * and the prolog begins + */ +void mark_defs1() + { + defs1_offset = 0; + action_array[action_index++] = '\0'; + action_offset = prolog_offset = action_index; + action_array[action_index] = '\0'; + } + + +/* mark_prolog - mark the current position in the action array as + * representing the end of the action prolog + */ +void mark_prolog() + { + action_array[action_index++] = '\0'; + action_offset = action_index; + action_array[action_index] = '\0'; + } + + +/* mk2data - generate a data statement for a two-dimensional array + * + * Generates a data statement initializing the current 2-D array to "value". + */ +void mk2data( value ) +int value; + { + if ( datapos >= NUMDATAITEMS ) + { + outc( ',' ); + dataflush(); + } + + if ( datapos == 0 ) + /* Indent. */ + out( " " ); + + else + outc( ',' ); + + ++datapos; + + out_dec( "%5d", value ); + } + + +/* mkdata - generate a data statement + * + * Generates a data statement initializing the current array element to + * "value". + */ +void mkdata( value ) +int value; + { + if ( datapos >= NUMDATAITEMS ) + { + outc( ',' ); + dataflush(); + } + + if ( datapos == 0 ) + /* Indent. */ + out( " " ); + else + outc( ',' ); + + ++datapos; + + out_dec( "%5d", value ); + } + + +/* myctoi - return the integer represented by a string of digits */ + +int myctoi( array ) +char array[]; + { + int val = 0; + + (void) sscanf( array, "%d", &val ); + + return val; + } + + +/* myesc - return character corresponding to escape sequence */ + +Char myesc( array ) +Char array[]; + { + Char c, esc_char; + + switch ( array[1] ) + { + case 'b': return '\b'; + case 'f': return '\f'; + case 'n': return '\n'; + case 'r': return '\r'; + case 't': return '\t'; + +#if __STDC__ + case 'a': return '\a'; + case 'v': return '\v'; +#else + case 'a': return '\007'; + case 'v': return '\013'; +#endif + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { /* \<octal> */ + int sptr = 1; + + while ( isascii( array[sptr] ) && + isdigit( array[sptr] ) ) + /* Don't increment inside loop control + * because if isdigit() is a macro it might + * expand into multiple increments ... + */ + ++sptr; + + c = array[sptr]; + array[sptr] = '\0'; + + esc_char = otoi( array + 1 ); + + array[sptr] = c; + + return esc_char; + } + + case 'x': + { /* \x<hex> */ + int sptr = 2; + + while ( isascii( array[sptr] ) && + isxdigit( (char) array[sptr] ) ) + /* Don't increment inside loop control + * because if isdigit() is a macro it might + * expand into multiple increments ... + */ + ++sptr; + + c = array[sptr]; + array[sptr] = '\0'; + + esc_char = htoi( array + 2 ); + + array[sptr] = c; + + return esc_char; + } + + default: + return array[1]; + } + } + + +/* otoi - convert an octal digit string to an integer value */ + +int otoi( str ) +Char str[]; + { + unsigned int result; + + (void) sscanf( (char *) str, "%o", &result ); + return result; + } + + +/* out - various flavors of outputing a (possibly formatted) string for the + * generated scanner, keeping track of the line count. + */ + +void out( str ) +const char str[]; + { + fputs( str, stdout ); + out_line_count( str ); + } + +void out_dec( fmt, n ) +const char fmt[]; +int n; + { + printf( fmt, n ); + out_line_count( fmt ); + } + +void out_dec2( fmt, n1, n2 ) +const char fmt[]; +int n1, n2; + { + printf( fmt, n1, n2 ); + out_line_count( fmt ); + } + +void out_hex( fmt, x ) +const char fmt[]; +unsigned int x; + { + printf( fmt, x ); + out_line_count( fmt ); + } + +void out_line_count( str ) +const char str[]; + { + int i; + + for ( i = 0; str[i]; ++i ) + if ( str[i] == '\n' ) + ++out_linenum; + } + +void out_str( fmt, str ) +const char fmt[], str[]; + { + printf( fmt, str ); + out_line_count( fmt ); + out_line_count( str ); + } + +void out_str3( fmt, s1, s2, s3 ) +const char fmt[], s1[], s2[], s3[]; + { + printf( fmt, s1, s2, s3 ); + out_line_count( fmt ); + out_line_count( s1 ); + out_line_count( s2 ); + out_line_count( s3 ); + } + +void out_str_dec( fmt, str, n ) +const char fmt[], str[]; +int n; + { + printf( fmt, str, n ); + out_line_count( fmt ); + out_line_count( str ); + } + +void outc( c ) +int c; + { + putc( c, stdout ); + + if ( c == '\n' ) + ++out_linenum; + } + +void outn( str ) +const char str[]; + { + puts( str ); + out_line_count( str ); + ++out_linenum; + } + + +/* readable_form - return the human-readable form of a character + * + * The returned string is in static storage. + */ + +char *readable_form( c ) +int c; + { + static char rform[10]; + + if ( (c >= 0 && c < 32) || c >= 127 ) + { + switch ( c ) + { + case '\b': return "\\b"; + case '\f': return "\\f"; + case '\n': return "\\n"; + case '\r': return "\\r"; + case '\t': return "\\t"; + +#if __STDC__ + case '\a': return "\\a"; + case '\v': return "\\v"; +#endif + + default: + (void) sprintf( rform, "\\%.3o", + (unsigned int) c ); + return rform; + } + } + + else if ( c == ' ' ) + return "' '"; + + else + { + rform[0] = c; + rform[1] = '\0'; + + return rform; + } + } + + +/* reallocate_array - increase the size of a dynamic array */ + +void *reallocate_array( array, size, element_size ) +void *array; +int size; +size_t element_size; + { + void *new_array; + size_t num_bytes = element_size * size; + + new_array = flex_realloc( array, num_bytes ); + if ( ! new_array ) + flexfatal( _( "attempt to increase array size failed" ) ); + + return new_array; + } + + +/* skelout - write out one section of the skeleton file + * + * Description + * Copies skelfile or skel array to stdout until a line beginning with + * "%%" or EOF is found. + */ +void skelout() + { + char buf_storage[MAXLINE]; + char *buf = buf_storage; + int do_copy = 1; + + /* Loop pulling lines either from the skelfile, if we're using + * one, or from the skel[] array. + */ + while ( skelfile ? + (fgets( buf, MAXLINE, skelfile ) != NULL) : + ((buf = (char *) skel[skel_ind++]) != 0) ) + { /* copy from skel array */ + if ( buf[0] == '%' ) + { /* control line */ + switch ( buf[1] ) + { + case '%': + return; + + case '+': + do_copy = C_plus_plus; + break; + + case '-': + do_copy = ! C_plus_plus; + break; + + case '*': + do_copy = 1; + break; + + default: + flexfatal( + _( "bad line in skeleton file" ) ); + } + } + + else if ( do_copy ) + { + if ( skelfile ) + /* Skeleton file reads include final + * newline, skel[] array does not. + */ + out( buf ); + else + outn( buf ); + } + } + } + + +/* transition_struct_out - output a yy_trans_info structure + * + * outputs the yy_trans_info structure with the two elements, element_v and + * element_n. Formats the output with spaces and carriage returns. + */ + +void transition_struct_out( element_v, element_n ) +int element_v, element_n; + { + out_dec2( " {%4d,%4d },", element_v, element_n ); + + datapos += TRANS_STRUCT_PRINT_LENGTH; + + if ( datapos >= 79 - TRANS_STRUCT_PRINT_LENGTH ) + { + outc( '\n' ); + + if ( ++dataline % 10 == 0 ) + outc( '\n' ); + + datapos = 0; + } + } + + +/* The following is only needed when building flex's parser using certain + * broken versions of bison. + */ +void *yy_flex_xmalloc( size ) +int size; + { + void *result = flex_alloc( (size_t) size ); + + if ( ! result ) + flexfatal( + _( "memory allocation failed in yy_flex_xmalloc()" ) ); + + return result; + } + + +/* zero_out - set a region of memory to 0 + * + * Sets region_ptr[0] through region_ptr[size_in_bytes - 1] to zero. + */ + +void zero_out( region_ptr, size_in_bytes ) +char *region_ptr; +size_t size_in_bytes; + { + char *rp, *rp_end; + + rp = region_ptr; + rp_end = region_ptr + size_in_bytes; + + while ( rp < rp_end ) + *rp++ = 0; + } diff --git a/usr.bin/lex/mkskel.sh b/usr.bin/lex/mkskel.sh new file mode 100755 index 0000000..a03a11a --- /dev/null +++ b/usr.bin/lex/mkskel.sh @@ -0,0 +1,16 @@ +#! /bin/sh + +cat <<! +/* File created from flex.skl via mkskel.sh */ + +#include "flexdef.h" + +const char *skel[] = { +! + +sed 's/\\/&&/g' $* | sed 's/"/\\"/g' | sed 's/.*/ "&",/' + +cat <<! + 0 +}; +! diff --git a/usr.bin/lex/nfa.c b/usr.bin/lex/nfa.c new file mode 100644 index 0000000..99b4ecf --- /dev/null +++ b/usr.bin/lex/nfa.c @@ -0,0 +1,711 @@ +/* nfa - NFA construction routines */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/nfa.c,v 2.17 95/03/04 16:11:42 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + + +/* declare functions that have forward references */ + +int dupmachine PROTO((int)); +void mkxtion PROTO((int, int)); + + +/* add_accept - add an accepting state to a machine + * + * accepting_number becomes mach's accepting number. + */ + +void add_accept( mach, accepting_number ) +int mach, accepting_number; + { + /* Hang the accepting number off an epsilon state. if it is associated + * with a state that has a non-epsilon out-transition, then the state + * will accept BEFORE it makes that transition, i.e., one character + * too soon. + */ + + if ( transchar[finalst[mach]] == SYM_EPSILON ) + accptnum[finalst[mach]] = accepting_number; + + else + { + int astate = mkstate( SYM_EPSILON ); + accptnum[astate] = accepting_number; + (void) link_machines( mach, astate ); + } + } + + +/* copysingl - make a given number of copies of a singleton machine + * + * synopsis + * + * newsng = copysingl( singl, num ); + * + * newsng - a new singleton composed of num copies of singl + * singl - a singleton machine + * num - the number of copies of singl to be present in newsng + */ + +int copysingl( singl, num ) +int singl, num; + { + int copy, i; + + copy = mkstate( SYM_EPSILON ); + + for ( i = 1; i <= num; ++i ) + copy = link_machines( copy, dupmachine( singl ) ); + + return copy; + } + + +/* dumpnfa - debugging routine to write out an nfa */ + +void dumpnfa( state1 ) +int state1; + + { + int sym, tsp1, tsp2, anum, ns; + + fprintf( stderr, + _( "\n\n********** beginning dump of nfa with start state %d\n" ), + state1 ); + + /* We probably should loop starting at firstst[state1] and going to + * lastst[state1], but they're not maintained properly when we "or" + * all of the rules together. So we use our knowledge that the machine + * starts at state 1 and ends at lastnfa. + */ + + /* for ( ns = firstst[state1]; ns <= lastst[state1]; ++ns ) */ + for ( ns = 1; ns <= lastnfa; ++ns ) + { + fprintf( stderr, _( "state # %4d\t" ), ns ); + + sym = transchar[ns]; + tsp1 = trans1[ns]; + tsp2 = trans2[ns]; + anum = accptnum[ns]; + + fprintf( stderr, "%3d: %4d, %4d", sym, tsp1, tsp2 ); + + if ( anum != NIL ) + fprintf( stderr, " [%d]", anum ); + + fprintf( stderr, "\n" ); + } + + fprintf( stderr, _( "********** end of dump\n" ) ); + } + + +/* dupmachine - make a duplicate of a given machine + * + * synopsis + * + * copy = dupmachine( mach ); + * + * copy - holds duplicate of mach + * mach - machine to be duplicated + * + * note that the copy of mach is NOT an exact duplicate; rather, all the + * transition states values are adjusted so that the copy is self-contained, + * as the original should have been. + * + * also note that the original MUST be contiguous, with its low and high + * states accessible by the arrays firstst and lastst + */ + +int dupmachine( mach ) +int mach; + { + int i, init, state_offset; + int state = 0; + int last = lastst[mach]; + + for ( i = firstst[mach]; i <= last; ++i ) + { + state = mkstate( transchar[i] ); + + if ( trans1[i] != NO_TRANSITION ) + { + mkxtion( finalst[state], trans1[i] + state - i ); + + if ( transchar[i] == SYM_EPSILON && + trans2[i] != NO_TRANSITION ) + mkxtion( finalst[state], + trans2[i] + state - i ); + } + + accptnum[state] = accptnum[i]; + } + + if ( state == 0 ) + flexfatal( _( "empty machine in dupmachine()" ) ); + + state_offset = state - i + 1; + + init = mach + state_offset; + firstst[init] = firstst[mach] + state_offset; + finalst[init] = finalst[mach] + state_offset; + lastst[init] = lastst[mach] + state_offset; + + return init; + } + + +/* finish_rule - finish up the processing for a rule + * + * An accepting number is added to the given machine. If variable_trail_rule + * is true then the rule has trailing context and both the head and trail + * are variable size. Otherwise if headcnt or trailcnt is non-zero then + * the machine recognizes a pattern with trailing context and headcnt is + * the number of characters in the matched part of the pattern, or zero + * if the matched part has variable length. trailcnt is the number of + * trailing context characters in the pattern, or zero if the trailing + * context has variable length. + */ + +void finish_rule( mach, variable_trail_rule, headcnt, trailcnt ) +int mach, variable_trail_rule, headcnt, trailcnt; + { + char action_text[MAXLINE]; + + add_accept( mach, num_rules ); + + /* We did this in new_rule(), but it often gets the wrong + * number because we do it before we start parsing the current rule. + */ + rule_linenum[num_rules] = linenum; + + /* If this is a continued action, then the line-number has already + * been updated, giving us the wrong number. + */ + if ( continued_action ) + --rule_linenum[num_rules]; + + sprintf( action_text, "case %d:\n", num_rules ); + add_action( action_text ); + + if ( variable_trail_rule ) + { + rule_type[num_rules] = RULE_VARIABLE; + + if ( performance_report > 0 ) + fprintf( stderr, + _( "Variable trailing context rule at line %d\n" ), + rule_linenum[num_rules] ); + + variable_trailing_context_rules = true; + } + + else + { + rule_type[num_rules] = RULE_NORMAL; + + if ( headcnt > 0 || trailcnt > 0 ) + { + /* Do trailing context magic to not match the trailing + * characters. + */ + char *scanner_cp = "yy_c_buf_p = yy_cp"; + char *scanner_bp = "yy_bp"; + + add_action( + "*yy_cp = yy_hold_char; /* undo effects of setting up yytext */\n" ); + + if ( headcnt > 0 ) + { + sprintf( action_text, "%s = %s + %d;\n", + scanner_cp, scanner_bp, headcnt ); + add_action( action_text ); + } + + else + { + sprintf( action_text, "%s -= %d;\n", + scanner_cp, trailcnt ); + add_action( action_text ); + } + + add_action( + "YY_DO_BEFORE_ACTION; /* set up yytext again */\n" ); + } + } + + /* Okay, in the action code at this point yytext and yyleng have + * their proper final values for this rule, so here's the point + * to do any user action. But don't do it for continued actions, + * as that'll result in multiple YY_RULE_SETUP's. + */ + if ( ! continued_action ) + add_action( "YY_RULE_SETUP\n" ); + + line_directive_out( (FILE *) 0, 1 ); + } + + +/* link_machines - connect two machines together + * + * synopsis + * + * new = link_machines( first, last ); + * + * new - a machine constructed by connecting first to last + * first - the machine whose successor is to be last + * last - the machine whose predecessor is to be first + * + * note: this routine concatenates the machine first with the machine + * last to produce a machine new which will pattern-match first first + * and then last, and will fail if either of the sub-patterns fails. + * FIRST is set to new by the operation. last is unmolested. + */ + +int link_machines( first, last ) +int first, last; + { + if ( first == NIL ) + return last; + + else if ( last == NIL ) + return first; + + else + { + mkxtion( finalst[first], last ); + finalst[first] = finalst[last]; + lastst[first] = MAX( lastst[first], lastst[last] ); + firstst[first] = MIN( firstst[first], firstst[last] ); + + return first; + } + } + + +/* mark_beginning_as_normal - mark each "beginning" state in a machine + * as being a "normal" (i.e., not trailing context- + * associated) states + * + * The "beginning" states are the epsilon closure of the first state + */ + +void mark_beginning_as_normal( mach ) +int mach; + { + switch ( state_type[mach] ) + { + case STATE_NORMAL: + /* Oh, we've already visited here. */ + return; + + case STATE_TRAILING_CONTEXT: + state_type[mach] = STATE_NORMAL; + + if ( transchar[mach] == SYM_EPSILON ) + { + if ( trans1[mach] != NO_TRANSITION ) + mark_beginning_as_normal( + trans1[mach] ); + + if ( trans2[mach] != NO_TRANSITION ) + mark_beginning_as_normal( + trans2[mach] ); + } + break; + + default: + flexerror( + _( "bad state type in mark_beginning_as_normal()" ) ); + break; + } + } + + +/* mkbranch - make a machine that branches to two machines + * + * synopsis + * + * branch = mkbranch( first, second ); + * + * branch - a machine which matches either first's pattern or second's + * first, second - machines whose patterns are to be or'ed (the | operator) + * + * Note that first and second are NEITHER destroyed by the operation. Also, + * the resulting machine CANNOT be used with any other "mk" operation except + * more mkbranch's. Compare with mkor() + */ + +int mkbranch( first, second ) +int first, second; + { + int eps; + + if ( first == NO_TRANSITION ) + return second; + + else if ( second == NO_TRANSITION ) + return first; + + eps = mkstate( SYM_EPSILON ); + + mkxtion( eps, first ); + mkxtion( eps, second ); + + return eps; + } + + +/* mkclos - convert a machine into a closure + * + * synopsis + * new = mkclos( state ); + * + * new - a new state which matches the closure of "state" + */ + +int mkclos( state ) +int state; + { + return mkopt( mkposcl( state ) ); + } + + +/* mkopt - make a machine optional + * + * synopsis + * + * new = mkopt( mach ); + * + * new - a machine which optionally matches whatever mach matched + * mach - the machine to make optional + * + * notes: + * 1. mach must be the last machine created + * 2. mach is destroyed by the call + */ + +int mkopt( mach ) +int mach; + { + int eps; + + if ( ! SUPER_FREE_EPSILON(finalst[mach]) ) + { + eps = mkstate( SYM_EPSILON ); + mach = link_machines( mach, eps ); + } + + /* Can't skimp on the following if FREE_EPSILON(mach) is true because + * some state interior to "mach" might point back to the beginning + * for a closure. + */ + eps = mkstate( SYM_EPSILON ); + mach = link_machines( eps, mach ); + + mkxtion( mach, finalst[mach] ); + + return mach; + } + + +/* mkor - make a machine that matches either one of two machines + * + * synopsis + * + * new = mkor( first, second ); + * + * new - a machine which matches either first's pattern or second's + * first, second - machines whose patterns are to be or'ed (the | operator) + * + * note that first and second are both destroyed by the operation + * the code is rather convoluted because an attempt is made to minimize + * the number of epsilon states needed + */ + +int mkor( first, second ) +int first, second; + { + int eps, orend; + + if ( first == NIL ) + return second; + + else if ( second == NIL ) + return first; + + else + { + /* See comment in mkopt() about why we can't use the first + * state of "first" or "second" if they satisfy "FREE_EPSILON". + */ + eps = mkstate( SYM_EPSILON ); + + first = link_machines( eps, first ); + + mkxtion( first, second ); + + if ( SUPER_FREE_EPSILON(finalst[first]) && + accptnum[finalst[first]] == NIL ) + { + orend = finalst[first]; + mkxtion( finalst[second], orend ); + } + + else if ( SUPER_FREE_EPSILON(finalst[second]) && + accptnum[finalst[second]] == NIL ) + { + orend = finalst[second]; + mkxtion( finalst[first], orend ); + } + + else + { + eps = mkstate( SYM_EPSILON ); + + first = link_machines( first, eps ); + orend = finalst[first]; + + mkxtion( finalst[second], orend ); + } + } + + finalst[first] = orend; + return first; + } + + +/* mkposcl - convert a machine into a positive closure + * + * synopsis + * new = mkposcl( state ); + * + * new - a machine matching the positive closure of "state" + */ + +int mkposcl( state ) +int state; + { + int eps; + + if ( SUPER_FREE_EPSILON(finalst[state]) ) + { + mkxtion( finalst[state], state ); + return state; + } + + else + { + eps = mkstate( SYM_EPSILON ); + mkxtion( eps, state ); + return link_machines( state, eps ); + } + } + + +/* mkrep - make a replicated machine + * + * synopsis + * new = mkrep( mach, lb, ub ); + * + * new - a machine that matches whatever "mach" matched from "lb" + * number of times to "ub" number of times + * + * note + * if "ub" is INFINITY then "new" matches "lb" or more occurrences of "mach" + */ + +int mkrep( mach, lb, ub ) +int mach, lb, ub; + { + int base_mach, tail, copy, i; + + base_mach = copysingl( mach, lb - 1 ); + + if ( ub == INFINITY ) + { + copy = dupmachine( mach ); + mach = link_machines( mach, + link_machines( base_mach, mkclos( copy ) ) ); + } + + else + { + tail = mkstate( SYM_EPSILON ); + + for ( i = lb; i < ub; ++i ) + { + copy = dupmachine( mach ); + tail = mkopt( link_machines( copy, tail ) ); + } + + mach = link_machines( mach, link_machines( base_mach, tail ) ); + } + + return mach; + } + + +/* mkstate - create a state with a transition on a given symbol + * + * synopsis + * + * state = mkstate( sym ); + * + * state - a new state matching sym + * sym - the symbol the new state is to have an out-transition on + * + * note that this routine makes new states in ascending order through the + * state array (and increments LASTNFA accordingly). The routine DUPMACHINE + * relies on machines being made in ascending order and that they are + * CONTIGUOUS. Change it and you will have to rewrite DUPMACHINE (kludge + * that it admittedly is) + */ + +int mkstate( sym ) +int sym; + { + if ( ++lastnfa >= current_mns ) + { + if ( (current_mns += MNS_INCREMENT) >= MAXIMUM_MNS ) + lerrif( + _( "input rules are too complicated (>= %d NFA states)" ), + current_mns ); + + ++num_reallocs; + + firstst = reallocate_integer_array( firstst, current_mns ); + lastst = reallocate_integer_array( lastst, current_mns ); + finalst = reallocate_integer_array( finalst, current_mns ); + transchar = reallocate_integer_array( transchar, current_mns ); + trans1 = reallocate_integer_array( trans1, current_mns ); + trans2 = reallocate_integer_array( trans2, current_mns ); + accptnum = reallocate_integer_array( accptnum, current_mns ); + assoc_rule = + reallocate_integer_array( assoc_rule, current_mns ); + state_type = + reallocate_integer_array( state_type, current_mns ); + } + + firstst[lastnfa] = lastnfa; + finalst[lastnfa] = lastnfa; + lastst[lastnfa] = lastnfa; + transchar[lastnfa] = sym; + trans1[lastnfa] = NO_TRANSITION; + trans2[lastnfa] = NO_TRANSITION; + accptnum[lastnfa] = NIL; + assoc_rule[lastnfa] = num_rules; + state_type[lastnfa] = current_state_type; + + /* Fix up equivalence classes base on this transition. Note that any + * character which has its own transition gets its own equivalence + * class. Thus only characters which are only in character classes + * have a chance at being in the same equivalence class. E.g. "a|b" + * puts 'a' and 'b' into two different equivalence classes. "[ab]" + * puts them in the same equivalence class (barring other differences + * elsewhere in the input). + */ + + if ( sym < 0 ) + { + /* We don't have to update the equivalence classes since + * that was already done when the ccl was created for the + * first time. + */ + } + + else if ( sym == SYM_EPSILON ) + ++numeps; + + else + { + check_char( sym ); + + if ( useecs ) + /* Map NUL's to csize. */ + mkechar( sym ? sym : csize, nextecm, ecgroup ); + } + + return lastnfa; + } + + +/* mkxtion - make a transition from one state to another + * + * synopsis + * + * mkxtion( statefrom, stateto ); + * + * statefrom - the state from which the transition is to be made + * stateto - the state to which the transition is to be made + */ + +void mkxtion( statefrom, stateto ) +int statefrom, stateto; + { + if ( trans1[statefrom] == NO_TRANSITION ) + trans1[statefrom] = stateto; + + else if ( (transchar[statefrom] != SYM_EPSILON) || + (trans2[statefrom] != NO_TRANSITION) ) + flexfatal( _( "found too many transitions in mkxtion()" ) ); + + else + { /* second out-transition for an epsilon state */ + ++eps2; + trans2[statefrom] = stateto; + } + } + +/* new_rule - initialize for a new rule */ + +void new_rule() + { + if ( ++num_rules >= current_max_rules ) + { + ++num_reallocs; + current_max_rules += MAX_RULES_INCREMENT; + rule_type = reallocate_integer_array( rule_type, + current_max_rules ); + rule_linenum = reallocate_integer_array( rule_linenum, + current_max_rules ); + rule_useful = reallocate_integer_array( rule_useful, + current_max_rules ); + } + + if ( num_rules > MAX_RULE ) + lerrif( _( "too many rules (> %d)!" ), MAX_RULE ); + + rule_linenum[num_rules] = linenum; + rule_useful[num_rules] = false; + } diff --git a/usr.bin/lex/parse.y b/usr.bin/lex/parse.y new file mode 100644 index 0000000..f3b56d8 --- /dev/null +++ b/usr.bin/lex/parse.y @@ -0,0 +1,914 @@ +/* parse.y - parser for flex input */ + +%token CHAR NUMBER SECTEND SCDECL XSCDECL NAME PREVCCL EOF_OP +%token OPTION_OP OPT_OUTFILE OPT_PREFIX OPT_YYCLASS + +%token CCE_ALNUM CCE_ALPHA CCE_BLANK CCE_CNTRL CCE_DIGIT CCE_GRAPH +%token CCE_LOWER CCE_PRINT CCE_PUNCT CCE_SPACE CCE_UPPER CCE_XDIGIT + +%{ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/parse.y,v 2.28 95/04/21 11:51:51 vern Exp $ */ +/* $FreeBSD$ */ + + +/* Some versions of bison are broken in that they use alloca() but don't + * declare it properly. The following is the patented (just kidding!) + * #ifdef chud to fix the problem, courtesy of Francois Pinard. + */ +#ifdef YYBISON +/* AIX requires this to be the first thing in the file. What a piece. */ +# ifdef _AIX + #pragma alloca +# endif +#endif + +#include "flexdef.h" + +/* The remainder of the alloca() cruft has to come after including flexdef.h, + * so HAVE_ALLOCA_H is (possibly) defined. + */ +#ifdef YYBISON +# ifdef __GNUC__ +# ifndef alloca +# define alloca __builtin_alloca +# endif +# else +# if HAVE_ALLOCA_H +# include <alloca.h> +# else +# ifdef __hpux +void *alloca (); +# else +# ifdef __TURBOC__ +# include <malloc.h> +# else +char *alloca (); +# endif +# endif +# endif +# endif +#endif + +/* Bletch, ^^^^ that was ugly! */ + + +int pat, scnum, eps, headcnt, trailcnt, anyccl, lastchar, i, rulelen; +int trlcontxt, xcluflg, currccl, cclsorted, varlength, variable_trail_rule; + +int *scon_stk; +int scon_stk_ptr; + +static int madeany = false; /* whether we've made the '.' character class */ +int previous_continued_action; /* whether the previous rule's action was '|' */ + +/* Expand a POSIX character class expression. */ +#define CCL_EXPR(func) \ + { \ + int c; \ + for ( c = 0; c < csize; ++c ) \ + if ( isascii(c) && func(c) ) \ + ccladd( currccl, c ); \ + } + +/* While POSIX defines isblank(), it's not ANSI C. */ +#define IS_BLANK(c) ((c) == ' ' || (c) == '\t') + +/* On some over-ambitious machines, such as DEC Alpha's, the default + * token type is "long" instead of "int"; this leads to problems with + * declaring yylval in flexdef.h. But so far, all the yacc's I've seen + * wrap their definitions of YYSTYPE with "#ifndef YYSTYPE"'s, so the + * following should ensure that the default token type is "int". + */ +#define YYSTYPE int + +%} + +%% +goal : initlex sect1 sect1end sect2 initforrule + { /* add default rule */ + int def_rule; + + pat = cclinit(); + cclnegate( pat ); + + def_rule = mkstate( -pat ); + + /* Remember the number of the default rule so we + * don't generate "can't match" warnings for it. + */ + default_rule = num_rules; + + finish_rule( def_rule, false, 0, 0 ); + + for ( i = 1; i <= lastsc; ++i ) + scset[i] = mkbranch( scset[i], def_rule ); + + if ( spprdflt ) + add_action( + "YY_FATAL_ERROR( \"flex scanner jammed\" )" ); + else + add_action( "ECHO" ); + + add_action( ";\n\tYY_BREAK\n" ); + } + ; + +initlex : + { /* initialize for processing rules */ + + /* Create default DFA start condition. */ + scinstal( "INITIAL", false ); + } + ; + +sect1 : sect1 startconddecl namelist1 + | sect1 options + | + | error + { synerr( "unknown error processing section 1" ); } + ; + +sect1end : SECTEND + { + check_options(); + scon_stk = allocate_integer_array( lastsc + 1 ); + scon_stk_ptr = 0; + } + ; + +startconddecl : SCDECL + { xcluflg = false; } + + | XSCDECL + { xcluflg = true; } + ; + +namelist1 : namelist1 NAME + { scinstal( nmstr, xcluflg ); } + + | NAME + { scinstal( nmstr, xcluflg ); } + + | error + { synerr( "bad start condition list" ); } + ; + +options : OPTION_OP optionlist + ; + +optionlist : optionlist option + | + ; + +option : OPT_OUTFILE '=' NAME + { + outfilename = copy_string( nmstr ); + did_outfilename = 1; + } + | OPT_PREFIX '=' NAME + { prefix = copy_string( nmstr ); } + | OPT_YYCLASS '=' NAME + { yyclass = copy_string( nmstr ); } + ; + +sect2 : sect2 scon initforrule flexrule '\n' + { scon_stk_ptr = $2; } + | sect2 scon '{' sect2 '}' + { scon_stk_ptr = $2; } + | + ; + +initforrule : + { + /* Initialize for a parse of one rule. */ + trlcontxt = variable_trail_rule = varlength = false; + trailcnt = headcnt = rulelen = 0; + current_state_type = STATE_NORMAL; + previous_continued_action = continued_action; + in_rule = true; + + new_rule(); + } + ; + +flexrule : '^' rule + { + pat = $2; + finish_rule( pat, variable_trail_rule, + headcnt, trailcnt ); + + if ( scon_stk_ptr > 0 ) + { + for ( i = 1; i <= scon_stk_ptr; ++i ) + scbol[scon_stk[i]] = + mkbranch( scbol[scon_stk[i]], + pat ); + } + + else + { + /* Add to all non-exclusive start conditions, + * including the default (0) start condition. + */ + + for ( i = 1; i <= lastsc; ++i ) + if ( ! scxclu[i] ) + scbol[i] = mkbranch( scbol[i], + pat ); + } + + if ( ! bol_needed ) + { + bol_needed = true; + + if ( performance_report > 1 ) + pinpoint_message( + "'^' operator results in sub-optimal performance" ); + } + } + + | rule + { + pat = $1; + finish_rule( pat, variable_trail_rule, + headcnt, trailcnt ); + + if ( scon_stk_ptr > 0 ) + { + for ( i = 1; i <= scon_stk_ptr; ++i ) + scset[scon_stk[i]] = + mkbranch( scset[scon_stk[i]], + pat ); + } + + else + { + for ( i = 1; i <= lastsc; ++i ) + if ( ! scxclu[i] ) + scset[i] = + mkbranch( scset[i], + pat ); + } + } + + | EOF_OP + { + if ( scon_stk_ptr > 0 ) + build_eof_action(); + + else + { + /* This EOF applies to all start conditions + * which don't already have EOF actions. + */ + for ( i = 1; i <= lastsc; ++i ) + if ( ! sceof[i] ) + scon_stk[++scon_stk_ptr] = i; + + if ( scon_stk_ptr == 0 ) + warn( + "all start conditions already have <<EOF>> rules" ); + + else + build_eof_action(); + } + } + + | error + { synerr( "unrecognized rule" ); } + ; + +scon_stk_ptr : + { $$ = scon_stk_ptr; } + ; + +scon : '<' scon_stk_ptr namelist2 '>' + { $$ = $2; } + + | '<' '*' '>' + { + $$ = scon_stk_ptr; + + for ( i = 1; i <= lastsc; ++i ) + { + int j; + + for ( j = 1; j <= scon_stk_ptr; ++j ) + if ( scon_stk[j] == i ) + break; + + if ( j > scon_stk_ptr ) + scon_stk[++scon_stk_ptr] = i; + } + } + + | + { $$ = scon_stk_ptr; } + ; + +namelist2 : namelist2 ',' sconname + + | sconname + + | error + { synerr( "bad start condition list" ); } + ; + +sconname : NAME + { + if ( (scnum = sclookup( nmstr )) == 0 ) + format_pinpoint_message( + "undeclared start condition %s", + nmstr ); + else + { + for ( i = 1; i <= scon_stk_ptr; ++i ) + if ( scon_stk[i] == scnum ) + { + format_warn( + "<%s> specified twice", + scname[scnum] ); + break; + } + + if ( i > scon_stk_ptr ) + scon_stk[++scon_stk_ptr] = scnum; + } + } + ; + +rule : re2 re + { + if ( transchar[lastst[$2]] != SYM_EPSILON ) + /* Provide final transition \now/ so it + * will be marked as a trailing context + * state. + */ + $2 = link_machines( $2, + mkstate( SYM_EPSILON ) ); + + mark_beginning_as_normal( $2 ); + current_state_type = STATE_NORMAL; + + if ( previous_continued_action ) + { + /* We need to treat this as variable trailing + * context so that the backup does not happen + * in the action but before the action switch + * statement. If the backup happens in the + * action, then the rules "falling into" this + * one's action will *also* do the backup, + * erroneously. + */ + if ( ! varlength || headcnt != 0 ) + warn( + "trailing context made variable due to preceding '|' action" ); + + /* Mark as variable. */ + varlength = true; + headcnt = 0; + } + + if ( lex_compat || (varlength && headcnt == 0) ) + { /* variable trailing context rule */ + /* Mark the first part of the rule as the + * accepting "head" part of a trailing + * context rule. + * + * By the way, we didn't do this at the + * beginning of this production because back + * then current_state_type was set up for a + * trail rule, and add_accept() can create + * a new state ... + */ + add_accept( $1, + num_rules | YY_TRAILING_HEAD_MASK ); + variable_trail_rule = true; + } + + else + trailcnt = rulelen; + + $$ = link_machines( $1, $2 ); + } + + | re2 re '$' + { synerr( "trailing context used twice" ); } + + | re '$' + { + headcnt = 0; + trailcnt = 1; + rulelen = 1; + varlength = false; + + current_state_type = STATE_TRAILING_CONTEXT; + + if ( trlcontxt ) + { + synerr( "trailing context used twice" ); + $$ = mkstate( SYM_EPSILON ); + } + + else if ( previous_continued_action ) + { + /* See the comment in the rule for "re2 re" + * above. + */ + warn( + "trailing context made variable due to preceding '|' action" ); + + varlength = true; + } + + if ( lex_compat || varlength ) + { + /* Again, see the comment in the rule for + * "re2 re" above. + */ + add_accept( $1, + num_rules | YY_TRAILING_HEAD_MASK ); + variable_trail_rule = true; + } + + trlcontxt = true; + + eps = mkstate( SYM_EPSILON ); + $$ = link_machines( $1, + link_machines( eps, mkstate( '\n' ) ) ); + } + + | re + { + $$ = $1; + + if ( trlcontxt ) + { + if ( lex_compat || (varlength && headcnt == 0) ) + /* Both head and trail are + * variable-length. + */ + variable_trail_rule = true; + else + trailcnt = rulelen; + } + } + ; + + +re : re '|' series + { + varlength = true; + $$ = mkor( $1, $3 ); + } + + | series + { $$ = $1; } + ; + + +re2 : re '/' + { + /* This rule is written separately so the + * reduction will occur before the trailing + * series is parsed. + */ + + if ( trlcontxt ) + synerr( "trailing context used twice" ); + else + trlcontxt = true; + + if ( varlength ) + /* We hope the trailing context is + * fixed-length. + */ + varlength = false; + else + headcnt = rulelen; + + rulelen = 0; + + current_state_type = STATE_TRAILING_CONTEXT; + $$ = $1; + } + ; + +series : series singleton + { + /* This is where concatenation of adjacent patterns + * gets done. + */ + $$ = link_machines( $1, $2 ); + } + + | singleton + { $$ = $1; } + ; + +singleton : singleton '*' + { + varlength = true; + + $$ = mkclos( $1 ); + } + + | singleton '+' + { + varlength = true; + $$ = mkposcl( $1 ); + } + + | singleton '?' + { + varlength = true; + $$ = mkopt( $1 ); + } + + | singleton '{' NUMBER ',' NUMBER '}' + { + varlength = true; + + if ( $3 > $5 || $3 < 0 ) + { + synerr( "bad iteration values" ); + $$ = $1; + } + else + { + if ( $3 == 0 ) + { + if ( $5 <= 0 ) + { + synerr( + "bad iteration values" ); + $$ = $1; + } + else + $$ = mkopt( + mkrep( $1, 1, $5 ) ); + } + else + $$ = mkrep( $1, $3, $5 ); + } + } + + | singleton '{' NUMBER ',' '}' + { + varlength = true; + + if ( $3 <= 0 ) + { + synerr( "iteration value must be positive" ); + $$ = $1; + } + + else + $$ = mkrep( $1, $3, INFINITY ); + } + + | singleton '{' NUMBER '}' + { + /* The singleton could be something like "(foo)", + * in which case we have no idea what its length + * is, so we punt here. + */ + varlength = true; + + if ( $3 <= 0 ) + { + synerr( "iteration value must be positive" ); + $$ = $1; + } + + else + $$ = link_machines( $1, + copysingl( $1, $3 - 1 ) ); + } + + | '.' + { + if ( ! madeany ) + { + /* Create the '.' character class. */ + anyccl = cclinit(); + ccladd( anyccl, '\n' ); + cclnegate( anyccl ); + + if ( useecs ) + mkeccl( ccltbl + cclmap[anyccl], + ccllen[anyccl], nextecm, + ecgroup, csize, csize ); + + madeany = true; + } + + ++rulelen; + + $$ = mkstate( -anyccl ); + } + + | fullccl + { + if ( ! cclsorted ) + /* Sort characters for fast searching. We + * use a shell sort since this list could + * be large. + */ + cshell( ccltbl + cclmap[$1], ccllen[$1], true ); + + if ( useecs ) + mkeccl( ccltbl + cclmap[$1], ccllen[$1], + nextecm, ecgroup, csize, csize ); + + ++rulelen; + + $$ = mkstate( -$1 ); + } + + | PREVCCL + { + ++rulelen; + + $$ = mkstate( -$1 ); + } + + | '"' string '"' + { $$ = $2; } + + | '(' re ')' + { $$ = $2; } + + | CHAR + { + ++rulelen; + + if ( caseins && $1 >= 'A' && $1 <= 'Z' ) + $1 = clower( $1 ); + + $$ = mkstate( $1 ); + } + ; + +fullccl : '[' ccl ']' + { $$ = $2; } + + | '[' '^' ccl ']' + { + cclnegate( $3 ); + $$ = $3; + } + ; + +ccl : ccl CHAR '-' CHAR + { + if ( caseins ) + { + if ( $2 >= 'A' && $2 <= 'Z' ) + $2 = clower( $2 ); + if ( $4 >= 'A' && $4 <= 'Z' ) + $4 = clower( $4 ); + } + + if ( $2 > $4 ) + synerr( "negative range in character class" ); + + else + { + for ( i = $2; i <= $4; ++i ) + ccladd( $1, i ); + + /* Keep track if this ccl is staying in + * alphabetical order. + */ + cclsorted = cclsorted && ($2 > lastchar); + lastchar = $4; + } + + $$ = $1; + } + + | ccl CHAR + { + if ( caseins && $2 >= 'A' && $2 <= 'Z' ) + $2 = clower( $2 ); + + ccladd( $1, $2 ); + cclsorted = cclsorted && ($2 > lastchar); + lastchar = $2; + $$ = $1; + } + + | ccl ccl_expr + { + /* Too hard to properly maintain cclsorted. */ + cclsorted = false; + $$ = $1; + } + + | + { + cclsorted = true; + lastchar = 0; + currccl = $$ = cclinit(); + } + ; + +ccl_expr: CCE_ALNUM { CCL_EXPR(isalnum) } + | CCE_ALPHA { CCL_EXPR(isalpha) } + | CCE_BLANK { CCL_EXPR(IS_BLANK) } + | CCE_CNTRL { CCL_EXPR(iscntrl) } + | CCE_DIGIT { CCL_EXPR(isdigit) } + | CCE_GRAPH { CCL_EXPR(isgraph) } + | CCE_LOWER { CCL_EXPR(islower) } + | CCE_PRINT { CCL_EXPR(isprint) } + | CCE_PUNCT { CCL_EXPR(ispunct) } + | CCE_SPACE { CCL_EXPR(isspace) } + | CCE_UPPER { + if ( caseins ) + CCL_EXPR(islower) + else + CCL_EXPR(isupper) + } + | CCE_XDIGIT { CCL_EXPR(isxdigit) } + ; + +string : string CHAR + { + if ( caseins && $2 >= 'A' && $2 <= 'Z' ) + $2 = clower( $2 ); + + ++rulelen; + + $$ = link_machines( $1, mkstate( $2 ) ); + } + + | + { $$ = mkstate( SYM_EPSILON ); } + ; + +%% + + +/* build_eof_action - build the "<<EOF>>" action for the active start + * conditions + */ + +void build_eof_action() + { + int i; + char action_text[MAXLINE]; + + for ( i = 1; i <= scon_stk_ptr; ++i ) + { + if ( sceof[scon_stk[i]] ) + format_pinpoint_message( + "multiple <<EOF>> rules for start condition %s", + scname[scon_stk[i]] ); + + else + { + sceof[scon_stk[i]] = true; + sprintf( action_text, "case YY_STATE_EOF(%s):\n", + scname[scon_stk[i]] ); + add_action( action_text ); + } + } + + line_directive_out( (FILE *) 0, 1 ); + + /* This isn't a normal rule after all - don't count it as + * such, so we don't have any holes in the rule numbering + * (which make generating "rule can never match" warnings + * more difficult. + */ + --num_rules; + ++num_eof_rules; + } + + +/* format_synerr - write out formatted syntax error */ + +void format_synerr( msg, arg ) +char msg[], arg[]; + { + char errmsg[MAXLINE]; + + (void) sprintf( errmsg, msg, arg ); + synerr( errmsg ); + } + + +/* synerr - report a syntax error */ + +void synerr( str ) +char str[]; + { + syntaxerror = true; + pinpoint_message( str ); + } + + +/* format_warn - write out formatted warning */ + +void format_warn( msg, arg ) +char msg[], arg[]; + { + char warn_msg[MAXLINE]; + + (void) sprintf( warn_msg, msg, arg ); + warn( warn_msg ); + } + + +/* warn - report a warning, unless -w was given */ + +void warn( str ) +char str[]; + { + line_warning( str, linenum ); + } + +/* format_pinpoint_message - write out a message formatted with one string, + * pinpointing its location + */ + +void format_pinpoint_message( msg, arg ) +char msg[], arg[]; + { + char errmsg[MAXLINE]; + + (void) sprintf( errmsg, msg, arg ); + pinpoint_message( errmsg ); + } + + +/* pinpoint_message - write out a message, pinpointing its location */ + +void pinpoint_message( str ) +char str[]; + { + line_pinpoint( str, linenum ); + } + + +/* line_warning - report a warning at a given line, unless -w was given */ + +void line_warning( str, line ) +char str[]; +int line; + { + char warning[MAXLINE]; + + if ( ! nowarn ) + { + sprintf( warning, "warning, %s", str ); + line_pinpoint( warning, line ); + } + } + + +/* line_pinpoint - write out a message, pinpointing it at the given line */ + +void line_pinpoint( str, line ) +char str[]; +int line; + { + fprintf( stderr, "\"%s\", line %d: %s\n", infilename, line, str ); + } + + +/* yyerror - eat up an error message from the parser; + * currently, messages are ignore + */ + +void yyerror( msg ) +char msg[]; + { + } diff --git a/usr.bin/lex/scan.l b/usr.bin/lex/scan.l new file mode 100644 index 0000000..b109a50 --- /dev/null +++ b/usr.bin/lex/scan.l @@ -0,0 +1,711 @@ +/* scan.l - scanner for flex input */ + +%{ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/scan.l,v 2.56 95/04/24 12:17:19 vern Exp $ */ +/* $FreeBSD$ */ + +#include "flexdef.h" +#include "parse.h" + +#define ACTION_ECHO add_action( yytext ) +#define ACTION_IFDEF(def, should_define) \ + { \ + if ( should_define ) \ + action_define( def, 1 ); \ + } + +#define MARK_END_OF_PROLOG mark_prolog(); + +#define YY_DECL \ + int flexscan() + +#define RETURNCHAR \ + yylval = (unsigned char) yytext[0]; \ + return CHAR; + +#define RETURNNAME \ + strcpy( nmstr, yytext ); \ + return NAME; + +#define PUT_BACK_STRING(str, start) \ + for ( i = strlen( str ) - 1; i >= start; --i ) \ + unput((str)[i]) + +#define CHECK_REJECT(str) \ + if ( all_upper( str ) ) \ + reject = true; + +#define CHECK_YYMORE(str) \ + if ( all_lower( str ) ) \ + yymore_used = true; +%} + +%option caseless nodefault outfile="scan.c" stack noyy_top_state +%option nostdinit + +%x SECT2 SECT2PROLOG SECT3 CODEBLOCK PICKUPDEF SC CARETISBOL NUM QUOTE +%x FIRSTCCL CCL ACTION RECOVER COMMENT ACTION_STRING PERCENT_BRACE_ACTION +%x OPTION LINEDIR + +WS [[:blank:]]+ +OPTWS [[:blank:]]* +NOT_WS [^[:blank:]\n] + +NL \r?\n + +NAME ([[:alpha:]_][[:alnum:]_-]*) +NOT_NAME [^[:alpha:]_*\n]+ + +SCNAME {NAME} + +ESCSEQ (\\([^\n]|[0-7]{1,3}|x[[:xdigit:]]{1,2})) + +FIRST_CCL_CHAR ([^\\\n]|{ESCSEQ}) +CCL_CHAR ([^\\\n\]]|{ESCSEQ}) +CCL_EXPR ("[:"[[:alpha:]]+":]") + +LEXOPT [aceknopr] + +%% + static int bracelevel, didadef, indented_code; + static int doing_rule_action = false; + static int option_sense; + + int doing_codeblock = false; + int i; + Char nmdef[MAXLINE], myesc(); + + +<INITIAL>{ + ^{WS} indented_code = true; BEGIN(CODEBLOCK); + ^"/*" ACTION_ECHO; yy_push_state( COMMENT ); + ^#{OPTWS}line{WS} yy_push_state( LINEDIR ); + ^"%s"{NAME}? return SCDECL; + ^"%x"{NAME}? return XSCDECL; + ^"%{".*{NL} { + ++linenum; + line_directive_out( (FILE *) 0, 1 ); + indented_code = false; + BEGIN(CODEBLOCK); + } + + {WS} /* discard */ + + ^"%%".* { + sectnum = 2; + bracelevel = 0; + mark_defs1(); + line_directive_out( (FILE *) 0, 1 ); + BEGIN(SECT2PROLOG); + return SECTEND; + } + + ^"%pointer".*{NL} yytext_is_array = false; ++linenum; + ^"%array".*{NL} yytext_is_array = true; ++linenum; + + ^"%option" BEGIN(OPTION); return OPTION_OP; + + ^"%"{LEXOPT}{OPTWS}[[:digit:]]*{OPTWS}{NL} ++linenum; /* ignore */ + ^"%"{LEXOPT}{WS}.*{NL} ++linenum; /* ignore */ + + ^"%"[^sxaceknopr{}].* synerr( _( "unrecognized '%' directive" ) ); + + ^{NAME} { + strcpy( nmstr, yytext ); + didadef = false; + BEGIN(PICKUPDEF); + } + + {SCNAME} RETURNNAME; + ^{OPTWS}{NL} ++linenum; /* allows blank lines in section 1 */ + {OPTWS}{NL} ACTION_ECHO; ++linenum; /* maybe end of comment line */ +} + + +<COMMENT>{ + "*/" ACTION_ECHO; yy_pop_state(); + "*" ACTION_ECHO; + [^*\n]+ ACTION_ECHO; + [^*\n]*{NL} ++linenum; ACTION_ECHO; +} + +<LINEDIR>{ + \n yy_pop_state(); + [[:digit:]]+ linenum = myctoi( yytext ); + + \"[^"\n]*\" { + flex_free( (void *) infilename ); + infilename = copy_string( yytext + 1 ); + infilename[strlen( infilename ) - 1] = '\0'; + } + . /* ignore spurious characters */ +} + +<CODEBLOCK>{ + ^"%}".*{NL} ++linenum; BEGIN(INITIAL); + + {NAME}|{NOT_NAME}|. ACTION_ECHO; + + {NL} { + ++linenum; + ACTION_ECHO; + if ( indented_code ) + BEGIN(INITIAL); + } +} + + +<PICKUPDEF>{ + {WS} /* separates name and definition */ + + {NOT_WS}.* { + strcpy( (char *) nmdef, yytext ); + + /* Skip trailing whitespace. */ + for ( i = strlen( (char *) nmdef ) - 1; + i >= 0 && (nmdef[i] == ' ' || nmdef[i] == '\t'); + --i ) + ; + + nmdef[i + 1] = '\0'; + + ndinstal( nmstr, nmdef ); + didadef = true; + } + + {NL} { + if ( ! didadef ) + synerr( _( "incomplete name definition" ) ); + BEGIN(INITIAL); + ++linenum; + } +} + + +<OPTION>{ + {NL} ++linenum; BEGIN(INITIAL); + {WS} option_sense = true; + + "=" return '='; + + no option_sense = ! option_sense; + + 7bit csize = option_sense ? 128 : 256; + 8bit csize = option_sense ? 256 : 128; + + align long_align = option_sense; + always-interactive { + action_define( "YY_ALWAYS_INTERACTIVE", option_sense ); + } + array yytext_is_array = option_sense; + backup backing_up_report = option_sense; + batch interactive = ! option_sense; + "c++" C_plus_plus = option_sense; + caseful|case-sensitive caseins = ! option_sense; + caseless|case-insensitive caseins = option_sense; + debug ddebug = option_sense; + default spprdflt = ! option_sense; + ecs useecs = option_sense; + fast { + useecs = usemecs = false; + use_read = fullspd = true; + } + full { + useecs = usemecs = false; + use_read = fulltbl = true; + } + input ACTION_IFDEF("YY_NO_INPUT", ! option_sense); + interactive interactive = option_sense; + lex-compat lex_compat = option_sense; + main { + action_define( "YY_MAIN", option_sense ); + do_yywrap = ! option_sense; + } + meta-ecs usemecs = option_sense; + never-interactive { + action_define( "YY_NEVER_INTERACTIVE", option_sense ); + } + perf-report performance_report += option_sense ? 1 : -1; + pointer yytext_is_array = ! option_sense; + read use_read = option_sense; + reject reject_really_used = option_sense; + stack action_define( "YY_STACK_USED", option_sense ); + stdinit do_stdinit = option_sense; + stdout use_stdout = option_sense; + unput ACTION_IFDEF("YY_NO_UNPUT", ! option_sense); + verbose printstats = option_sense; + warn nowarn = ! option_sense; + yylineno do_yylineno = option_sense; + yymore yymore_really_used = option_sense; + yywrap do_yywrap = option_sense; + + yy_push_state ACTION_IFDEF("YY_NO_PUSH_STATE", ! option_sense); + yy_pop_state ACTION_IFDEF("YY_NO_POP_STATE", ! option_sense); + yy_top_state ACTION_IFDEF("YY_NO_TOP_STATE", ! option_sense); + + yy_scan_buffer ACTION_IFDEF("YY_NO_SCAN_BUFFER", ! option_sense); + yy_scan_bytes ACTION_IFDEF("YY_NO_SCAN_BYTES", ! option_sense); + yy_scan_string ACTION_IFDEF("YY_NO_SCAN_STRING", ! option_sense); + + outfile return OPT_OUTFILE; + prefix return OPT_PREFIX; + yyclass return OPT_YYCLASS; + + \"[^"\n]*\" { + strcpy( nmstr, yytext + 1 ); + nmstr[strlen( nmstr ) - 1] = '\0'; + return NAME; + } + + (([a-mo-z]|n[a-np-z])[[:alpha:]\-+]*)|. { + format_synerr( _( "unrecognized %%option: %s" ), + yytext ); + BEGIN(RECOVER); + } +} + +<RECOVER>.*{NL} ++linenum; BEGIN(INITIAL); + + +<SECT2PROLOG>{ + ^"%{".* ++bracelevel; yyless( 2 ); /* eat only %{ */ + ^"%}".* --bracelevel; yyless( 2 ); /* eat only %} */ + + ^{WS}.* ACTION_ECHO; /* indented code in prolog */ + + ^{NOT_WS}.* { /* non-indented code */ + if ( bracelevel <= 0 ) + { /* not in %{ ... %} */ + yyless( 0 ); /* put it all back */ + yy_set_bol( 1 ); + mark_prolog(); + BEGIN(SECT2); + } + else + ACTION_ECHO; + } + + .* ACTION_ECHO; + {NL} ++linenum; ACTION_ECHO; + + <<EOF>> { + mark_prolog(); + sectnum = 0; + yyterminate(); /* to stop the parser */ + } +} + +<SECT2>{ + ^{OPTWS}{NL} ++linenum; /* allow blank lines in section 2 */ + + ^{OPTWS}"%{" { + indented_code = false; + doing_codeblock = true; + bracelevel = 1; + BEGIN(PERCENT_BRACE_ACTION); + } + + ^{OPTWS}"<" BEGIN(SC); return '<'; + ^{OPTWS}"^" return '^'; + \" BEGIN(QUOTE); return '"'; + "{"/[[:digit:]] BEGIN(NUM); return '{'; + "$"/([[:blank:]]|{NL}) return '$'; + + {WS}"%{" { + bracelevel = 1; + BEGIN(PERCENT_BRACE_ACTION); + + if ( in_rule ) + { + doing_rule_action = true; + in_rule = false; + return '\n'; + } + } + {WS}"|".*{NL} continued_action = true; ++linenum; return '\n'; + + ^{WS}"/*" { + yyless( yyleng - 2 ); /* put back '/', '*' */ + bracelevel = 0; + continued_action = false; + BEGIN(ACTION); + } + + ^{WS} /* allow indented rules */ + + {WS} { + /* This rule is separate from the one below because + * otherwise we get variable trailing context, so + * we can't build the scanner using -{f,F}. + */ + bracelevel = 0; + continued_action = false; + BEGIN(ACTION); + + if ( in_rule ) + { + doing_rule_action = true; + in_rule = false; + return '\n'; + } + } + + {OPTWS}{NL} { + bracelevel = 0; + continued_action = false; + BEGIN(ACTION); + unput( '\n' ); /* so <ACTION> sees it */ + + if ( in_rule ) + { + doing_rule_action = true; + in_rule = false; + return '\n'; + } + } + + ^{OPTWS}"<<EOF>>" | + "<<EOF>>" return EOF_OP; + + ^"%%".* { + sectnum = 3; + BEGIN(SECT3); + yyterminate(); /* to stop the parser */ + } + + "["({FIRST_CCL_CHAR}|{CCL_EXPR})({CCL_CHAR}|{CCL_EXPR})* { + int cclval; + + strcpy( nmstr, yytext ); + + /* Check to see if we've already encountered this + * ccl. + */ + if ( (cclval = ccllookup( (Char *) nmstr )) != 0 ) + { + if ( input() != ']' ) + synerr( _( "bad character class" ) ); + + yylval = cclval; + ++cclreuse; + return PREVCCL; + } + else + { + /* We fudge a bit. We know that this ccl will + * soon be numbered as lastccl + 1 by cclinit. + */ + cclinstal( (Char *) nmstr, lastccl + 1 ); + + /* Push back everything but the leading bracket + * so the ccl can be rescanned. + */ + yyless( 1 ); + + BEGIN(FIRSTCCL); + return '['; + } + } + + "{"{NAME}"}" { + Char *nmdefptr; + Char *ndlookup(); + + strcpy( nmstr, yytext + 1 ); + nmstr[yyleng - 2] = '\0'; /* chop trailing brace */ + + if ( (nmdefptr = ndlookup( nmstr )) == 0 ) + format_synerr( + _( "undefined definition {%s}" ), + nmstr ); + + else + { /* push back name surrounded by ()'s */ + int len = strlen( (char *) nmdefptr ); + + if ( lex_compat || nmdefptr[0] == '^' || + (len > 0 && nmdefptr[len - 1] == '$') ) + { /* don't use ()'s after all */ + PUT_BACK_STRING((char *) nmdefptr, 0); + + if ( nmdefptr[0] == '^' ) + BEGIN(CARETISBOL); + } + + else + { + unput(')'); + PUT_BACK_STRING((char *) nmdefptr, 0); + unput('('); + } + } + } + + [/|*+?.(){}] return (unsigned char) yytext[0]; + . RETURNCHAR; +} + + +<SC>{ + [,*] return (unsigned char) yytext[0]; + ">" BEGIN(SECT2); return '>'; + ">"/^ BEGIN(CARETISBOL); return '>'; + {SCNAME} RETURNNAME; + . { + format_synerr( _( "bad <start condition>: %s" ), + yytext ); + } +} + +<CARETISBOL>"^" BEGIN(SECT2); return '^'; + + +<QUOTE>{ + [^"\n] RETURNCHAR; + \" BEGIN(SECT2); return '"'; + + {NL} { + synerr( _( "missing quote" ) ); + BEGIN(SECT2); + ++linenum; + return '"'; + } +} + + +<FIRSTCCL>{ + "^"/[^-\]\n] BEGIN(CCL); return '^'; + "^"/("-"|"]") return '^'; + . BEGIN(CCL); RETURNCHAR; +} + +<CCL>{ + -/[^\]\n] return '-'; + [^\]\n] RETURNCHAR; + "]" BEGIN(SECT2); return ']'; + .|{NL} { + synerr( _( "bad character class" ) ); + BEGIN(SECT2); + return ']'; + } +} + +<FIRSTCCL,CCL>{ + "[:alnum:]" BEGIN(CCL); return CCE_ALNUM; + "[:alpha:]" BEGIN(CCL); return CCE_ALPHA; + "[:blank:]" BEGIN(CCL); return CCE_BLANK; + "[:cntrl:]" BEGIN(CCL); return CCE_CNTRL; + "[:digit:]" BEGIN(CCL); return CCE_DIGIT; + "[:graph:]" BEGIN(CCL); return CCE_GRAPH; + "[:lower:]" BEGIN(CCL); return CCE_LOWER; + "[:print:]" BEGIN(CCL); return CCE_PRINT; + "[:punct:]" BEGIN(CCL); return CCE_PUNCT; + "[:space:]" BEGIN(CCL); return CCE_SPACE; + "[:upper:]" BEGIN(CCL); return CCE_UPPER; + "[:xdigit:]" BEGIN(CCL); return CCE_XDIGIT; + {CCL_EXPR} { + format_synerr( + _( "bad character class expression: %s" ), + yytext ); + BEGIN(CCL); return CCE_ALNUM; + } +} + +<NUM>{ + [[:digit:]]+ { + yylval = myctoi( yytext ); + return NUMBER; + } + + "," return ','; + "}" BEGIN(SECT2); return '}'; + + . { + synerr( _( "bad character inside {}'s" ) ); + BEGIN(SECT2); + return '}'; + } + + {NL} { + synerr( _( "missing }" ) ); + BEGIN(SECT2); + ++linenum; + return '}'; + } +} + + +<PERCENT_BRACE_ACTION>{ + {OPTWS}"%}".* bracelevel = 0; + + <ACTION>"/*" ACTION_ECHO; yy_push_state( COMMENT ); + + <CODEBLOCK,ACTION>{ + "reject" { + ACTION_ECHO; + CHECK_REJECT(yytext); + } + "yymore" { + ACTION_ECHO; + CHECK_YYMORE(yytext); + } + } + + {NAME}|{NOT_NAME}|. ACTION_ECHO; + {NL} { + ++linenum; + ACTION_ECHO; + if ( bracelevel == 0 || + (doing_codeblock && indented_code) ) + { + if ( doing_rule_action ) + add_action( "\tYY_BREAK\n" ); + + doing_rule_action = doing_codeblock = false; + BEGIN(SECT2); + } + } +} + + + /* Reject and YYmore() are checked for above, in PERCENT_BRACE_ACTION */ +<ACTION>{ + "{" ACTION_ECHO; ++bracelevel; + "}" ACTION_ECHO; --bracelevel; + [^[:alpha:]_{}"'/\n]+ ACTION_ECHO; + {NAME} ACTION_ECHO; + "'"([^'\\\n]|\\.)*"'" ACTION_ECHO; /* character constant */ + \" ACTION_ECHO; BEGIN(ACTION_STRING); + {NL} { + ++linenum; + ACTION_ECHO; + if ( bracelevel == 0 ) + { + if ( doing_rule_action ) + add_action( "\tYY_BREAK\n" ); + + doing_rule_action = false; + BEGIN(SECT2); + } + } + . ACTION_ECHO; +} + +<ACTION_STRING>{ + [^"\\\n]+ ACTION_ECHO; + \\. ACTION_ECHO; + {NL} ++linenum; ACTION_ECHO; + \" ACTION_ECHO; BEGIN(ACTION); + . ACTION_ECHO; +} + +<COMMENT,ACTION,ACTION_STRING><<EOF>> { + synerr( _( "EOF encountered inside an action" ) ); + yyterminate(); + } + + +<SECT2,QUOTE,FIRSTCCL,CCL>{ESCSEQ} { + yylval = myesc( (Char *) yytext ); + + if ( YY_START == FIRSTCCL ) + BEGIN(CCL); + + return CHAR; + } + + +<SECT3>{ + .*(\n?) ECHO; + <<EOF>> sectnum = 0; yyterminate(); +} + +<*>.|\n format_synerr( _( "bad character: %s" ), yytext ); + +%% + + +int yywrap() + { + if ( --num_input_files > 0 ) + { + set_input_file( *++input_files ); + return 0; + } + + else + return 1; + } + + +/* set_input_file - open the given file (if NULL, stdin) for scanning */ + +void set_input_file( file ) +char *file; + { + if ( file && strcmp( file, "-" ) ) + { + infilename = copy_string( file ); + yyin = fopen( infilename, "r" ); + + if ( yyin == NULL ) + lerrsf( _( "can't open %s" ), file ); + } + + else + { + yyin = stdin; + infilename = copy_string( "<stdin>" ); + } + + linenum = 1; + } + + +/* Wrapper routines for accessing the scanner's malloc routines. */ + +void *flex_alloc( size ) +size_t size; + { + return (void *) malloc( size ); + } + +void *flex_realloc( ptr, size ) +void *ptr; +size_t size; + { + return (void *) realloc( ptr, size ); + } + +void flex_free( ptr ) +void *ptr; + { + if ( ptr ) + free( ptr ); + } diff --git a/usr.bin/lex/sym.c b/usr.bin/lex/sym.c new file mode 100644 index 0000000..2f43b50 --- /dev/null +++ b/usr.bin/lex/sym.c @@ -0,0 +1,264 @@ +/* sym - symbol table routines */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/sym.c,v 2.19 95/03/04 16:11:04 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + + +/* declare functions that have forward references */ + +int hashfunct PROTO((char[], int)); + + +struct hash_entry *ndtbl[NAME_TABLE_HASH_SIZE]; +struct hash_entry *sctbl[START_COND_HASH_SIZE]; +struct hash_entry *ccltab[CCL_HASH_SIZE]; + +struct hash_entry *findsym(); + + +/* addsym - add symbol and definitions to symbol table + * + * -1 is returned if the symbol already exists, and the change not made. + */ + +int addsym( sym, str_def, int_def, table, table_size ) +char sym[]; +char *str_def; +int int_def; +hash_table table; +int table_size; + { + int hash_val = hashfunct( sym, table_size ); + struct hash_entry *sym_entry = table[hash_val]; + struct hash_entry *new_entry; + struct hash_entry *successor; + + while ( sym_entry ) + { + if ( ! strcmp( sym, sym_entry->name ) ) + { /* entry already exists */ + return -1; + } + + sym_entry = sym_entry->next; + } + + /* create new entry */ + new_entry = (struct hash_entry *) + flex_alloc( sizeof( struct hash_entry ) ); + + if ( new_entry == NULL ) + flexfatal( _( "symbol table memory allocation failed" ) ); + + if ( (successor = table[hash_val]) != 0 ) + { + new_entry->next = successor; + successor->prev = new_entry; + } + else + new_entry->next = NULL; + + new_entry->prev = NULL; + new_entry->name = sym; + new_entry->str_val = str_def; + new_entry->int_val = int_def; + + table[hash_val] = new_entry; + + return 0; + } + + +/* cclinstal - save the text of a character class */ + +void cclinstal( ccltxt, cclnum ) +Char ccltxt[]; +int cclnum; + { + /* We don't bother checking the return status because we are not + * called unless the symbol is new. + */ + Char *copy_unsigned_string(); + + (void) addsym( (char *) copy_unsigned_string( ccltxt ), + (char *) 0, cclnum, + ccltab, CCL_HASH_SIZE ); + } + + +/* ccllookup - lookup the number associated with character class text + * + * Returns 0 if there's no CCL associated with the text. + */ + +int ccllookup( ccltxt ) +Char ccltxt[]; + { + return findsym( (char *) ccltxt, ccltab, CCL_HASH_SIZE )->int_val; + } + + +/* findsym - find symbol in symbol table */ + +struct hash_entry *findsym( sym, table, table_size ) +char sym[]; +hash_table table; +int table_size; + { + static struct hash_entry empty_entry = + { + (struct hash_entry *) 0, (struct hash_entry *) 0, + (char *) 0, (char *) 0, 0, + } ; + struct hash_entry *sym_entry = + table[hashfunct( sym, table_size )]; + + while ( sym_entry ) + { + if ( ! strcmp( sym, sym_entry->name ) ) + return sym_entry; + sym_entry = sym_entry->next; + } + + return &empty_entry; + } + + +/* hashfunct - compute the hash value for "str" and hash size "hash_size" */ + +int hashfunct( str, hash_size ) +char str[]; +int hash_size; + { + int hashval; + int locstr; + + hashval = 0; + locstr = 0; + + while ( str[locstr] ) + { + hashval = (hashval << 1) + (unsigned char) str[locstr++]; + hashval %= hash_size; + } + + return hashval; + } + + +/* ndinstal - install a name definition */ + +void ndinstal( name, definition ) +char name[]; +Char definition[]; + { + char *copy_string(); + Char *copy_unsigned_string(); + + if ( addsym( copy_string( name ), + (char *) copy_unsigned_string( definition ), 0, + ndtbl, NAME_TABLE_HASH_SIZE ) ) + synerr( _( "name defined twice" ) ); + } + + +/* ndlookup - lookup a name definition + * + * Returns a nil pointer if the name definition does not exist. + */ + +Char *ndlookup( nd ) +char nd[]; + { + return (Char *) findsym( nd, ndtbl, NAME_TABLE_HASH_SIZE )->str_val; + } + + +/* scextend - increase the maximum number of start conditions */ + +void scextend() + { + current_max_scs += MAX_SCS_INCREMENT; + + ++num_reallocs; + + scset = reallocate_integer_array( scset, current_max_scs ); + scbol = reallocate_integer_array( scbol, current_max_scs ); + scxclu = reallocate_integer_array( scxclu, current_max_scs ); + sceof = reallocate_integer_array( sceof, current_max_scs ); + scname = reallocate_char_ptr_array( scname, current_max_scs ); + } + + +/* scinstal - make a start condition + * + * NOTE + * The start condition is "exclusive" if xcluflg is true. + */ + +void scinstal( str, xcluflg ) +char str[]; +int xcluflg; + { + char *copy_string(); + + /* Generate start condition definition, for use in BEGIN et al. */ + action_define( str, lastsc ); + + if ( ++lastsc >= current_max_scs ) + scextend(); + + scname[lastsc] = copy_string( str ); + + if ( addsym( scname[lastsc], (char *) 0, lastsc, + sctbl, START_COND_HASH_SIZE ) ) + format_pinpoint_message( + _( "start condition %s declared twice" ), + str ); + + scset[lastsc] = mkstate( SYM_EPSILON ); + scbol[lastsc] = mkstate( SYM_EPSILON ); + scxclu[lastsc] = xcluflg; + sceof[lastsc] = false; + } + + +/* sclookup - lookup the number associated with a start condition + * + * Returns 0 if no such start condition. + */ + +int sclookup( str ) +char str[]; + { + return findsym( str, sctbl, START_COND_HASH_SIZE )->int_val; + } diff --git a/usr.bin/lex/tblcmp.c b/usr.bin/lex/tblcmp.c new file mode 100644 index 0000000..d5d623d --- /dev/null +++ b/usr.bin/lex/tblcmp.c @@ -0,0 +1,889 @@ +/* tblcmp - table compression routines */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/tblcmp.c,v 2.11 94/11/05 17:08:28 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "flexdef.h" + + +/* declarations for functions that have forward references */ + +void mkentry PROTO((int*, int, int, int, int)); +void mkprot PROTO((int[], int, int)); +void mktemplate PROTO((int[], int, int)); +void mv2front PROTO((int)); +int tbldiff PROTO((int[], int, int[])); + + +/* bldtbl - build table entries for dfa state + * + * synopsis + * int state[numecs], statenum, totaltrans, comstate, comfreq; + * bldtbl( state, statenum, totaltrans, comstate, comfreq ); + * + * State is the statenum'th dfa state. It is indexed by equivalence class and + * gives the number of the state to enter for a given equivalence class. + * totaltrans is the total number of transitions out of the state. Comstate + * is that state which is the destination of the most transitions out of State. + * Comfreq is how many transitions there are out of State to Comstate. + * + * A note on terminology: + * "protos" are transition tables which have a high probability of + * either being redundant (a state processed later will have an identical + * transition table) or nearly redundant (a state processed later will have + * many of the same out-transitions). A "most recently used" queue of + * protos is kept around with the hope that most states will find a proto + * which is similar enough to be usable, and therefore compacting the + * output tables. + * "templates" are a special type of proto. If a transition table is + * homogeneous or nearly homogeneous (all transitions go to the same + * destination) then the odds are good that future states will also go + * to the same destination state on basically the same character set. + * These homogeneous states are so common when dealing with large rule + * sets that they merit special attention. If the transition table were + * simply made into a proto, then (typically) each subsequent, similar + * state will differ from the proto for two out-transitions. One of these + * out-transitions will be that character on which the proto does not go + * to the common destination, and one will be that character on which the + * state does not go to the common destination. Templates, on the other + * hand, go to the common state on EVERY transition character, and therefore + * cost only one difference. + */ + +void bldtbl( state, statenum, totaltrans, comstate, comfreq ) +int state[], statenum, totaltrans, comstate, comfreq; + { + int extptr, extrct[2][CSIZE + 1]; + int mindiff, minprot, i, d; + + /* If extptr is 0 then the first array of extrct holds the result + * of the "best difference" to date, which is those transitions + * which occur in "state" but not in the proto which, to date, + * has the fewest differences between itself and "state". If + * extptr is 1 then the second array of extrct hold the best + * difference. The two arrays are toggled between so that the + * best difference to date can be kept around and also a difference + * just created by checking against a candidate "best" proto. + */ + + extptr = 0; + + /* If the state has too few out-transitions, don't bother trying to + * compact its tables. + */ + + if ( (totaltrans * 100) < (numecs * PROTO_SIZE_PERCENTAGE) ) + mkentry( state, numecs, statenum, JAMSTATE, totaltrans ); + + else + { + /* "checkcom" is true if we should only check "state" against + * protos which have the same "comstate" value. + */ + int checkcom = + comfreq * 100 > totaltrans * CHECK_COM_PERCENTAGE; + + minprot = firstprot; + mindiff = totaltrans; + + if ( checkcom ) + { + /* Find first proto which has the same "comstate". */ + for ( i = firstprot; i != NIL; i = protnext[i] ) + if ( protcomst[i] == comstate ) + { + minprot = i; + mindiff = tbldiff( state, minprot, + extrct[extptr] ); + break; + } + } + + else + { + /* Since we've decided that the most common destination + * out of "state" does not occur with a high enough + * frequency, we set the "comstate" to zero, assuring + * that if this state is entered into the proto list, + * it will not be considered a template. + */ + comstate = 0; + + if ( firstprot != NIL ) + { + minprot = firstprot; + mindiff = tbldiff( state, minprot, + extrct[extptr] ); + } + } + + /* We now have the first interesting proto in "minprot". If + * it matches within the tolerances set for the first proto, + * we don't want to bother scanning the rest of the proto list + * to see if we have any other reasonable matches. + */ + + if ( mindiff * 100 > totaltrans * FIRST_MATCH_DIFF_PERCENTAGE ) + { + /* Not a good enough match. Scan the rest of the + * protos. + */ + for ( i = minprot; i != NIL; i = protnext[i] ) + { + d = tbldiff( state, i, extrct[1 - extptr] ); + if ( d < mindiff ) + { + extptr = 1 - extptr; + mindiff = d; + minprot = i; + } + } + } + + /* Check if the proto we've decided on as our best bet is close + * enough to the state we want to match to be usable. + */ + + if ( mindiff * 100 > totaltrans * ACCEPTABLE_DIFF_PERCENTAGE ) + { + /* No good. If the state is homogeneous enough, + * we make a template out of it. Otherwise, we + * make a proto. + */ + + if ( comfreq * 100 >= + totaltrans * TEMPLATE_SAME_PERCENTAGE ) + mktemplate( state, statenum, comstate ); + + else + { + mkprot( state, statenum, comstate ); + mkentry( state, numecs, statenum, + JAMSTATE, totaltrans ); + } + } + + else + { /* use the proto */ + mkentry( extrct[extptr], numecs, statenum, + prottbl[minprot], mindiff ); + + /* If this state was sufficiently different from the + * proto we built it from, make it, too, a proto. + */ + + if ( mindiff * 100 >= + totaltrans * NEW_PROTO_DIFF_PERCENTAGE ) + mkprot( state, statenum, comstate ); + + /* Since mkprot added a new proto to the proto queue, + * it's possible that "minprot" is no longer on the + * proto queue (if it happened to have been the last + * entry, it would have been bumped off). If it's + * not there, then the new proto took its physical + * place (though logically the new proto is at the + * beginning of the queue), so in that case the + * following call will do nothing. + */ + + mv2front( minprot ); + } + } + } + + +/* cmptmps - compress template table entries + * + * Template tables are compressed by using the 'template equivalence + * classes', which are collections of transition character equivalence + * classes which always appear together in templates - really meta-equivalence + * classes. + */ + +void cmptmps() + { + int tmpstorage[CSIZE + 1]; + int *tmp = tmpstorage, i, j; + int totaltrans, trans; + + peakpairs = numtemps * numecs + tblend; + + if ( usemecs ) + { + /* Create equivalence classes based on data gathered on + * template transitions. + */ + nummecs = cre8ecs( tecfwd, tecbck, numecs ); + } + + else + nummecs = numecs; + + while ( lastdfa + numtemps + 1 >= current_max_dfas ) + increase_max_dfas(); + + /* Loop through each template. */ + + for ( i = 1; i <= numtemps; ++i ) + { + /* Number of non-jam transitions out of this template. */ + totaltrans = 0; + + for ( j = 1; j <= numecs; ++j ) + { + trans = tnxt[numecs * i + j]; + + if ( usemecs ) + { + /* The absolute value of tecbck is the + * meta-equivalence class of a given + * equivalence class, as set up by cre8ecs(). + */ + if ( tecbck[j] > 0 ) + { + tmp[tecbck[j]] = trans; + + if ( trans > 0 ) + ++totaltrans; + } + } + + else + { + tmp[j] = trans; + + if ( trans > 0 ) + ++totaltrans; + } + } + + /* It is assumed (in a rather subtle way) in the skeleton + * that if we're using meta-equivalence classes, the def[] + * entry for all templates is the jam template, i.e., + * templates never default to other non-jam table entries + * (e.g., another template) + */ + + /* Leave room for the jam-state after the last real state. */ + mkentry( tmp, nummecs, lastdfa + i + 1, JAMSTATE, totaltrans ); + } + } + + + +/* expand_nxt_chk - expand the next check arrays */ + +void expand_nxt_chk() + { + int old_max = current_max_xpairs; + + current_max_xpairs += MAX_XPAIRS_INCREMENT; + + ++num_reallocs; + + nxt = reallocate_integer_array( nxt, current_max_xpairs ); + chk = reallocate_integer_array( chk, current_max_xpairs ); + + zero_out( (char *) (chk + old_max), + (size_t) (MAX_XPAIRS_INCREMENT * sizeof( int )) ); + } + + +/* find_table_space - finds a space in the table for a state to be placed + * + * synopsis + * int *state, numtrans, block_start; + * int find_table_space(); + * + * block_start = find_table_space( state, numtrans ); + * + * State is the state to be added to the full speed transition table. + * Numtrans is the number of out-transitions for the state. + * + * find_table_space() returns the position of the start of the first block (in + * chk) able to accommodate the state + * + * In determining if a state will or will not fit, find_table_space() must take + * into account the fact that an end-of-buffer state will be added at [0], + * and an action number will be added in [-1]. + */ + +int find_table_space( state, numtrans ) +int *state, numtrans; + { + /* Firstfree is the position of the first possible occurrence of two + * consecutive unused records in the chk and nxt arrays. + */ + int i; + int *state_ptr, *chk_ptr; + int *ptr_to_last_entry_in_state; + + /* If there are too many out-transitions, put the state at the end of + * nxt and chk. + */ + if ( numtrans > MAX_XTIONS_FULL_INTERIOR_FIT ) + { + /* If table is empty, return the first available spot in + * chk/nxt, which should be 1. + */ + if ( tblend < 2 ) + return 1; + + /* Start searching for table space near the end of + * chk/nxt arrays. + */ + i = tblend - numecs; + } + + else + /* Start searching for table space from the beginning + * (skipping only the elements which will definitely not + * hold the new state). + */ + i = firstfree; + + while ( 1 ) /* loops until a space is found */ + { + while ( i + numecs >= current_max_xpairs ) + expand_nxt_chk(); + + /* Loops until space for end-of-buffer and action number + * are found. + */ + while ( 1 ) + { + /* Check for action number space. */ + if ( chk[i - 1] == 0 ) + { + /* Check for end-of-buffer space. */ + if ( chk[i] == 0 ) + break; + + else + /* Since i != 0, there is no use + * checking to see if (++i) - 1 == 0, + * because that's the same as i == 0, + * so we skip a space. + */ + i += 2; + } + + else + ++i; + + while ( i + numecs >= current_max_xpairs ) + expand_nxt_chk(); + } + + /* If we started search from the beginning, store the new + * firstfree for the next call of find_table_space(). + */ + if ( numtrans <= MAX_XTIONS_FULL_INTERIOR_FIT ) + firstfree = i + 1; + + /* Check to see if all elements in chk (and therefore nxt) + * that are needed for the new state have not yet been taken. + */ + + state_ptr = &state[1]; + ptr_to_last_entry_in_state = &chk[i + numecs + 1]; + + for ( chk_ptr = &chk[i + 1]; + chk_ptr != ptr_to_last_entry_in_state; ++chk_ptr ) + if ( *(state_ptr++) != 0 && *chk_ptr != 0 ) + break; + + if ( chk_ptr == ptr_to_last_entry_in_state ) + return i; + + else + ++i; + } + } + + +/* inittbl - initialize transition tables + * + * Initializes "firstfree" to be one beyond the end of the table. Initializes + * all "chk" entries to be zero. + */ +void inittbl() + { + int i; + + zero_out( (char *) chk, (size_t) (current_max_xpairs * sizeof( int )) ); + + tblend = 0; + firstfree = tblend + 1; + numtemps = 0; + + if ( usemecs ) + { + /* Set up doubly-linked meta-equivalence classes; these + * are sets of equivalence classes which all have identical + * transitions out of TEMPLATES. + */ + + tecbck[1] = NIL; + + for ( i = 2; i <= numecs; ++i ) + { + tecbck[i] = i - 1; + tecfwd[i - 1] = i; + } + + tecfwd[numecs] = NIL; + } + } + + +/* mkdeftbl - make the default, "jam" table entries */ + +void mkdeftbl() + { + int i; + + jamstate = lastdfa + 1; + + ++tblend; /* room for transition on end-of-buffer character */ + + while ( tblend + numecs >= current_max_xpairs ) + expand_nxt_chk(); + + /* Add in default end-of-buffer transition. */ + nxt[tblend] = end_of_buffer_state; + chk[tblend] = jamstate; + + for ( i = 1; i <= numecs; ++i ) + { + nxt[tblend + i] = 0; + chk[tblend + i] = jamstate; + } + + jambase = tblend; + + base[jamstate] = jambase; + def[jamstate] = 0; + + tblend += numecs; + ++numtemps; + } + + +/* mkentry - create base/def and nxt/chk entries for transition array + * + * synopsis + * int state[numchars + 1], numchars, statenum, deflink, totaltrans; + * mkentry( state, numchars, statenum, deflink, totaltrans ); + * + * "state" is a transition array "numchars" characters in size, "statenum" + * is the offset to be used into the base/def tables, and "deflink" is the + * entry to put in the "def" table entry. If "deflink" is equal to + * "JAMSTATE", then no attempt will be made to fit zero entries of "state" + * (i.e., jam entries) into the table. It is assumed that by linking to + * "JAMSTATE" they will be taken care of. In any case, entries in "state" + * marking transitions to "SAME_TRANS" are treated as though they will be + * taken care of by wherever "deflink" points. "totaltrans" is the total + * number of transitions out of the state. If it is below a certain threshold, + * the tables are searched for an interior spot that will accommodate the + * state array. + */ + +void mkentry( state, numchars, statenum, deflink, totaltrans ) +int *state; +int numchars, statenum, deflink, totaltrans; + { + int minec, maxec, i, baseaddr; + int tblbase, tbllast; + + if ( totaltrans == 0 ) + { /* there are no out-transitions */ + if ( deflink == JAMSTATE ) + base[statenum] = JAMSTATE; + else + base[statenum] = 0; + + def[statenum] = deflink; + return; + } + + for ( minec = 1; minec <= numchars; ++minec ) + { + if ( state[minec] != SAME_TRANS ) + if ( state[minec] != 0 || deflink != JAMSTATE ) + break; + } + + if ( totaltrans == 1 ) + { + /* There's only one out-transition. Save it for later to fill + * in holes in the tables. + */ + stack1( statenum, minec, state[minec], deflink ); + return; + } + + for ( maxec = numchars; maxec > 0; --maxec ) + { + if ( state[maxec] != SAME_TRANS ) + if ( state[maxec] != 0 || deflink != JAMSTATE ) + break; + } + + /* Whether we try to fit the state table in the middle of the table + * entries we have already generated, or if we just take the state + * table at the end of the nxt/chk tables, we must make sure that we + * have a valid base address (i.e., non-negative). Note that + * negative base addresses dangerous at run-time (because indexing + * the nxt array with one and a low-valued character will access + * memory before the start of the array. + */ + + /* Find the first transition of state that we need to worry about. */ + if ( totaltrans * 100 <= numchars * INTERIOR_FIT_PERCENTAGE ) + { + /* Attempt to squeeze it into the middle of the tables. */ + baseaddr = firstfree; + + while ( baseaddr < minec ) + { + /* Using baseaddr would result in a negative base + * address below; find the next free slot. + */ + for ( ++baseaddr; chk[baseaddr] != 0; ++baseaddr ) + ; + } + + while ( baseaddr + maxec - minec + 1 >= current_max_xpairs ) + expand_nxt_chk(); + + for ( i = minec; i <= maxec; ++i ) + if ( state[i] != SAME_TRANS && + (state[i] != 0 || deflink != JAMSTATE) && + chk[baseaddr + i - minec] != 0 ) + { /* baseaddr unsuitable - find another */ + for ( ++baseaddr; + baseaddr < current_max_xpairs && + chk[baseaddr] != 0; ++baseaddr ) + ; + + while ( baseaddr + maxec - minec + 1 >= + current_max_xpairs ) + expand_nxt_chk(); + + /* Reset the loop counter so we'll start all + * over again next time it's incremented. + */ + + i = minec - 1; + } + } + + else + { + /* Ensure that the base address we eventually generate is + * non-negative. + */ + baseaddr = MAX( tblend + 1, minec ); + } + + tblbase = baseaddr - minec; + tbllast = tblbase + maxec; + + while ( tbllast + 1 >= current_max_xpairs ) + expand_nxt_chk(); + + base[statenum] = tblbase; + def[statenum] = deflink; + + for ( i = minec; i <= maxec; ++i ) + if ( state[i] != SAME_TRANS ) + if ( state[i] != 0 || deflink != JAMSTATE ) + { + nxt[tblbase + i] = state[i]; + chk[tblbase + i] = statenum; + } + + if ( baseaddr == firstfree ) + /* Find next free slot in tables. */ + for ( ++firstfree; chk[firstfree] != 0; ++firstfree ) + ; + + tblend = MAX( tblend, tbllast ); + } + + +/* mk1tbl - create table entries for a state (or state fragment) which + * has only one out-transition + */ + +void mk1tbl( state, sym, onenxt, onedef ) +int state, sym, onenxt, onedef; + { + if ( firstfree < sym ) + firstfree = sym; + + while ( chk[firstfree] != 0 ) + if ( ++firstfree >= current_max_xpairs ) + expand_nxt_chk(); + + base[state] = firstfree - sym; + def[state] = onedef; + chk[firstfree] = state; + nxt[firstfree] = onenxt; + + if ( firstfree > tblend ) + { + tblend = firstfree++; + + if ( firstfree >= current_max_xpairs ) + expand_nxt_chk(); + } + } + + +/* mkprot - create new proto entry */ + +void mkprot( state, statenum, comstate ) +int state[], statenum, comstate; + { + int i, slot, tblbase; + + if ( ++numprots >= MSP || numecs * numprots >= PROT_SAVE_SIZE ) + { + /* Gotta make room for the new proto by dropping last entry in + * the queue. + */ + slot = lastprot; + lastprot = protprev[lastprot]; + protnext[lastprot] = NIL; + } + + else + slot = numprots; + + protnext[slot] = firstprot; + + if ( firstprot != NIL ) + protprev[firstprot] = slot; + + firstprot = slot; + prottbl[slot] = statenum; + protcomst[slot] = comstate; + + /* Copy state into save area so it can be compared with rapidly. */ + tblbase = numecs * (slot - 1); + + for ( i = 1; i <= numecs; ++i ) + protsave[tblbase + i] = state[i]; + } + + +/* mktemplate - create a template entry based on a state, and connect the state + * to it + */ + +void mktemplate( state, statenum, comstate ) +int state[], statenum, comstate; + { + int i, numdiff, tmpbase, tmp[CSIZE + 1]; + Char transset[CSIZE + 1]; + int tsptr; + + ++numtemps; + + tsptr = 0; + + /* Calculate where we will temporarily store the transition table + * of the template in the tnxt[] array. The final transition table + * gets created by cmptmps(). + */ + + tmpbase = numtemps * numecs; + + if ( tmpbase + numecs >= current_max_template_xpairs ) + { + current_max_template_xpairs += MAX_TEMPLATE_XPAIRS_INCREMENT; + + ++num_reallocs; + + tnxt = reallocate_integer_array( tnxt, + current_max_template_xpairs ); + } + + for ( i = 1; i <= numecs; ++i ) + if ( state[i] == 0 ) + tnxt[tmpbase + i] = 0; + else + { + transset[tsptr++] = i; + tnxt[tmpbase + i] = comstate; + } + + if ( usemecs ) + mkeccl( transset, tsptr, tecfwd, tecbck, numecs, 0 ); + + mkprot( tnxt + tmpbase, -numtemps, comstate ); + + /* We rely on the fact that mkprot adds things to the beginning + * of the proto queue. + */ + + numdiff = tbldiff( state, firstprot, tmp ); + mkentry( tmp, numecs, statenum, -numtemps, numdiff ); + } + + +/* mv2front - move proto queue element to front of queue */ + +void mv2front( qelm ) +int qelm; + { + if ( firstprot != qelm ) + { + if ( qelm == lastprot ) + lastprot = protprev[lastprot]; + + protnext[protprev[qelm]] = protnext[qelm]; + + if ( protnext[qelm] != NIL ) + protprev[protnext[qelm]] = protprev[qelm]; + + protprev[qelm] = NIL; + protnext[qelm] = firstprot; + protprev[firstprot] = qelm; + firstprot = qelm; + } + } + + +/* place_state - place a state into full speed transition table + * + * State is the statenum'th state. It is indexed by equivalence class and + * gives the number of the state to enter for a given equivalence class. + * Transnum is the number of out-transitions for the state. + */ + +void place_state( state, statenum, transnum ) +int *state, statenum, transnum; + { + int i; + int *state_ptr; + int position = find_table_space( state, transnum ); + + /* "base" is the table of start positions. */ + base[statenum] = position; + + /* Put in action number marker; this non-zero number makes sure that + * find_table_space() knows that this position in chk/nxt is taken + * and should not be used for another accepting number in another + * state. + */ + chk[position - 1] = 1; + + /* Put in end-of-buffer marker; this is for the same purposes as + * above. + */ + chk[position] = 1; + + /* Place the state into chk and nxt. */ + state_ptr = &state[1]; + + for ( i = 1; i <= numecs; ++i, ++state_ptr ) + if ( *state_ptr != 0 ) + { + chk[position + i] = i; + nxt[position + i] = *state_ptr; + } + + if ( position + numecs > tblend ) + tblend = position + numecs; + } + + +/* stack1 - save states with only one out-transition to be processed later + * + * If there's room for another state on the "one-transition" stack, the + * state is pushed onto it, to be processed later by mk1tbl. If there's + * no room, we process the sucker right now. + */ + +void stack1( statenum, sym, nextstate, deflink ) +int statenum, sym, nextstate, deflink; + { + if ( onesp >= ONE_STACK_SIZE - 1 ) + mk1tbl( statenum, sym, nextstate, deflink ); + + else + { + ++onesp; + onestate[onesp] = statenum; + onesym[onesp] = sym; + onenext[onesp] = nextstate; + onedef[onesp] = deflink; + } + } + + +/* tbldiff - compute differences between two state tables + * + * "state" is the state array which is to be extracted from the pr'th + * proto. "pr" is both the number of the proto we are extracting from + * and an index into the save area where we can find the proto's complete + * state table. Each entry in "state" which differs from the corresponding + * entry of "pr" will appear in "ext". + * + * Entries which are the same in both "state" and "pr" will be marked + * as transitions to "SAME_TRANS" in "ext". The total number of differences + * between "state" and "pr" is returned as function value. Note that this + * number is "numecs" minus the number of "SAME_TRANS" entries in "ext". + */ + +int tbldiff( state, pr, ext ) +int state[], pr, ext[]; + { + int i, *sp = state, *ep = ext, *protp; + int numdiff = 0; + + protp = &protsave[numecs * (pr - 1)]; + + for ( i = numecs; i > 0; --i ) + { + if ( *++protp == *++sp ) + *++ep = SAME_TRANS; + else + { + *++ep = *sp; + ++numdiff; + } + } + + return numdiff; + } diff --git a/usr.bin/lex/version.h b/usr.bin/lex/version.h new file mode 100644 index 0000000..5dbdf94 --- /dev/null +++ b/usr.bin/lex/version.h @@ -0,0 +1,3 @@ +/* $FreeBSD$ */ + +#define FLEX_VERSION "2.5.4" diff --git a/usr.bin/lex/yylex.c b/usr.bin/lex/yylex.c new file mode 100644 index 0000000..e90c5d4f --- /dev/null +++ b/usr.bin/lex/yylex.c @@ -0,0 +1,218 @@ +/* yylex - scanner front-end for flex */ + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms are permitted provided + * that: (1) source distributions retain this entire copyright notice and + * comment, and (2) distributions including binaries display the following + * acknowledgement: ``This product includes software developed by the + * University of California, Berkeley and its contributors'' in the + * documentation or other materials provided with the distribution and in + * all advertising materials mentioning features or use of this software. + * Neither the name of the University nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/yylex.c,v 2.13 95/03/04 16:10:41 vern Exp $ */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <ctype.h> +#include "flexdef.h" +#include "parse.h" + + +/* yylex - scan for a regular expression token */ + +int yylex() + { + int toktype; + static int beglin = false; + extern char *yytext; + + if ( eofseen ) + toktype = EOF; + else + toktype = flexscan(); + + if ( toktype == EOF || toktype == 0 ) + { + eofseen = 1; + + if ( sectnum == 1 ) + { + synerr( _( "premature EOF" ) ); + sectnum = 2; + toktype = SECTEND; + } + + else + toktype = 0; + } + + if ( trace ) + { + if ( beglin ) + { + fprintf( stderr, "%d\t", num_rules + 1 ); + beglin = 0; + } + + switch ( toktype ) + { + case '<': + case '>': + case '^': + case '$': + case '"': + case '[': + case ']': + case '{': + case '}': + case '|': + case '(': + case ')': + case '-': + case '/': + case '\\': + case '?': + case '.': + case '*': + case '+': + case ',': + (void) putc( toktype, stderr ); + break; + + case '\n': + (void) putc( '\n', stderr ); + + if ( sectnum == 2 ) + beglin = 1; + + break; + + case SCDECL: + fputs( "%s", stderr ); + break; + + case XSCDECL: + fputs( "%x", stderr ); + break; + + case SECTEND: + fputs( "%%\n", stderr ); + + /* We set beglin to be true so we'll start + * writing out numbers as we echo rules. + * flexscan() has already assigned sectnum. + */ + if ( sectnum == 2 ) + beglin = 1; + + break; + + case NAME: + fprintf( stderr, "'%s'", nmstr ); + break; + + case CHAR: + switch ( yylval ) + { + case '<': + case '>': + case '^': + case '$': + case '"': + case '[': + case ']': + case '{': + case '}': + case '|': + case '(': + case ')': + case '-': + case '/': + case '\\': + case '?': + case '.': + case '*': + case '+': + case ',': + fprintf( stderr, "\\%c", + yylval ); + break; + + default: + if ( ! isascii( yylval ) || + ! isprint( yylval ) ) + fprintf( stderr, + "\\%.3o", + (unsigned int) yylval ); + else + (void) putc( yylval, + stderr ); + break; + } + + break; + + case NUMBER: + fprintf( stderr, "%d", yylval ); + break; + + case PREVCCL: + fprintf( stderr, "[%d]", yylval ); + break; + + case EOF_OP: + fprintf( stderr, "<<EOF>>" ); + break; + + case OPTION_OP: + fprintf( stderr, "%s ", yytext ); + break; + + case OPT_OUTFILE: + case OPT_PREFIX: + case CCE_ALNUM: + case CCE_ALPHA: + case CCE_BLANK: + case CCE_CNTRL: + case CCE_DIGIT: + case CCE_GRAPH: + case CCE_LOWER: + case CCE_PRINT: + case CCE_PUNCT: + case CCE_SPACE: + case CCE_UPPER: + case CCE_XDIGIT: + fprintf( stderr, "%s", yytext ); + break; + + case 0: + fprintf( stderr, _( "End Marker\n" ) ); + break; + + default: + fprintf( stderr, + _( "*Something Weird* - tok: %d val: %d\n" ), + toktype, yylval ); + break; + } + } + + return toktype; + } |
