1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
|
/* dfa.h - declarations for GNU deterministic regexp compiler
Copyright (C) 1988, 1998 Free Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA */
/* Written June, 1988 by Mike Haertel */
/* FIXME:
2. We should not export so much of the DFA internals.
In addition to clobbering modularity, we eat up valuable
name space. */
# undef PARAMS
#if __STDC__
# ifndef _PTR_T
# define _PTR_T
typedef void * ptr_t;
# endif
# define PARAMS(x) x
#else
# ifndef _PTR_T
# define _PTR_T
typedef char * ptr_t;
# endif
# define PARAMS(x) ()
#endif
/* Number of bits in an unsigned char. */
#ifndef CHARBITS
#define CHARBITS 8
#endif
/* First integer value that is greater than any character code. */
#define NOTCHAR (1 << CHARBITS)
/* INTBITS need not be exact, just a lower bound. */
#ifndef INTBITS
#define INTBITS (CHARBITS * sizeof (int))
#endif
/* Number of ints required to hold a bit for every character. */
#define CHARCLASS_INTS ((NOTCHAR + INTBITS - 1) / INTBITS)
/* Sets of unsigned characters are stored as bit vectors in arrays of ints. */
typedef int charclass[CHARCLASS_INTS];
/* The regexp is parsed into an array of tokens in postfix form. Some tokens
are operators and others are terminal symbols. Most (but not all) of these
codes are returned by the lexical analyzer. */
typedef enum
{
END = -1, /* END is a terminal symbol that matches the
end of input; any value of END or less in
the parse tree is such a symbol. Accepting
states of the DFA are those that would have
a transition on END. */
/* Ordinary character values are terminal symbols that match themselves. */
EMPTY = NOTCHAR, /* EMPTY is a terminal symbol that matches
the empty string. */
BACKREF, /* BACKREF is generated by \<digit>; it
it not completely handled. If the scanner
detects a transition on backref, it returns
a kind of "semi-success" indicating that
the match will have to be verified with
a backtracking matcher. */
BEGLINE, /* BEGLINE is a terminal symbol that matches
the empty string if it is at the beginning
of a line. */
ENDLINE, /* ENDLINE is a terminal symbol that matches
the empty string if it is at the end of
a line. */
BEGWORD, /* BEGWORD is a terminal symbol that matches
the empty string if it is at the beginning
of a word. */
ENDWORD, /* ENDWORD is a terminal symbol that matches
the empty string if it is at the end of
a word. */
LIMWORD, /* LIMWORD is a terminal symbol that matches
the empty string if it is at the beginning
or the end of a word. */
NOTLIMWORD, /* NOTLIMWORD is a terminal symbol that
matches the empty string if it is not at
the beginning or end of a word. */
QMARK, /* QMARK is an operator of one argument that
matches zero or one occurences of its
argument. */
STAR, /* STAR is an operator of one argument that
matches the Kleene closure (zero or more
occurrences) of its argument. */
PLUS, /* PLUS is an operator of one argument that
matches the positive closure (one or more
occurrences) of its argument. */
REPMN, /* REPMN is a lexical token corresponding
to the {m,n} construct. REPMN never
appears in the compiled token vector. */
CAT, /* CAT is an operator of two arguments that
matches the concatenation of its
arguments. CAT is never returned by the
lexical analyzer. */
OR, /* OR is an operator of two arguments that
matches either of its arguments. */
ORTOP, /* OR at the toplevel in the parse tree.
This is used for a boyer-moore heuristic. */
LPAREN, /* LPAREN never appears in the parse tree,
it is only a lexeme. */
RPAREN, /* RPAREN never appears in the parse tree. */
CSET /* CSET and (and any value greater) is a
terminal symbol that matches any of a
class of characters. */
} token;
/* Sets are stored in an array in the compiled dfa; the index of the
array corresponding to a given set token is given by SET_INDEX(t). */
#define SET_INDEX(t) ((t) - CSET)
/* Sometimes characters can only be matched depending on the surrounding
context. Such context decisions depend on what the previous character
was, and the value of the current (lookahead) character. Context
dependent constraints are encoded as 8 bit integers. Each bit that
is set indicates that the constraint succeeds in the corresponding
context.
bit 7 - previous and current are newlines
bit 6 - previous was newline, current isn't
bit 5 - previous wasn't newline, current is
bit 4 - neither previous nor current is a newline
bit 3 - previous and current are word-constituents
bit 2 - previous was word-constituent, current isn't
bit 1 - previous wasn't word-constituent, current is
bit 0 - neither previous nor current is word-constituent
Word-constituent characters are those that satisfy isalnum().
The macro SUCCEEDS_IN_CONTEXT determines whether a a given constraint
succeeds in a particular context. Prevn is true if the previous character
was a newline, currn is true if the lookahead character is a newline.
Prevl and currl similarly depend upon whether the previous and current
characters are word-constituent letters. */
#define MATCHES_NEWLINE_CONTEXT(constraint, prevn, currn) \
((constraint) & 1 << (((prevn) ? 2 : 0) + ((currn) ? 1 : 0) + 4))
#define MATCHES_LETTER_CONTEXT(constraint, prevl, currl) \
((constraint) & 1 << (((prevl) ? 2 : 0) + ((currl) ? 1 : 0)))
#define SUCCEEDS_IN_CONTEXT(constraint, prevn, currn, prevl, currl) \
(MATCHES_NEWLINE_CONTEXT(constraint, prevn, currn) \
&& MATCHES_LETTER_CONTEXT(constraint, prevl, currl))
/* The following macros give information about what a constraint depends on. */
#define PREV_NEWLINE_DEPENDENT(constraint) \
(((constraint) & 0xc0) >> 2 != ((constraint) & 0x30))
#define PREV_LETTER_DEPENDENT(constraint) \
(((constraint) & 0x0c) >> 2 != ((constraint) & 0x03))
/* Tokens that match the empty string subject to some constraint actually
work by applying that constraint to determine what may follow them,
taking into account what has gone before. The following values are
the constraints corresponding to the special tokens previously defined. */
#define NO_CONSTRAINT 0xff
#define BEGLINE_CONSTRAINT 0xcf
#define ENDLINE_CONSTRAINT 0xaf
#define BEGWORD_CONSTRAINT 0xf2
#define ENDWORD_CONSTRAINT 0xf4
#define LIMWORD_CONSTRAINT 0xf6
#define NOTLIMWORD_CONSTRAINT 0xf9
/* States of the recognizer correspond to sets of positions in the parse
tree, together with the constraints under which they may be matched.
So a position is encoded as an index into the parse tree together with
a constraint. */
typedef struct
{
unsigned index; /* Index into the parse array. */
unsigned constraint; /* Constraint for matching this position. */
} position;
/* Sets of positions are stored as arrays. */
typedef struct
{
position *elems; /* Elements of this position set. */
int nelem; /* Number of elements in this set. */
} position_set;
/* A state of the dfa consists of a set of positions, some flags,
and the token value of the lowest-numbered position of the state that
contains an END token. */
typedef struct
{
int hash; /* Hash of the positions of this state. */
position_set elems; /* Positions this state could match. */
char newline; /* True if previous state matched newline. */
char letter; /* True if previous state matched a letter. */
char backref; /* True if this state matches a \<digit>. */
unsigned char constraint; /* Constraint for this state to accept. */
int first_end; /* Token value of the first END in elems. */
} dfa_state;
/* Element of a list of strings, at least one of which is known to
appear in any R.E. matching the DFA. */
struct dfamust
{
int exact;
char *must;
struct dfamust *next;
};
/* A compiled regular expression. */
struct dfa
{
/* Stuff built by the scanner. */
charclass *charclasses; /* Array of character sets for CSET tokens. */
int cindex; /* Index for adding new charclasses. */
int calloc; /* Number of charclasses currently allocated. */
/* Stuff built by the parser. */
token *tokens; /* Postfix parse array. */
int tindex; /* Index for adding new tokens. */
int talloc; /* Number of tokens currently allocated. */
int depth; /* Depth required of an evaluation stack
used for depth-first traversal of the
parse tree. */
int nleaves; /* Number of leaves on the parse tree. */
int nregexps; /* Count of parallel regexps being built
with dfaparse(). */
/* Stuff owned by the state builder. */
dfa_state *states; /* States of the dfa. */
int sindex; /* Index for adding new states. */
int salloc; /* Number of states currently allocated. */
/* Stuff built by the structure analyzer. */
position_set *follows; /* Array of follow sets, indexed by position
index. The follow of a position is the set
of positions containing characters that
could conceivably follow a character
matching the given position in a string
matching the regexp. Allocated to the
maximum possible position index. */
int searchflag; /* True if we are supposed to build a searching
as opposed to an exact matcher. A searching
matcher finds the first and shortest string
matching a regexp anywhere in the buffer,
whereas an exact matcher finds the longest
string matching, but anchored to the
beginning of the buffer. */
/* Stuff owned by the executor. */
int tralloc; /* Number of transition tables that have
slots so far. */
int trcount; /* Number of transition tables that have
actually been built. */
int **trans; /* Transition tables for states that can
never accept. If the transitions for a
state have not yet been computed, or the
state could possibly accept, its entry in
this table is NULL. */
int **realtrans; /* Trans always points to realtrans + 1; this
is so trans[-1] can contain NULL. */
int **fails; /* Transition tables after failing to accept
on a state that potentially could do so. */
int *success; /* Table of acceptance conditions used in
dfaexec and computed in build_state. */
int *newlines; /* Transitions on newlines. The entry for a
newline in any transition table is always
-1 so we can count lines without wasting
too many cycles. The transition for a
newline is stored separately and handled
as a special case. Newline is also used
as a sentinel at the end of the buffer. */
struct dfamust *musts; /* List of strings, at least one of which
is known to appear in any r.e. matching
the dfa. */
};
/* Some macros for user access to dfa internals. */
/* ACCEPTING returns true if s could possibly be an accepting state of r. */
#define ACCEPTING(s, r) ((r).states[s].constraint)
/* ACCEPTS_IN_CONTEXT returns true if the given state accepts in the
specified context. */
#define ACCEPTS_IN_CONTEXT(prevn, currn, prevl, currl, state, dfa) \
SUCCEEDS_IN_CONTEXT((dfa).states[state].constraint, \
prevn, currn, prevl, currl)
/* FIRST_MATCHING_REGEXP returns the index number of the first of parallel
regexps that a given state could accept. Parallel regexps are numbered
starting at 1. */
#define FIRST_MATCHING_REGEXP(state, dfa) (-(dfa).states[state].first_end)
/* Entry points. */
/* dfasyntax() takes two arguments; the first sets the syntax bits described
earlier in this file, and the second sets the case-folding flag. */
extern void dfasyntax PARAMS ((reg_syntax_t, int));
/* Compile the given string of the given length into the given struct dfa.
Final argument is a flag specifying whether to build a searching or an
exact matcher. */
extern void dfacomp PARAMS ((char *, size_t, struct dfa *, int));
/* Execute the given struct dfa on the buffer of characters. The
first char * points to the beginning, and the second points to the
first character after the end of the buffer, which must be a writable
place so a sentinel end-of-buffer marker can be stored there. The
second-to-last argument is a flag telling whether to allow newlines to
be part of a string matching the regexp. The next-to-last argument,
if non-NULL, points to a place to increment every time we see a
newline. The final argument, if non-NULL, points to a flag that will
be set if further examination by a backtracking matcher is needed in
order to verify backreferencing; otherwise the flag will be cleared.
Returns NULL if no match is found, or a pointer to the first
character after the first & shortest matching string in the buffer. */
extern char *dfaexec PARAMS ((struct dfa *, char *, char *, int, int *, int *));
/* Free the storage held by the components of a struct dfa. */
extern void dfafree PARAMS ((struct dfa *));
/* Entry points for people who know what they're doing. */
/* Initialize the components of a struct dfa. */
extern void dfainit PARAMS ((struct dfa *));
/* Incrementally parse a string of given length into a struct dfa. */
extern void dfaparse PARAMS ((char *, size_t, struct dfa *));
/* Analyze a parsed regexp; second argument tells whether to build a searching
or an exact matcher. */
extern void dfaanalyze PARAMS ((struct dfa *, int));
/* Compute, for each possible character, the transitions out of a given
state, storing them in an array of integers. */
extern void dfastate PARAMS ((int, struct dfa *, int []));
/* Error handling. */
/* dfaerror() is called by the regexp routines whenever an error occurs. It
takes a single argument, a NUL-terminated string describing the error.
The default dfaerror() prints the error message to stderr and exits.
The user can provide a different dfafree() if so desired. */
extern void dfaerror PARAMS ((const char *));
|