Line data Source code
1 : %top{
2 : /*-------------------------------------------------------------------------
3 : *
4 : * scan.l
5 : * lexical scanner for PostgreSQL
6 : *
7 : * NOTE NOTE NOTE:
8 : *
9 : * The rules in this file must be kept in sync with src/fe_utils/psqlscan.l
10 : * and src/interfaces/ecpg/preproc/pgc.l!
11 : *
12 : * The rules are designed so that the scanner never has to backtrack,
13 : * in the sense that there is always a rule that can match the input
14 : * consumed so far (the rule action may internally throw back some input
15 : * with yyless(), however). As explained in the flex manual, this makes
16 : * for a useful speed increase --- several percent faster when measuring
17 : * raw parsing (Flex + Bison). The extra complexity is mostly in the rules
18 : * for handling float numbers and continued string literals. If you change
19 : * the lexical rules, verify that you haven't broken the no-backtrack
20 : * property by running flex with the "-b" option and checking that the
21 : * resulting "lex.backup" file says that no backing up is needed. (As of
22 : * Postgres 9.2, this check is made automatically by the Makefile.)
23 : *
24 : *
25 : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
26 : * Portions Copyright (c) 1994, Regents of the University of California
27 : *
28 : * IDENTIFICATION
29 : * src/backend/parser/scan.l
30 : *
31 : *-------------------------------------------------------------------------
32 : */
33 : #include "postgres.h"
34 :
35 : #include <ctype.h>
36 : #include <unistd.h>
37 :
38 : #include "common/string.h"
39 : #include "gramparse.h"
40 : #include "nodes/miscnodes.h"
41 : #include "parser/parser.h" /* only needed for GUC variables */
42 : #include "parser/scansup.h"
43 : #include "port/pg_bitutils.h"
44 : #include "mb/pg_wchar.h"
45 : #include "utils/builtins.h"
46 : }
47 :
48 : %{
49 :
50 : /* LCOV_EXCL_START */
51 :
52 : /* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
53 : #undef fprintf
54 : #define fprintf(file, fmt, msg) fprintf_to_ereport(fmt, msg)
55 :
56 : static void
57 0 : fprintf_to_ereport(const char *fmt, const char *msg)
58 : {
59 0 : ereport(ERROR, (errmsg_internal("%s", msg)));
60 : }
61 :
62 : /*
63 : * GUC variable. This is a DIRECT violation of the warning given at the
64 : * head of gram.y, ie flex/bison code must not depend on any GUC variables;
65 : * as such, changing its value can induce very unintuitive behavior.
66 : * In practice, backslash_quote is not too awful since it only controls
67 : * whether to throw an error: it cannot change non-error results.
68 : */
69 : int backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING;
70 :
71 : /*
72 : * Constant data exported from this file. This array maps from the
73 : * zero-based keyword numbers returned by ScanKeywordLookup to the
74 : * Bison token numbers needed by gram.y. This is exported because
75 : * callers need to pass it to scanner_init, if they are using the
76 : * standard keyword list ScanKeywords.
77 : */
78 : #define PG_KEYWORD(kwname, value, category, collabel) value,
79 :
80 : const uint16 ScanKeywordTokens[] = {
81 : #include "parser/kwlist.h"
82 : };
83 :
84 : #undef PG_KEYWORD
85 :
86 : /*
87 : * Set the type of YYSTYPE.
88 : */
89 : #define YYSTYPE core_YYSTYPE
90 :
91 : /*
92 : * Each call to yylex must set yylloc to the location of the found token
93 : * (expressed as a byte offset from the start of the input text).
94 : * When we parse a token that requires multiple lexer rules to process,
95 : * this should be done in the first such rule, else yylloc will point
96 : * into the middle of the token.
97 : */
98 : #define SET_YYLLOC() (*(yylloc) = yytext - yyextra->scanbuf)
99 :
100 : /*
101 : * Advance yylloc by the given number of bytes.
102 : */
103 : #define ADVANCE_YYLLOC(delta) ( *(yylloc) += (delta) )
104 :
105 : /*
106 : * Sometimes, we do want yylloc to point into the middle of a token; this is
107 : * useful for instance to throw an error about an escape sequence within a
108 : * string literal. But if we find no error there, we want to revert yylloc
109 : * to the token start, so that that's the location reported to the parser.
110 : * Use PUSH_YYLLOC/POP_YYLLOC to save/restore yylloc around such code.
111 : * (Currently the implied "stack" is just one location, but someday we might
112 : * need to nest these.)
113 : */
114 : #define PUSH_YYLLOC() (yyextra->save_yylloc = *(yylloc))
115 : #define POP_YYLLOC() (*(yylloc) = yyextra->save_yylloc)
116 :
117 : #define startlit() ( yyextra->literallen = 0 )
118 : static void addlit(char *ytext, int yleng, core_yyscan_t yyscanner);
119 : static void addlitchar(unsigned char ychar, core_yyscan_t yyscanner);
120 : static char *litbufdup(core_yyscan_t yyscanner);
121 : static unsigned char unescape_single_char(unsigned char c, core_yyscan_t yyscanner);
122 : static int process_integer_literal(const char *token, YYSTYPE *lval, int base);
123 : static void addunicode(char32_t c, yyscan_t yyscanner);
124 :
125 : #define yyerror(msg) scanner_yyerror(msg, yyscanner)
126 :
127 : #define lexer_errposition() scanner_errposition(*(yylloc), yyscanner)
128 :
129 : %}
130 :
131 : %option reentrant
132 : %option bison-bridge
133 : %option bison-locations
134 : %option 8bit
135 : %option never-interactive
136 : %option nodefault
137 : %option noinput
138 : %option nounput
139 : %option noyywrap
140 : %option noyyalloc
141 : %option noyyrealloc
142 : %option noyyfree
143 : %option warn
144 : %option prefix="core_yy"
145 : %option extra-type="core_yy_extra_type *"
146 :
147 : /*
148 : * OK, here is a short description of lex/flex rules behavior.
149 : * The longest pattern which matches an input string is always chosen.
150 : * For equal-length patterns, the first occurring in the rules list is chosen.
151 : * INITIAL is the starting state, to which all non-conditional rules apply.
152 : * Exclusive states change parsing rules while the state is active. When in
153 : * an exclusive state, only those rules defined for that state apply.
154 : *
155 : * We use exclusive states for quoted strings, extended comments,
156 : * and to eliminate parsing troubles for numeric strings.
157 : * Exclusive states:
158 : * <xb> bit string literal
159 : * <xc> extended C-style comments
160 : * <xd> delimited identifiers (double-quoted identifiers)
161 : * <xh> hexadecimal byte string
162 : * <xq> standard quoted strings
163 : * <xqs> quote stop (detect continued strings)
164 : * <xe> extended quoted strings (support backslash escape sequences)
165 : * <xdolq> $foo$ quoted strings
166 : * <xui> quoted identifier with Unicode escapes
167 : * <xus> quoted string with Unicode escapes
168 : * <xeu> Unicode surrogate pair in extended quoted string
169 : *
170 : * Remember to add an <<EOF>> case whenever you add a new exclusive state!
171 : * The default one is probably not the right thing.
172 : */
173 :
174 : %x xb
175 : %x xc
176 : %x xd
177 : %x xh
178 : %x xq
179 : %x xqs
180 : %x xe
181 : %x xdolq
182 : %x xui
183 : %x xus
184 : %x xeu
185 :
186 : /*
187 : * In order to make the world safe for Windows and Mac clients as well as
188 : * Unix ones, we accept either \n or \r as a newline. A DOS-style \r\n
189 : * sequence will be seen as two successive newlines, but that doesn't cause
190 : * any problems. Comments that start with -- and extend to the next
191 : * newline are treated as equivalent to a single whitespace character.
192 : *
193 : * NOTE a fine point: if there is no newline following --, we will absorb
194 : * everything to the end of the input as a comment. This is correct. Older
195 : * versions of Postgres failed to recognize -- as a comment if the input
196 : * did not end with a newline.
197 : *
198 : * non_newline_space tracks all the other space characters except newlines.
199 : *
200 : * XXX if you change the set of whitespace characters, fix scanner_isspace()
201 : * to agree.
202 : */
203 :
204 : space [ \t\n\r\f\v]
205 : non_newline_space [ \t\f\v]
206 : newline [\n\r]
207 : non_newline [^\n\r]
208 :
209 : comment ("--"{non_newline}*)
210 :
211 : whitespace ({space}+|{comment})
212 :
213 : /*
214 : * SQL requires at least one newline in the whitespace separating
215 : * string literals that are to be concatenated. Silly, but who are we
216 : * to argue? Note that {whitespace_with_newline} should not have * after
217 : * it, whereas {whitespace} should generally have a * after it...
218 : */
219 :
220 : special_whitespace ({space}+|{comment}{newline})
221 : non_newline_whitespace ({non_newline_space}|{comment})
222 : whitespace_with_newline ({non_newline_whitespace}*{newline}{special_whitespace}*)
223 :
224 : quote '
225 : /* If we see {quote} then {quotecontinue}, the quoted string continues */
226 : quotecontinue {whitespace_with_newline}{quote}
227 :
228 : /*
229 : * {quotecontinuefail} is needed to avoid lexer backup when we fail to match
230 : * {quotecontinue}. It might seem that this could just be {whitespace}*,
231 : * but if there's a dash after {whitespace_with_newline}, it must be consumed
232 : * to see if there's another dash --- which would start a {comment} and thus
233 : * allow continuation of the {quotecontinue} token.
234 : */
235 : quotecontinuefail {whitespace}*"-"?
236 :
237 : /* Bit string
238 : * It is tempting to scan the string for only those characters
239 : * which are allowed. However, this leads to silently swallowed
240 : * characters if illegal characters are included in the string.
241 : * For example, if xbinside is [01] then B'ABCD' is interpreted
242 : * as a zero-length string, and the ABCD' is lost!
243 : * Better to pass the string forward and let the input routines
244 : * validate the contents.
245 : */
246 : xbstart [bB]{quote}
247 : xbinside [^']*
248 :
249 : /* Hexadecimal byte string */
250 : xhstart [xX]{quote}
251 : xhinside [^']*
252 :
253 : /* National character */
254 : xnstart [nN]{quote}
255 :
256 : /* Quoted string that allows backslash escapes */
257 : xestart [eE]{quote}
258 : xeinside [^\\']+
259 : xeescape [\\][^0-7]
260 : xeoctesc [\\][0-7]{1,3}
261 : xehexesc [\\]x[0-9A-Fa-f]{1,2}
262 : xeunicode [\\](u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})
263 : xeunicodefail [\\](u[0-9A-Fa-f]{0,3}|U[0-9A-Fa-f]{0,7})
264 :
265 : /* Extended quote
266 : * xqdouble implements embedded quote, ''''
267 : */
268 : xqstart {quote}
269 : xqdouble {quote}{quote}
270 : xqinside [^']+
271 :
272 : /* $foo$ style quotes ("dollar quoting")
273 : * The quoted string starts with $foo$ where "foo" is an optional string
274 : * in the form of an identifier, except that it may not contain "$",
275 : * and extends to the first occurrence of an identical string.
276 : * There is *no* processing of the quoted text.
277 : *
278 : * {dolqfailed} is an error rule to avoid scanner backup when {dolqdelim}
279 : * fails to match its trailing "$".
280 : */
281 : dolq_start [A-Za-z\200-\377_]
282 : dolq_cont [A-Za-z\200-\377_0-9]
283 : dolqdelim \$({dolq_start}{dolq_cont}*)?\$
284 : dolqfailed \${dolq_start}{dolq_cont}*
285 : dolqinside [^$]+
286 :
287 : /* Double quote
288 : * Allows embedded spaces and other special characters into identifiers.
289 : */
290 : dquote \"
291 : xdstart {dquote}
292 : xdstop {dquote}
293 : xddouble {dquote}{dquote}
294 : xdinside [^"]+
295 :
296 : /* Quoted identifier with Unicode escapes */
297 : xuistart [uU]&{dquote}
298 :
299 : /* Quoted string with Unicode escapes */
300 : xusstart [uU]&{quote}
301 :
302 : /* error rule to avoid backup */
303 : xufailed [uU]&
304 :
305 :
306 : /* C-style comments
307 : *
308 : * The "extended comment" syntax closely resembles allowable operator syntax.
309 : * The tricky part here is to get lex to recognize a string starting with
310 : * slash-star as a comment, when interpreting it as an operator would produce
311 : * a longer match --- remember lex will prefer a longer match! Also, if we
312 : * have something like plus-slash-star, lex will think this is a 3-character
313 : * operator whereas we want to see it as a + operator and a comment start.
314 : * The solution is two-fold:
315 : * 1. append {op_chars}* to xcstart so that it matches as much text as
316 : * {operator} would. Then the tie-breaker (first matching rule of same
317 : * length) ensures xcstart wins. We put back the extra stuff with yyless()
318 : * in case it contains a star-slash that should terminate the comment.
319 : * 2. In the operator rule, check for slash-star within the operator, and
320 : * if found throw it back with yyless(). This handles the plus-slash-star
321 : * problem.
322 : * Dash-dash comments have similar interactions with the operator rule.
323 : */
324 : xcstart \/\*{op_chars}*
325 : xcstop \*+\/
326 : xcinside [^*/]+
327 :
328 : ident_start [A-Za-z\200-\377_]
329 : ident_cont [A-Za-z\200-\377_0-9\$]
330 :
331 : identifier {ident_start}{ident_cont}*
332 :
333 : /* Assorted special-case operators and operator-like tokens */
334 : typecast "::"
335 : dot_dot \.\.
336 : colon_equals ":="
337 :
338 : /*
339 : * These operator-like tokens (unlike the above ones) also match the {operator}
340 : * rule, which means that they might be overridden by a longer match if they
341 : * are followed by a comment start or a + or - character. Accordingly, if you
342 : * add to this list, you must also add corresponding code to the {operator}
343 : * block to return the correct token in such cases. (This is not needed in
344 : * psqlscan.l since the token value is ignored there.)
345 : */
346 : equals_greater "=>"
347 : less_equals "<="
348 : greater_equals ">="
349 : less_greater "<>"
350 : not_equals "!="
351 :
352 : /*
353 : * "self" is the set of chars that should be returned as single-character
354 : * tokens. "op_chars" is the set of chars that can make up "Op" tokens,
355 : * which can be one or more characters long (but if a single-char token
356 : * appears in the "self" set, it is not to be returned as an Op). Note
357 : * that the sets overlap, but each has some chars that are not in the other.
358 : *
359 : * If you change either set, adjust the character lists appearing in the
360 : * rule for "operator"!
361 : */
362 : self [,()\[\].;\:\+\-\*\/\%\^\<\>\=]
363 : op_chars [\~\!\@\#\^\&\|\`\?\+\-\*\/\%\<\>\=]
364 : operator {op_chars}+
365 :
366 : /*
367 : * Numbers
368 : *
369 : * Unary minus is not part of a number here. Instead we pass it separately to
370 : * the parser, and there it gets coerced via doNegate().
371 : *
372 : * {numericfail} is used because we would like "1..10" to lex as 1, dot_dot, 10.
373 : *
374 : * {realfail} is added to prevent the need for scanner
375 : * backup when the {real} rule fails to match completely.
376 : */
377 : decdigit [0-9]
378 : hexdigit [0-9A-Fa-f]
379 : octdigit [0-7]
380 : bindigit [0-1]
381 :
382 : decinteger {decdigit}(_?{decdigit})*
383 : hexinteger 0[xX](_?{hexdigit})+
384 : octinteger 0[oO](_?{octdigit})+
385 : bininteger 0[bB](_?{bindigit})+
386 :
387 : hexfail 0[xX]_?
388 : octfail 0[oO]_?
389 : binfail 0[bB]_?
390 :
391 : numeric (({decinteger}\.{decinteger}?)|(\.{decinteger}))
392 : numericfail {decinteger}\.\.
393 :
394 : real ({decinteger}|{numeric})[Ee][-+]?{decinteger}
395 : realfail ({decinteger}|{numeric})[Ee][-+]
396 :
397 : /* Positional parameters don't accept underscores. */
398 : param \${decdigit}+
399 :
400 : /*
401 : * An identifier immediately following an integer literal is disallowed because
402 : * in some cases it's ambiguous what is meant: for example, 0x1234 could be
403 : * either a hexinteger or a decinteger "0" and an identifier "x1234". We can
404 : * detect such problems by seeing if integer_junk matches a longer substring
405 : * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
406 : * bininteger). One "junk" pattern is sufficient because
407 : * {decinteger}{identifier} will match all the same strings we'd match with
408 : * {hexinteger}{identifier} etc.
409 : *
410 : * Note that the rule for integer_junk must appear after the ones for
411 : * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
412 : * and integer_junk, and we need hexinteger to be chosen in that case.
413 : *
414 : * Also disallow strings matched by numeric_junk, real_junk and param_junk
415 : * for consistency.
416 : */
417 : integer_junk {decinteger}{identifier}
418 : numeric_junk {numeric}{identifier}
419 : real_junk {real}{identifier}
420 : param_junk \${decdigit}+{identifier}
421 :
422 : other .
423 :
424 : /*
425 : * Dollar quoted strings are totally opaque, and no escaping is done on them.
426 : * Other quoted strings must allow some special characters such as single-quote
427 : * and newline.
428 : * Embedded single-quotes are implemented both in the SQL standard
429 : * style of two adjacent single quotes "''" and in the Postgres/Java style
430 : * of escaped-quote "\'".
431 : * Other embedded escaped characters are matched explicitly and the leading
432 : * backslash is dropped from the string.
433 : * Note that xcstart must appear before operator, as explained above!
434 : * Also whitespace (comment) must appear before operator.
435 : */
436 :
437 : %%
438 :
439 : {whitespace} {
440 : /* ignore */
441 : }
442 5464570 :
443 9172 : {xcstart} {
444 : /* Set location in case of syntax error in comment */
445 9172 : SET_YYLLOC();
446 9172 : yyextra->xcdepth = 0;
447 9172 : BEGIN(xc);
448 : /* Put back any characters past slash-star; see above */
449 9172 : yyless(2);
450 : }
451 9172 :
452 : <xc>{
453 9 : {xcstart} {
454 9 : (yyextra->xcdepth)++;
455 : /* Put back any characters past slash-star; see above */
456 9 : yyless(2);
457 : }
458 9 :
459 9181 : {xcstop} {
460 9181 : if (yyextra->xcdepth <= 0)
461 9172 : BEGIN(INITIAL);
462 : else
463 9 : (yyextra->xcdepth)--;
464 : }
465 9181 :
466 53378 : {xcinside} {
467 : /* ignore */
468 : }
469 53378 :
470 44223 : {op_chars} {
471 : /* ignore */
472 : }
473 44223 :
474 0 : \*+ {
475 : /* ignore */
476 : }
477 0 :
478 0 : <<EOF>> {
479 0 : yyerror("unterminated /* comment");
480 : }
481 : } /* <xc> */
482 :
483 409 : {xbstart} {
484 : /* Binary bit type.
485 : * At some point we should simply pass the string
486 : * forward to the parser and label it there.
487 : * In the meantime, place a leading "b" on the string
488 : * to mark it for the input routine as a binary string.
489 : */
490 409 : SET_YYLLOC();
491 409 : BEGIN(xb);
492 409 : startlit();
493 409 : addlitchar('b', yyscanner);
494 : }
495 409 : <xh>{xhinside} |
496 2585 : <xb>{xbinside} {
497 2585 : addlit(yytext, yyleng, yyscanner);
498 : }
499 2585 : <xb><<EOF>> { yyerror("unterminated bit string literal"); }
500 0 :
501 2192 : {xhstart} {
502 : /* Hexadecimal bit type.
503 : * At some point we should simply pass the string
504 : * forward to the parser and label it there.
505 : * In the meantime, place a leading "x" on the string
506 : * to mark it for the input routine as a hex string.
507 : */
508 2192 : SET_YYLLOC();
509 2192 : BEGIN(xh);
510 2192 : startlit();
511 2192 : addlitchar('x', yyscanner);
512 : }
513 2192 : <xh><<EOF>> { yyerror("unterminated hexadecimal string literal"); }
514 0 :
515 2 : {xnstart} {
516 : /* National character.
517 : * We will pass this along as a normal character string,
518 : * but preceded with an internally-generated "NCHAR".
519 : */
520 : int kwnum;
521 :
522 2 : SET_YYLLOC();
523 2 : yyless(1); /* eat only 'n' this time */
524 :
525 2 : kwnum = ScanKeywordLookup("nchar",
526 2 : yyextra->keywordlist);
527 2 : if (kwnum >= 0)
528 : {
529 4 : yylval->keyword = GetScanKeyword(kwnum,
530 2 : yyextra->keywordlist);
531 2 : return yyextra->keyword_tokens[kwnum];
532 : }
533 : else
534 : {
535 : /* If NCHAR isn't a keyword, just return "n" */
536 0 : yylval->str = pstrdup("n");
537 0 : return IDENT;
538 : }
539 : }
540 :
541 419587 : {xqstart} {
542 419587 : yyextra->saw_non_ascii = false;
543 419587 : SET_YYLLOC();
544 419587 : BEGIN(xq);
545 419587 : startlit();
546 : }
547 419587 : {xestart} {
548 2600 : yyextra->saw_non_ascii = false;
549 2600 : SET_YYLLOC();
550 2600 : BEGIN(xe);
551 2600 : startlit();
552 : }
553 2600 : {xusstart} {
554 315 : SET_YYLLOC();
555 315 : BEGIN(xus);
556 315 : startlit();
557 : }
558 315 :
559 425099 : <xb,xh,xq,xe,xus>{quote} {
560 : /*
561 : * When we are scanning a quoted string and see an end
562 : * quote, we must look ahead for a possible continuation.
563 : * If we don't see one, we know the end quote was in fact
564 : * the end of the string. To reduce the lexer table size,
565 : * we use a single "xqs" state to do the lookahead for all
566 : * types of strings.
567 : */
568 425099 : yyextra->state_before_str_stop = YYSTATE;
569 425099 : BEGIN(xqs);
570 : }
571 425099 : <xqs>{quotecontinue} {
572 20 : /*
573 : * Found a quote continuation, so return to the in-quote
574 : * state and continue scanning the literal. Nothing is
575 : * added to the literal's contents.
576 : */
577 20 : BEGIN(yyextra->state_before_str_stop);
578 : }
579 20 : <xqs>{quotecontinuefail} |
580 425079 : <xqs>{other} |
581 : <xqs><<EOF>> {
582 : /*
583 : * Failed to see a quote continuation. Throw back
584 : * everything after the end quote, and handle the string
585 : * according to the state we were in previously.
586 : */
587 425079 : yyless(0);
588 425079 : BEGIN(INITIAL);
589 :
590 425079 : switch (yyextra->state_before_str_stop)
591 : {
592 409 : case xb:
593 409 : yylval->str = litbufdup(yyscanner);
594 409 : return BCONST;
595 2192 : case xh:
596 2192 : yylval->str = litbufdup(yyscanner);
597 2192 : return XCONST;
598 422163 : case xq:
599 : case xe:
600 : /*
601 : * Check that the data remains valid, if it might
602 : * have been made invalid by unescaping any chars.
603 : */
604 422163 : if (yyextra->saw_non_ascii)
605 3 : pg_verifymbstr(yyextra->literalbuf,
606 3 : yyextra->literallen,
607 : false);
608 422163 : yylval->str = litbufdup(yyscanner);
609 422163 : return SCONST;
610 315 : case xus:
611 315 : yylval->str = litbufdup(yyscanner);
612 315 : return USCONST;
613 0 : default:
614 0 : yyerror("unhandled previous state in xqs");
615 : }
616 : }
617 :
618 3445 : <xq,xe,xus>{xqdouble} {
619 3445 : addlitchar('\'', yyscanner);
620 : }
621 3445 : <xq,xus>{xqinside} {
622 411877 : addlit(yytext, yyleng, yyscanner);
623 : }
624 411877 : <xe>{xeinside} {
625 2890 : addlit(yytext, yyleng, yyscanner);
626 : }
627 2890 : <xe>{xeunicode} {
628 102 : char32_t c = strtoul(yytext + 2, NULL, 16);
629 :
630 : /* Remember start of overall string token ... */
631 102 : PUSH_YYLLOC();
632 : /* ... and set the error cursor to point at this esc seq */
633 102 : SET_YYLLOC();
634 :
635 102 : if (is_utf16_surrogate_first(c))
636 : {
637 15 : yyextra->utf16_first_part = c;
638 15 : BEGIN(xeu);
639 : }
640 87 : else if (is_utf16_surrogate_second(c))
641 0 : yyerror("invalid Unicode surrogate pair");
642 : else
643 87 : addunicode(c, yyscanner);
644 :
645 : /* Restore yylloc to be start of string token */
646 99 : POP_YYLLOC();
647 : }
648 99 : <xeu>{xeunicode} {
649 6 : char32_t c = strtoul(yytext + 2, NULL, 16);
650 :
651 : /* Remember start of overall string token ... */
652 6 : PUSH_YYLLOC();
653 : /* ... and set the error cursor to point at this esc seq */
654 6 : SET_YYLLOC();
655 :
656 6 : if (!is_utf16_surrogate_second(c))
657 6 : yyerror("invalid Unicode surrogate pair");
658 :
659 0 : c = surrogate_pair_to_codepoint(yyextra->utf16_first_part, c);
660 :
661 0 : addunicode(c, yyscanner);
662 :
663 : /* Restore yylloc to be start of string token */
664 0 : POP_YYLLOC();
665 :
666 0 : BEGIN(xe);
667 : }
668 0 : <xeu>. |
669 9 : <xeu>\n |
670 : <xeu><<EOF>> {
671 : /* Set the error cursor to point at missing esc seq */
672 9 : SET_YYLLOC();
673 9 : yyerror("invalid Unicode surrogate pair");
674 : }
675 : <xe,xeu>{xeunicodefail} {
676 6 : /* Set the error cursor to point at malformed esc seq */
677 6 : SET_YYLLOC();
678 6 : ereport(ERROR,
679 : (errcode(ERRCODE_INVALID_ESCAPE_SEQUENCE),
680 : errmsg("invalid Unicode escape"),
681 : errhint("Unicode escapes must be \\uXXXX or \\UXXXXXXXX."),
682 : lexer_errposition()));
683 : }
684 : <xe>{xeescape} {
685 1517 : if (yytext[1] == '\'')
686 : {
687 0 : if (yyextra->backslash_quote == BACKSLASH_QUOTE_OFF ||
688 0 : (yyextra->backslash_quote == BACKSLASH_QUOTE_SAFE_ENCODING &&
689 0 : PG_ENCODING_IS_CLIENT_ONLY(pg_get_client_encoding())))
690 0 : ereport(ERROR,
691 : (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
692 : errmsg("unsafe use of \\' in a string literal"),
693 : errhint("Use '' to write quotes in strings. \\' is insecure in client-only encodings."),
694 : lexer_errposition()));
695 : }
696 1517 : addlitchar(unescape_single_char(yytext[1], yyscanner),
697 : yyscanner);
698 : }
699 1517 : <xe>{xeoctesc} {
700 24 : unsigned char c = strtoul(yytext + 1, NULL, 8);
701 :
702 24 : addlitchar(c, yyscanner);
703 24 : if (c == '\0' || IS_HIGHBIT_SET(c))
704 0 : yyextra->saw_non_ascii = true;
705 : }
706 24 : <xe>{xehexesc} {
707 6 : unsigned char c = strtoul(yytext + 2, NULL, 16);
708 :
709 6 : addlitchar(c, yyscanner);
710 6 : if (c == '\0' || IS_HIGHBIT_SET(c))
711 5 : yyextra->saw_non_ascii = true;
712 : }
713 6 : <xe>. {
714 0 : /* This is only needed for \ just before EOF */
715 0 : addlitchar(yytext[0], yyscanner);
716 : }
717 0 : <xq,xe,xus><<EOF>> { yyerror("unterminated quoted string"); }
718 0 :
719 4918 : {dolqdelim} {
720 4918 : SET_YYLLOC();
721 4918 : yyextra->dolqstart = pstrdup(yytext);
722 4918 : BEGIN(xdolq);
723 4918 : startlit();
724 : }
725 4918 : {dolqfailed} {
726 0 : SET_YYLLOC();
727 : /* throw back all but the initial "$" */
728 0 : yyless(1);
729 : /* and treat it as {other} */
730 0 : return yytext[0];
731 : }
732 : <xdolq>{dolqdelim} {
733 5148 : if (strcmp(yytext, yyextra->dolqstart) == 0)
734 : {
735 4918 : pfree(yyextra->dolqstart);
736 4918 : yyextra->dolqstart = NULL;
737 4918 : BEGIN(INITIAL);
738 4918 : yylval->str = litbufdup(yyscanner);
739 4918 : return SCONST;
740 : }
741 : else
742 : {
743 : /*
744 : * When we fail to match $...$ to dolqstart, transfer
745 : * the $... part to the output, but put back the final
746 : * $ for rescanning. Consider $delim$...$junk$delim$
747 : */
748 230 : addlit(yytext, yyleng - 1, yyscanner);
749 230 : yyless(yyleng - 1);
750 : }
751 : }
752 230 : <xdolq>{dolqinside} {
753 7361 : addlit(yytext, yyleng, yyscanner);
754 : }
755 7361 : <xdolq>{dolqfailed} {
756 565 : addlit(yytext, yyleng, yyscanner);
757 : }
758 565 : <xdolq>. {
759 1878 : /* This is only needed for $ inside the quoted text */
760 1878 : addlitchar(yytext[0], yyscanner);
761 : }
762 1878 : <xdolq><<EOF>> { yyerror("unterminated dollar-quoted string"); }
763 0 :
764 62290 : {xdstart} {
765 62290 : SET_YYLLOC();
766 62290 : BEGIN(xd);
767 62290 : startlit();
768 : }
769 62290 : {xuistart} {
770 14 : SET_YYLLOC();
771 14 : BEGIN(xui);
772 14 : startlit();
773 : }
774 14 : <xd>{xdstop} {
775 62290 : char *ident;
776 :
777 62290 : BEGIN(INITIAL);
778 62290 : if (yyextra->literallen == 0)
779 3 : yyerror("zero-length delimited identifier");
780 62287 : ident = litbufdup(yyscanner);
781 62287 : if (yyextra->literallen >= NAMEDATALEN)
782 0 : truncate_identifier(ident, yyextra->literallen, true);
783 62287 : yylval->str = ident;
784 62287 : return IDENT;
785 : }
786 : <xui>{dquote} {
787 14 : BEGIN(INITIAL);
788 14 : if (yyextra->literallen == 0)
789 0 : yyerror("zero-length delimited identifier");
790 : /* can't truncate till after we de-escape the ident */
791 14 : yylval->str = litbufdup(yyscanner);
792 14 : return UIDENT;
793 : }
794 : <xd,xui>{xddouble} {
795 83 : addlitchar('"', yyscanner);
796 : }
797 83 : <xd,xui>{xdinside} {
798 62379 : addlit(yytext, yyleng, yyscanner);
799 : }
800 62379 : <xd,xui><<EOF>> { yyerror("unterminated quoted identifier"); }
801 0 :
802 0 : {xufailed} {
803 : char *ident;
804 :
805 0 : SET_YYLLOC();
806 : /* throw back all but the initial u/U */
807 0 : yyless(1);
808 : /* and treat it as {identifier} */
809 0 : ident = downcase_truncate_identifier(yytext, yyleng, true);
810 0 : yylval->str = ident;
811 0 : return IDENT;
812 : }
813 :
814 128014 : {typecast} {
815 128014 : SET_YYLLOC();
816 128014 : return TYPECAST;
817 : }
818 :
819 416 : {dot_dot} {
820 416 : SET_YYLLOC();
821 416 : return DOT_DOT;
822 : }
823 :
824 28332 : {colon_equals} {
825 28332 : SET_YYLLOC();
826 28332 : return COLON_EQUALS;
827 : }
828 :
829 1115 : {equals_greater} {
830 1115 : SET_YYLLOC();
831 1115 : return EQUALS_GREATER;
832 : }
833 :
834 3092 : {less_equals} {
835 3092 : SET_YYLLOC();
836 3092 : return LESS_EQUALS;
837 : }
838 :
839 7152 : {greater_equals} {
840 7152 : SET_YYLLOC();
841 7152 : return GREATER_EQUALS;
842 : }
843 :
844 6784 : {less_greater} {
845 : /* We accept both "<>" and "!=" as meaning NOT_EQUALS */
846 6784 : SET_YYLLOC();
847 6784 : return NOT_EQUALS;
848 : }
849 :
850 15881 : {not_equals} {
851 : /* We accept both "<>" and "!=" as meaning NOT_EQUALS */
852 15881 : SET_YYLLOC();
853 15881 : return NOT_EQUALS;
854 : }
855 :
856 3545375 : {self} {
857 3545375 : SET_YYLLOC();
858 3545375 : return yytext[0];
859 : }
860 :
861 32257 : {operator} {
862 : /*
863 : * Check for embedded slash-star or dash-dash; those
864 : * are comment starts, so operator must stop there.
865 : * Note that slash-star or dash-dash at the first
866 : * character will match a prior rule, not this one.
867 : */
868 32257 : int nchars = yyleng;
869 32257 : char *slashstar = strstr(yytext, "/*");
870 32257 : char *dashdash = strstr(yytext, "--");
871 :
872 32257 : if (slashstar && dashdash)
873 : {
874 : /* if both appear, take the first one */
875 0 : if (slashstar > dashdash)
876 0 : slashstar = dashdash;
877 : }
878 32257 : else if (!slashstar)
879 32222 : slashstar = dashdash;
880 32257 : if (slashstar)
881 54 : nchars = slashstar - yytext;
882 :
883 : /*
884 : * For SQL compatibility, '+' and '-' cannot be the
885 : * last char of a multi-char operator unless the operator
886 : * contains chars that are not in SQL operators.
887 : * The idea is to lex '=-' as two operators, but not
888 : * to forbid operator names like '?-' that could not be
889 : * sequences of SQL operators.
890 : */
891 32257 : if (nchars > 1 &&
892 26054 : (yytext[nchars - 1] == '+' ||
893 26050 : yytext[nchars - 1] == '-'))
894 : {
895 : int ic;
896 :
897 1807 : for (ic = nchars - 2; ic >= 0; ic--)
898 : {
899 1023 : char c = yytext[ic];
900 1023 : if (c == '~' || c == '!' || c == '@' ||
901 973 : c == '#' || c == '^' || c == '&' ||
902 829 : c == '|' || c == '`' || c == '?' ||
903 : c == '%')
904 : break;
905 : }
906 988 : if (ic < 0)
907 : {
908 : /*
909 : * didn't find a qualifying character, so remove
910 : * all trailing [+-]
911 : */
912 : do {
913 784 : nchars--;
914 784 : } while (nchars > 1 &&
915 23 : (yytext[nchars - 1] == '+' ||
916 23 : yytext[nchars - 1] == '-'));
917 : }
918 : }
919 :
920 32257 : SET_YYLLOC();
921 :
922 32257 : if (nchars < yyleng)
923 : {
924 : /* Strip the unwanted chars from the token */
925 838 : yyless(nchars);
926 : /*
927 : * If what we have left is only one char, and it's
928 : * one of the characters matching "self", then
929 : * return it as a character token the same way
930 : * that the "self" rule would have.
931 : */
932 838 : if (nchars == 1 &&
933 761 : strchr(",()[].;:+-*/%^<>=", yytext[0]))
934 761 : return yytext[0];
935 : /*
936 : * Likewise, if what we have left is two chars, and
937 : * those match the tokens ">=", "<=", "=>", "<>" or
938 : * "!=", then we must return the appropriate token
939 : * rather than the generic Op.
940 : */
941 77 : if (nchars == 2)
942 : {
943 77 : if (yytext[0] == '=' && yytext[1] == '>')
944 23 : return EQUALS_GREATER;
945 54 : if (yytext[0] == '>' && yytext[1] == '=')
946 11 : return GREATER_EQUALS;
947 43 : if (yytext[0] == '<' && yytext[1] == '=')
948 11 : return LESS_EQUALS;
949 32 : if (yytext[0] == '<' && yytext[1] == '>')
950 14 : return NOT_EQUALS;
951 18 : if (yytext[0] == '!' && yytext[1] == '=')
952 15 : return NOT_EQUALS;
953 : }
954 : }
955 :
956 : /*
957 : * Complain if operator is too long. Unlike the case
958 : * for identifiers, we make this an error not a notice-
959 : * and-truncate, because the odds are we are looking at
960 : * a syntactic mistake anyway.
961 : */
962 31422 : if (nchars >= NAMEDATALEN)
963 0 : yyerror("operator too long");
964 :
965 31422 : yylval->str = pstrdup(yytext);
966 31422 : return Op;
967 : }
968 :
969 24640 : {param} {
970 24640 : ErrorSaveContext escontext = {T_ErrorSaveContext};
971 : int32 val;
972 :
973 24640 : SET_YYLLOC();
974 24640 : val = pg_strtoint32_safe(yytext + 1, (Node *) &escontext);
975 24640 : if (escontext.error_occurred)
976 3 : yyerror("parameter number too large");
977 24637 : yylval->ival = val;
978 24637 : return PARAM;
979 : }
980 : {param_junk} {
981 6 : SET_YYLLOC();
982 6 : yyerror("trailing junk after parameter");
983 : }
984 :
985 238241 : {decinteger} {
986 238241 : SET_YYLLOC();
987 238241 : return process_integer_literal(yytext, yylval, 10);
988 : }
989 : {hexinteger} {
990 478 : SET_YYLLOC();
991 478 : return process_integer_literal(yytext, yylval, 16);
992 : }
993 : {octinteger} {
994 30 : SET_YYLLOC();
995 30 : return process_integer_literal(yytext, yylval, 8);
996 : }
997 : {bininteger} {
998 31 : SET_YYLLOC();
999 31 : return process_integer_literal(yytext, yylval, 2);
1000 : }
1001 : {hexfail} {
1002 3 : SET_YYLLOC();
1003 3 : yyerror("invalid hexadecimal integer");
1004 : }
1005 : {octfail} {
1006 3 : SET_YYLLOC();
1007 3 : yyerror("invalid octal integer");
1008 : }
1009 : {binfail} {
1010 3 : SET_YYLLOC();
1011 3 : yyerror("invalid binary integer");
1012 : }
1013 : {numeric} {
1014 5796 : SET_YYLLOC();
1015 5796 : yylval->str = pstrdup(yytext);
1016 5796 : return FCONST;
1017 : }
1018 : {numericfail} {
1019 53 : /* throw back the .., and treat as integer */
1020 53 : yyless(yyleng - 2);
1021 53 : SET_YYLLOC();
1022 53 : return process_integer_literal(yytext, yylval, 10);
1023 : }
1024 : {real} {
1025 680 : SET_YYLLOC();
1026 680 : yylval->str = pstrdup(yytext);
1027 680 : return FCONST;
1028 : }
1029 : {realfail} {
1030 3 : SET_YYLLOC();
1031 3 : yyerror("trailing junk after numeric literal");
1032 : }
1033 : {integer_junk} {
1034 33 : SET_YYLLOC();
1035 33 : yyerror("trailing junk after numeric literal");
1036 : }
1037 : {numeric_junk} {
1038 24 : SET_YYLLOC();
1039 24 : yyerror("trailing junk after numeric literal");
1040 : }
1041 : {real_junk} {
1042 0 : SET_YYLLOC();
1043 0 : yyerror("trailing junk after numeric literal");
1044 : }
1045 :
1046 5845336 :
1047 : {identifier} {
1048 : int kwnum;
1049 : char *ident;
1050 :
1051 5845336 : SET_YYLLOC();
1052 :
1053 : /* Is it a keyword? */
1054 5845336 : kwnum = ScanKeywordLookup(yytext,
1055 5845336 : yyextra->keywordlist);
1056 5845336 : if (kwnum >= 0)
1057 : {
1058 5228088 : yylval->keyword = GetScanKeyword(kwnum,
1059 2614044 : yyextra->keywordlist);
1060 2614044 : return yyextra->keyword_tokens[kwnum];
1061 : }
1062 :
1063 : /*
1064 : * No. Convert the identifier to lower case, and truncate
1065 : * if necessary.
1066 : */
1067 3231292 : ident = downcase_truncate_identifier(yytext, yyleng, true);
1068 3231292 : yylval->str = ident;
1069 3231292 : return IDENT;
1070 : }
1071 :
1072 0 : {other} {
1073 0 : SET_YYLLOC();
1074 0 : return yytext[0];
1075 : }
1076 :
1077 413246 : <<EOF>> {
1078 413246 : SET_YYLLOC();
1079 413246 : yyterminate();
1080 : }
1081 :
1082 0 : %%
1083 :
1084 : /* LCOV_EXCL_STOP */
1085 :
1086 : /*
1087 : * Arrange access to yyextra for subroutines of the main yylex() function.
1088 : * We expect each subroutine to have a yyscanner parameter. Rather than
1089 : * use the yyget_xxx functions, which might or might not get inlined by the
1090 : * compiler, we cheat just a bit and cast yyscanner to the right type.
1091 : */
1092 : #undef yyextra
1093 : #define yyextra (((struct yyguts_t *) yyscanner)->yyextra_r)
1094 :
1095 : /* Likewise for a couple of other things we need. */
1096 : #undef yylloc
1097 : #define yylloc (((struct yyguts_t *) yyscanner)->yylloc_r)
1098 : #undef yyleng
1099 : #define yyleng (((struct yyguts_t *) yyscanner)->yyleng_r)
1100 :
1101 :
1102 : /*
1103 : * scanner_errposition
1104 : * Report a lexer or grammar error cursor position, if possible.
1105 : *
1106 : * This is expected to be used within an ereport() call, or via an error
1107 : * callback such as setup_scanner_errposition_callback(). The return value
1108 : * is a dummy (always 0, in fact).
1109 : *
1110 : * Note that this can only be used for messages emitted during raw parsing
1111 : * (essentially, scan.l, parser.c, and gram.y), since it requires the
1112 : * yyscanner struct to still be available.
1113 : */
1114 : int
1115 620 : scanner_errposition(int location, core_yyscan_t yyscanner)
1116 : {
1117 : int pos;
1118 :
1119 620 : if (location < 0)
1120 0 : return 0; /* no-op if location is unknown */
1121 :
1122 : /* Convert byte offset to character number */
1123 620 : pos = pg_mbstrlen_with_len(yyextra->scanbuf, location) + 1;
1124 : /* And pass it to the ereport mechanism */
1125 620 : return errposition(pos);
1126 : }
1127 :
1128 : /*
1129 : * Error context callback for inserting scanner error location.
1130 : *
1131 : * Note that this will be called for *any* error occurring while the
1132 : * callback is installed. We avoid inserting an irrelevant error location
1133 : * if the error is a query cancel --- are there any other important cases?
1134 : */
1135 : static void
1136 18 : scb_error_callback(void *arg)
1137 : {
1138 18 : ScannerCallbackState *scbstate = (ScannerCallbackState *) arg;
1139 :
1140 18 : if (geterrcode() != ERRCODE_QUERY_CANCELED)
1141 18 : (void) scanner_errposition(scbstate->location, scbstate->yyscanner);
1142 18 : }
1143 :
1144 : /*
1145 : * setup_scanner_errposition_callback
1146 : * Arrange for non-scanner errors to report an error position
1147 : *
1148 : * Sometimes the scanner calls functions that aren't part of the scanner
1149 : * subsystem and can't reasonably be passed the yyscanner pointer; yet
1150 : * we would like any errors thrown in those functions to be tagged with an
1151 : * error location. Use this function to set up an error context stack
1152 : * entry that will accomplish that. Usage pattern:
1153 : *
1154 : * declare a local variable "ScannerCallbackState scbstate"
1155 : * ...
1156 : * setup_scanner_errposition_callback(&scbstate, yyscanner, location);
1157 : * call function that might throw error;
1158 : * cancel_scanner_errposition_callback(&scbstate);
1159 : */
1160 : void
1161 485 : setup_scanner_errposition_callback(ScannerCallbackState *scbstate,
1162 : core_yyscan_t yyscanner,
1163 : int location)
1164 : {
1165 : /* Setup error traceback support for ereport() */
1166 485 : scbstate->yyscanner = yyscanner;
1167 485 : scbstate->location = location;
1168 485 : scbstate->errcallback.callback = scb_error_callback;
1169 485 : scbstate->errcallback.arg = scbstate;
1170 485 : scbstate->errcallback.previous = error_context_stack;
1171 485 : error_context_stack = &scbstate->errcallback;
1172 485 : }
1173 :
1174 : /*
1175 : * Cancel a previously-set-up errposition callback.
1176 : */
1177 : void
1178 467 : cancel_scanner_errposition_callback(ScannerCallbackState *scbstate)
1179 : {
1180 : /* Pop the error context stack */
1181 467 : error_context_stack = scbstate->errcallback.previous;
1182 467 : }
1183 :
1184 : /*
1185 : * scanner_yyerror
1186 : * Report a lexer or grammar error.
1187 : *
1188 : * The message's cursor position is whatever YYLLOC was last set to,
1189 : * ie, the start of the current token if called within yylex(), or the
1190 : * most recently lexed token if called from the grammar.
1191 : * This is OK for syntax error messages from the Bison parser, because Bison
1192 : * parsers report error as soon as the first unparsable token is reached.
1193 : * Beware of using yyerror for other purposes, as the cursor position might
1194 : * be misleading!
1195 : */
1196 : void
1197 461 : scanner_yyerror(const char *message, core_yyscan_t yyscanner)
1198 : {
1199 461 : const char *loc = yyextra->scanbuf + *yylloc;
1200 :
1201 461 : if (*loc == YY_END_OF_BUFFER_CHAR)
1202 : {
1203 9 : ereport(ERROR,
1204 : (errcode(ERRCODE_SYNTAX_ERROR),
1205 : /* translator: %s is typically the translation of "syntax error" */
1206 : errmsg("%s at end of input", _(message)),
1207 : lexer_errposition()));
1208 : }
1209 : else
1210 : {
1211 452 : ereport(ERROR,
1212 : (errcode(ERRCODE_SYNTAX_ERROR),
1213 : /* translator: first %s is typically the translation of "syntax error" */
1214 : errmsg("%s at or near \"%s\"", _(message), loc),
1215 : lexer_errposition()));
1216 : }
1217 : }
1218 :
1219 :
1220 : /*
1221 : * Called before any actual parsing is done
1222 : */
1223 : core_yyscan_t
1224 425003 : scanner_init(const char *str,
1225 : core_yy_extra_type *yyext,
1226 : const ScanKeywordList *keywordlist,
1227 : const uint16 *keyword_tokens)
1228 : {
1229 425003 : Size slen = strlen(str);
1230 : yyscan_t scanner;
1231 :
1232 425003 : if (yylex_init(&scanner) != 0)
1233 0 : elog(ERROR, "yylex_init() failed: %m");
1234 :
1235 425003 : core_yyset_extra(yyext, scanner);
1236 :
1237 425003 : yyext->keywordlist = keywordlist;
1238 425003 : yyext->keyword_tokens = keyword_tokens;
1239 :
1240 425003 : yyext->backslash_quote = backslash_quote;
1241 :
1242 : /*
1243 : * Make a scan buffer with special termination needed by flex.
1244 : */
1245 425003 : yyext->scanbuf = (char *) palloc(slen + 2);
1246 425003 : yyext->scanbuflen = slen;
1247 425003 : memcpy(yyext->scanbuf, str, slen);
1248 425003 : yyext->scanbuf[slen] = yyext->scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
1249 425003 : yy_scan_buffer(yyext->scanbuf, slen + 2, scanner);
1250 :
1251 : /* initialize literal buffer to a reasonable but expansible size */
1252 425003 : yyext->literalalloc = 1024;
1253 425003 : yyext->literalbuf = (char *) palloc(yyext->literalalloc);
1254 425003 : yyext->literallen = 0;
1255 :
1256 425003 : return scanner;
1257 : }
1258 :
1259 :
1260 : /*
1261 : * Called after parsing is done to clean up after scanner_init()
1262 : */
1263 : void
1264 424303 : scanner_finish(core_yyscan_t yyscanner)
1265 : {
1266 : /*
1267 : * We don't bother to call yylex_destroy(), because all it would do is
1268 : * pfree a small amount of control storage. It's cheaper to leak the
1269 : * storage until the parsing context is destroyed. The amount of space
1270 : * involved is usually negligible compared to the output parse tree
1271 : * anyway.
1272 : *
1273 : * We do bother to pfree the scanbuf and literal buffer, but only if they
1274 : * represent a nontrivial amount of space. The 8K cutoff is arbitrary.
1275 : */
1276 424303 : if (yyextra->scanbuflen >= 8192)
1277 57 : pfree(yyextra->scanbuf);
1278 424303 : if (yyextra->literalalloc >= 8192)
1279 41 : pfree(yyextra->literalbuf);
1280 424303 : }
1281 :
1282 :
1283 : static void
1284 487971 : addlit(char *ytext, int yleng, core_yyscan_t yyscanner)
1285 : {
1286 : /* enlarge buffer if needed */
1287 487971 : if ((yyextra->literallen + yleng) >= yyextra->literalalloc)
1288 : {
1289 184 : yyextra->literalalloc = pg_nextpower2_32(yyextra->literallen + yleng + 1);
1290 184 : yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
1291 184 : yyextra->literalalloc);
1292 : }
1293 : /* append new data */
1294 487971 : memcpy(yyextra->literalbuf + yyextra->literallen, ytext, yleng);
1295 487971 : yyextra->literallen += yleng;
1296 487971 : }
1297 :
1298 :
1299 : static void
1300 9554 : addlitchar(unsigned char ychar, core_yyscan_t yyscanner)
1301 : {
1302 : /* enlarge buffer if needed */
1303 9554 : if ((yyextra->literallen + 1) >= yyextra->literalalloc)
1304 : {
1305 0 : yyextra->literalalloc *= 2;
1306 0 : yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
1307 0 : yyextra->literalalloc);
1308 : }
1309 : /* append new data */
1310 9554 : yyextra->literalbuf[yyextra->literallen] = ychar;
1311 9554 : yyextra->literallen += 1;
1312 9554 : }
1313 :
1314 :
1315 : /*
1316 : * Create a palloc'd copy of literalbuf, adding a trailing null.
1317 : */
1318 : static char *
1319 492298 : litbufdup(core_yyscan_t yyscanner)
1320 : {
1321 492298 : int llen = yyextra->literallen;
1322 : char *new;
1323 :
1324 492298 : new = palloc(llen + 1);
1325 492298 : memcpy(new, yyextra->literalbuf, llen);
1326 492298 : new[llen] = '\0';
1327 492298 : return new;
1328 : }
1329 :
1330 : /*
1331 : * Process {decinteger}, {hexinteger}, etc. Note this will also do the right
1332 : * thing with {numeric}, ie digits and a decimal point.
1333 : */
1334 : static int
1335 238833 : process_integer_literal(const char *token, YYSTYPE *lval, int base)
1336 : {
1337 238833 : ErrorSaveContext escontext = {T_ErrorSaveContext};
1338 : int32 val;
1339 :
1340 238833 : val = pg_strtoint32_safe(token, (Node *) &escontext);
1341 238833 : if (escontext.error_occurred)
1342 : {
1343 : /* integer too large (or contains decimal pt), treat it as a float */
1344 852 : lval->str = pstrdup(token);
1345 852 : return FCONST;
1346 : }
1347 237981 : lval->ival = val;
1348 237981 : return ICONST;
1349 : }
1350 :
1351 : static void
1352 87 : addunicode(char32_t c, core_yyscan_t yyscanner)
1353 : {
1354 : ScannerCallbackState scbstate;
1355 : char buf[MAX_UNICODE_EQUIVALENT_STRING + 1];
1356 :
1357 87 : if (!is_valid_unicode_codepoint(c))
1358 3 : yyerror("invalid Unicode escape value");
1359 :
1360 : /*
1361 : * We expect that pg_unicode_to_server() will complain about any
1362 : * unconvertible code point, so we don't have to set saw_non_ascii.
1363 : */
1364 84 : setup_scanner_errposition_callback(&scbstate, yyscanner, *(yylloc));
1365 84 : pg_unicode_to_server(c, (unsigned char *) buf);
1366 84 : cancel_scanner_errposition_callback(&scbstate);
1367 84 : addlit(buf, strlen(buf), yyscanner);
1368 84 : }
1369 :
1370 : static unsigned char
1371 1517 : unescape_single_char(unsigned char c, core_yyscan_t yyscanner)
1372 : {
1373 1517 : switch (c)
1374 : {
1375 13 : case 'b':
1376 13 : return '\b';
1377 1 : case 'f':
1378 1 : return '\f';
1379 716 : case 'n':
1380 716 : return '\n';
1381 36 : case 'r':
1382 36 : return '\r';
1383 14 : case 't':
1384 14 : return '\t';
1385 0 : case 'v':
1386 0 : return '\v';
1387 737 : default:
1388 : /* check for backslash followed by non-7-bit-ASCII */
1389 737 : if (c == '\0' || IS_HIGHBIT_SET(c))
1390 0 : yyextra->saw_non_ascii = true;
1391 :
1392 737 : return c;
1393 : }
1394 : }
1395 :
1396 : /*
1397 : * Interface functions to make flex use palloc() instead of malloc().
1398 : * It'd be better to make these static, but flex insists otherwise.
1399 : */
1400 :
1401 : void *
1402 1275009 : core_yyalloc(yy_size_t bytes, core_yyscan_t yyscanner)
1403 : {
1404 1275009 : return palloc(bytes);
1405 : }
1406 :
1407 : void *
1408 0 : core_yyrealloc(void *ptr, yy_size_t bytes, core_yyscan_t yyscanner)
1409 : {
1410 0 : if (ptr)
1411 0 : return repalloc(ptr, bytes);
1412 : else
1413 0 : return palloc(bytes);
1414 : }
1415 :
1416 : void
1417 0 : core_yyfree(void *ptr, core_yyscan_t yyscanner)
1418 : {
1419 0 : if (ptr)
1420 0 : pfree(ptr);
1421 0 : }
|
