Line data Source code
1 : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
2 :
3 : // This file is part of GCC.
4 :
5 : // GCC is free software; you can redistribute it and/or modify it under
6 : // the terms of the GNU General Public License as published by the Free
7 : // Software Foundation; either version 3, or (at your option) any later
8 : // version.
9 :
10 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 : // for more details.
14 :
15 : // You should have received a copy of the GNU General Public License
16 : // along with GCC; see the file COPYING3. If not see
17 : // <http://www.gnu.org/licenses/>.
18 :
19 : #include "rust-codepoint.h"
20 : #include "rust-system.h"
21 : #include "rust-lex.h"
22 : #include "rust-diagnostics.h"
23 : #include "rust-linemap.h"
24 : #include "rust-edition.h"
25 : #include "safe-ctype.h"
26 : #include "cpplib.h"
27 : #include "rust-keyword-values.h"
28 :
29 : namespace Rust {
30 : // TODO: move to separate compilation unit?
31 : // overload += for uint32_t to allow 32-bit encoded utf-8 to be added
32 : std::string &
33 9613199 : operator+= (std::string &str, Codepoint char32)
34 : {
35 9613199 : if (char32.value < 0x80)
36 : {
37 9606011 : str += static_cast<char> (char32.value);
38 : }
39 7188 : else if (char32.value < (0x1F + 1) << (1 * 6))
40 : {
41 2831 : str += static_cast<char> (0xC0 | ((char32.value >> 6) & 0x1F));
42 2831 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
43 : }
44 4357 : else if (char32.value < (0x0F + 1) << (2 * 6))
45 : {
46 3389 : str += static_cast<char> (0xE0 | ((char32.value >> 12) & 0x0F));
47 3389 : str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
48 3389 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
49 : }
50 968 : else if (char32.value < (0x07 + 1) << (3 * 6))
51 : {
52 964 : str += static_cast<char> (0xF0 | ((char32.value >> 18) & 0x07));
53 964 : str += static_cast<char> (0x80 | ((char32.value >> 12) & 0x3F));
54 964 : str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
55 964 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
56 : }
57 : else
58 : {
59 4 : rust_debug ("Invalid unicode codepoint found: '%u' ", char32.value);
60 : }
61 9613199 : return str;
62 : }
63 :
64 : std::string
65 7316231 : Codepoint::as_string ()
66 : {
67 7316231 : std::string str;
68 :
69 : // str += Codepoint (value);
70 7316231 : str += *this;
71 :
72 7316231 : return str;
73 : }
74 :
75 : /* Includes all allowable float digits EXCEPT _ and . as that needs lookahead
76 : * for handling. */
77 : bool
78 0 : is_float_digit (uint32_t number)
79 : {
80 0 : return ISDIGIT (number) || number == 'E' || number == 'e';
81 : }
82 :
83 : /* Basically ISXDIGIT from safe-ctype but may change if Rust's encoding or
84 : * whatever is different */
85 : bool
86 62752 : is_x_digit (uint32_t number)
87 : {
88 62752 : return ISXDIGIT (number);
89 : }
90 :
91 : bool
92 87 : is_octal_digit (uint32_t number)
93 : {
94 87 : return number >= '0' && number <= '7';
95 : }
96 :
97 : bool
98 19102 : is_bin_digit (uint32_t number)
99 : {
100 19102 : return number == '0' || number == '1';
101 : }
102 :
103 : bool
104 10731 : check_valid_float_dot_end (uint32_t character)
105 : {
106 10731 : return character != '.' && character != '_' && !ISALPHA (character);
107 : }
108 :
109 : bool
110 7359 : is_whitespace (uint32_t character)
111 : {
112 : // https://doc.rust-lang.org/reference/whitespace.html
113 7359 : switch (character)
114 : {
115 : case '\t':
116 : case '\n':
117 : case '\v':
118 : case '\f':
119 : case '\r':
120 : case ' ':
121 : case 0x0085: // next line
122 : case 0x200e: // left-to-right mark
123 : case 0x200f: // right-to-left mark
124 : case 0x2028: // line separator
125 : case 0x2029: // paragraph separator
126 : return true;
127 3582 : default:
128 3582 : return false;
129 : }
130 : }
131 :
132 : bool
133 17127 : is_non_decimal_int_literal_separator (uint32_t character)
134 : {
135 17127 : return character == 'x' || character == 'o' || character == 'b';
136 : }
137 :
138 : bool
139 631634 : is_identifier_start (uint32_t codepoint)
140 : {
141 631634 : return (cpp_check_xid_property (codepoint) & CPP_XID_START)
142 631634 : || codepoint == '_';
143 : }
144 :
145 : bool
146 2436806 : is_identifier_continue (uint32_t codepoint)
147 : {
148 2436806 : return cpp_check_xid_property (codepoint) & CPP_XID_CONTINUE;
149 : }
150 :
151 118 : Lexer::Lexer (const std::string &input, Linemap *linemap)
152 118 : : input (RAIIFile::create_error ()), current_line (1), current_column (1),
153 118 : line_map (linemap), dump_lex_out ({}),
154 118 : raw_input_source (new BufferInputSource (input, 0)),
155 118 : input_queue{*raw_input_source}, token_queue (TokenSource (this))
156 118 : {}
157 :
158 5239 : Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap,
159 5239 : tl::optional<std::ofstream &> dump_lex_opt)
160 5239 : : input (std::move (file_input)), current_line (1), current_column (1),
161 5239 : line_map (linemap), dump_lex_out (dump_lex_opt),
162 10478 : raw_input_source (new FileInputSource (input.get_raw ())),
163 10478 : input_queue{*raw_input_source}, token_queue (TokenSource (this))
164 : {
165 : // inform line_table that file is being entered and is in line 1
166 5239 : if (linemap)
167 5239 : line_map->start_file (filename, current_line);
168 5239 : }
169 :
170 5348 : Lexer::~Lexer ()
171 : {
172 : /* ok apparently stop (which is equivalent of original code in destructor) is
173 : * meant to be called after all files have finished parsing, for cleanup. On
174 : * the other hand, actual code that it calls to leave a certain line map is
175 : * mentioned in GCC docs as being useful for "just leaving an included header"
176 : * and stuff like that, so this line mapping functionality may need fixing.
177 : * FIXME: find out whether this occurs. */
178 :
179 : // line_map->stop();
180 5348 : }
181 :
182 : bool
183 4983 : Lexer::input_source_is_valid_utf8 ()
184 : {
185 4983 : return raw_input_source->is_valid ();
186 : }
187 :
188 : location_t
189 3599854 : Lexer::get_current_location ()
190 : {
191 3599854 : if (line_map)
192 3599629 : return linemap_position_for_column (line_table, current_column);
193 : else
194 : // If we have no linemap, we're lexing something without proper locations
195 : return UNDEF_LOCATION;
196 : }
197 :
198 : Codepoint
199 10404305 : Lexer::peek_input (int n)
200 : {
201 10404305 : return input_queue.peek (n);
202 : }
203 :
204 : Codepoint
205 9941174 : Lexer::peek_input ()
206 : {
207 9941174 : return peek_input (0);
208 : }
209 :
210 : void
211 8701974 : Lexer::skip_input (int n)
212 : {
213 8701974 : input_queue.skip (n);
214 8701974 : }
215 :
216 : void
217 8647154 : Lexer::skip_input ()
218 : {
219 8647154 : skip_input (0);
220 8647154 : }
221 :
222 : void
223 1520504 : Lexer::skip_token (int n)
224 : {
225 : // dump tokens if dump-lex option is enabled
226 1520504 : if (dump_lex_out.has_value ())
227 55 : dump_and_skip (n);
228 : else
229 1520449 : token_queue.skip (n);
230 1520504 : }
231 :
232 : void
233 55 : Lexer::dump_and_skip (int n)
234 : {
235 55 : std::ofstream &out = dump_lex_out.value ();
236 55 : bool found_eof = false;
237 55 : const_TokenPtr tok;
238 110 : for (int i = 0; i < n + 1; i++)
239 : {
240 55 : if (!found_eof)
241 : {
242 55 : tok = peek_token ();
243 55 : found_eof |= tok->get_id () == Rust::END_OF_FILE;
244 :
245 55 : location_t loc = tok->get_locus ();
246 :
247 55 : out << "<id=";
248 55 : out << tok->token_id_to_str ();
249 55 : out << (tok->should_have_str ()
250 167 : ? (std::string (", text=") + tok->get_str ()
251 167 : + std::string (", typehint=")
252 93 : + std::string (tok->get_type_hint_str ()))
253 91 : : "")
254 110 : << " ";
255 110 : out << Linemap::location_to_string (loc) << '\n';
256 : }
257 :
258 55 : token_queue.skip (0);
259 : }
260 55 : }
261 :
262 : void
263 0 : Lexer::replace_current_token (TokenPtr replacement)
264 : {
265 0 : token_queue.replace_current_value (replacement);
266 :
267 0 : rust_debug ("called 'replace_current_token' - this is deprecated");
268 0 : }
269 :
270 : /* Determines whether the string passed in is a keyword or not. If it is, it
271 : * returns the keyword name. */
272 : TokenId
273 514254 : Lexer::classify_keyword (const std::string &str)
274 : {
275 514254 : auto &keywords = Rust::Values::Keywords::keywords_tokens;
276 514254 : auto keyword = keywords.find (str);
277 :
278 514254 : if (keyword == keywords.end ())
279 : return IDENTIFIER;
280 :
281 148550 : auto id = keyword->second;
282 :
283 : // We now have the expected token ID of the reserved keyword. However, some
284 : // keywords are reserved starting in certain editions. For example, `try` is
285 : // only a reserved keyword in editions >=2018. The language might gain new
286 : // reserved keywords in the future.
287 : //
288 : // https://doc.rust-lang.org/reference/keywords.html#reserved-keywords
289 :
290 : // `try` is not a reserved keyword before 2018
291 148550 : if (get_rust_edition () == Edition::E2015 && id == TRY)
292 : return IDENTIFIER;
293 :
294 : return id;
295 : }
296 :
297 : TokenPtr
298 1530751 : Lexer::build_token ()
299 : {
300 : // loop to go through multiple characters to build a single token
301 3570552 : while (true)
302 : {
303 3570552 : location_t loc = get_current_location ();
304 :
305 3570552 : current_char = peek_input ();
306 3570552 : skip_input ();
307 :
308 : // detect shebang
309 : // Must be the first thing on the first line, starting with #!
310 : // But since an attribute can also start with an #! we don't count it as a
311 : // shebang line when after any whitespace or comments there is a [. If it
312 : // is a shebang line we simple drop the line. Otherwise we don't consume
313 : // any characters and fall through to the real tokenizer.
314 35912 : if (current_line == 1 && current_column == 1 && current_char == '#'
315 3606464 : && peek_input () == '!')
316 : {
317 : int n = 1;
318 3399 : while (true)
319 : {
320 3399 : Codepoint next_char = peek_input (n);
321 3399 : if (is_whitespace (next_char.value))
322 7 : n++;
323 3392 : else if ((next_char == '/' && peek_input (n + 1) == '/'
324 7 : && peek_input (n + 2) != '!'
325 7 : && peek_input (n + 2) != '/')
326 3413 : || (next_char == '/' && peek_input (n + 1) == '/'
327 0 : && peek_input (n + 2) == '/'
328 0 : && peek_input (n + 3) == '/'))
329 : {
330 : // two // or four ////
331 : // A single line comment
332 : // (but not an inner or outer doc comment)
333 7 : n += 2;
334 7 : next_char = peek_input (n);
335 119 : while (next_char != '\n' && !next_char.is_eof ())
336 : {
337 112 : n++;
338 112 : next_char = peek_input (n);
339 : }
340 7 : if (next_char == '\n')
341 7 : n++;
342 : }
343 3385 : else if (next_char == '/' && peek_input (n + 1) == '*'
344 0 : && peek_input (n + 2) == '*'
345 3385 : && peek_input (n + 3) == '/')
346 : {
347 : /**/
348 0 : n += 4;
349 : }
350 3385 : else if (next_char == '/' && peek_input (n + 1) == '*'
351 0 : && peek_input (n + 2) == '*' && peek_input (n + 3) == '*'
352 3385 : && peek_input (n + 4) == '/')
353 : {
354 : /***/
355 0 : n += 5;
356 : }
357 3385 : else if ((next_char == '/' && peek_input (n + 1) == '*'
358 0 : && peek_input (n + 2) != '*'
359 0 : && peek_input (n + 2) != '!')
360 3406 : || (next_char == '/' && peek_input (n + 1) == '*'
361 0 : && peek_input (n + 2) == '*'
362 0 : && peek_input (n + 3) == '*'))
363 : {
364 : // one /* or three /***
365 : // Start of a block comment
366 : // (but not an inner or outer doc comment)
367 0 : n += 2;
368 0 : int level = 1;
369 0 : while (level > 0)
370 : {
371 0 : if (peek_input (n).is_eof ())
372 : break;
373 0 : else if (peek_input (n) == '/'
374 0 : && peek_input (n + 1) == '*')
375 : {
376 0 : n += 2;
377 0 : level += 1;
378 : }
379 0 : else if (peek_input (n) == '*'
380 0 : && peek_input (n + 1) == '/')
381 : {
382 0 : n += 2;
383 0 : level -= 1;
384 : }
385 : else
386 0 : n++;
387 : }
388 : }
389 3385 : else if (next_char != '[')
390 : {
391 : // definitely shebang, ignore the first line
392 518 : while (current_char != '\n' && !current_char.is_eof ())
393 : {
394 490 : current_char = peek_input ();
395 490 : skip_input ();
396 : }
397 :
398 : // newline
399 28 : current_line++;
400 28 : current_column = 1;
401 : // tell line_table that new line starts
402 28 : start_line (current_line, max_column_hint);
403 28 : break;
404 : }
405 : else
406 : break; /* Definitely not a shebang line. */
407 : }
408 : }
409 :
410 : // return end of file token if end of file
411 3570552 : if (current_char.is_eof ())
412 5642 : return Token::make (END_OF_FILE, loc);
413 :
414 : // if not end of file, start tokenising
415 3564910 : switch (current_char.value)
416 : {
417 : /* ignore whitespace characters for tokens but continue updating
418 : * location */
419 265623 : case '\n': // newline
420 265623 : case 0x0085: // next line
421 265623 : case 0x2028: // line separator
422 265623 : case 0x2029: // paragraph separator
423 265623 : current_line++;
424 265623 : current_column = 1;
425 : // tell line_table that new line starts
426 265623 : start_line (current_line, max_column_hint);
427 265623 : continue;
428 252 : case '\r': // cr
429 : // Ignore, we expect a newline (lf) soon.
430 252 : continue;
431 1759259 : case ' ': // space
432 1759259 : current_column++;
433 1759259 : continue;
434 114 : case '\t': // horizontal tab
435 : // width of a tab is not well-defined, assume 8 spaces
436 114 : current_column += 8;
437 114 : continue;
438 28 : case '\v': // vertical tab
439 28 : case 0x000c: // form feed
440 28 : case 0x200e: // left-to-right mark
441 28 : case 0x200f: // right-to-left mark
442 : // Ignored.
443 28 : continue;
444 :
445 : // punctuation - actual tokens
446 58426 : case '=':
447 58426 : if (peek_input () == '>')
448 : {
449 : // match arm arrow
450 6806 : skip_input ();
451 6806 : current_column += 2;
452 6806 : loc += 1;
453 :
454 6806 : return Token::make (MATCH_ARROW, loc);
455 : }
456 51620 : else if (peek_input () == '=')
457 : {
458 : // equality operator
459 1000 : skip_input ();
460 1000 : current_column += 2;
461 1000 : loc += 1;
462 :
463 1000 : return Token::make (EQUAL_EQUAL, loc);
464 : }
465 : else
466 : {
467 : // assignment operator
468 50620 : current_column++;
469 50620 : return Token::make (EQUAL, loc);
470 : }
471 102628 : case '(':
472 102628 : current_column++;
473 102628 : return Token::make (LEFT_PAREN, loc);
474 21810 : case '-':
475 21810 : if (peek_input () == '>')
476 : {
477 : // return type specifier
478 16814 : skip_input ();
479 16814 : current_column += 2;
480 16814 : loc += 1;
481 :
482 16814 : return Token::make (RETURN_TYPE, loc);
483 : }
484 4996 : else if (peek_input () == '=')
485 : {
486 : // minus-assign
487 197 : skip_input ();
488 197 : current_column += 2;
489 197 : loc += 1;
490 :
491 197 : return Token::make (MINUS_EQ, loc);
492 : }
493 : else
494 : {
495 : // minus
496 4799 : current_column++;
497 4799 : return Token::make (MINUS, loc);
498 : }
499 2696 : case '+':
500 2696 : if (peek_input () == '=')
501 : {
502 : // add-assign
503 311 : skip_input ();
504 311 : current_column += 2;
505 311 : loc += 1;
506 :
507 311 : return Token::make (PLUS_EQ, loc);
508 : }
509 : else
510 : {
511 : // add
512 2385 : current_column++;
513 2385 : return Token::make (PLUS, loc);
514 : }
515 102609 : case ')':
516 102609 : current_column++;
517 102609 : return Token::make (RIGHT_PAREN, loc);
518 49575 : case ';':
519 49575 : current_column++;
520 49575 : return Token::make (SEMICOLON, loc);
521 13617 : case '*':
522 13617 : if (peek_input () == '=')
523 : {
524 : // multiplication-assign
525 14 : skip_input ();
526 14 : current_column += 2;
527 14 : loc += 1;
528 :
529 14 : return Token::make (ASTERISK_EQ, loc);
530 : }
531 : else
532 : {
533 : // multiplication
534 13603 : current_column++;
535 13603 : return Token::make (ASTERISK, loc);
536 : }
537 113307 : case ',':
538 113307 : current_column++;
539 113307 : return Token::make (COMMA, loc);
540 67108 : case '/':
541 67108 : if (peek_input () == '=')
542 : {
543 : // division-assign
544 33 : skip_input ();
545 33 : current_column += 2;
546 33 : loc += 1;
547 :
548 33 : return Token::make (DIV_EQ, loc);
549 : }
550 67075 : else if ((peek_input () == '/' && peek_input (1) != '!'
551 63278 : && peek_input (1) != '/')
552 119435 : || (peek_input () == '/' && peek_input (1) == '/'
553 49784 : && peek_input (2) == '/'))
554 : {
555 : // two // or four ////
556 : // single line comment
557 : // (but not an inner or outer doc comment)
558 13580 : skip_input ();
559 13580 : current_column += 2;
560 13580 : current_char = peek_input ();
561 :
562 : // basically ignore until line finishes
563 688159 : while (current_char != '\n' && !current_char.is_eof ())
564 : {
565 660999 : skip_input ();
566 660999 : current_column++; // not used
567 660999 : current_char = peek_input ();
568 : }
569 13580 : continue;
570 : }
571 53495 : else if (peek_input () == '/'
572 53495 : && (peek_input (1) == '!' || peek_input (1) == '/'))
573 : {
574 : /* single line doc comment, inner or outer. */
575 52274 : bool is_inner = peek_input (1) == '!';
576 52274 : skip_input (1);
577 52274 : current_column += 3;
578 :
579 52274 : std::string str;
580 52274 : str.reserve (32);
581 52274 : current_char = peek_input ();
582 1908546 : while (current_char != '\n')
583 : {
584 1804047 : skip_input ();
585 1804047 : if (current_char == '\r')
586 : {
587 51 : Codepoint next_char = peek_input ();
588 51 : if (next_char == '\n')
589 : {
590 49 : current_char = '\n';
591 49 : break;
592 : }
593 2 : rust_error_at (
594 : loc, "Isolated CR %<\\r%> not allowed in doc comment");
595 2 : current_char = next_char;
596 2 : continue;
597 2 : }
598 1803996 : if (current_char.is_eof ())
599 : {
600 0 : rust_error_at (
601 : loc, ErrorCode::E0758,
602 : "unexpected EOF while looking for end of comment");
603 0 : break;
604 : }
605 1803996 : str += current_char;
606 1803996 : current_char = peek_input ();
607 : }
608 52274 : skip_input ();
609 52274 : current_line++;
610 52274 : current_column = 1;
611 : // tell line_table that new line starts
612 52274 : start_line (current_line, max_column_hint);
613 :
614 52274 : str.shrink_to_fit ();
615 :
616 52274 : loc += str.size () - 1;
617 52274 : if (is_inner)
618 2576 : return Token::make_inner_doc_comment (loc, std::move (str));
619 : else
620 49698 : return Token::make_outer_doc_comment (loc, std::move (str));
621 52274 : }
622 1221 : else if (peek_input () == '*' && peek_input (1) == '*'
623 1306 : && peek_input (2) == '/')
624 : {
625 : /**/
626 14 : skip_input (2);
627 14 : current_column += 4;
628 14 : continue;
629 : }
630 1207 : else if (peek_input () == '*' && peek_input (1) == '*'
631 1278 : && peek_input (2) == '*' && peek_input (3) == '/')
632 : {
633 : /***/
634 14 : skip_input (3);
635 14 : current_column += 5;
636 14 : continue;
637 : }
638 1193 : else if ((peek_input () == '*' && peek_input (1) != '!'
639 959 : && peek_input (1) != '*')
640 1324 : || (peek_input () == '*' && peek_input (1) == '*'
641 57 : && peek_input (2) == '*'))
642 : {
643 : // one /* or three /***
644 : // block comment
645 : // (but not an inner or outer doc comment)
646 916 : skip_input ();
647 916 : current_column += 2;
648 :
649 916 : int level = 1;
650 50752 : while (level > 0)
651 : {
652 49837 : current_char = peek_input ();
653 :
654 49837 : if (current_char.is_eof ())
655 : {
656 1 : rust_error_at (
657 : loc, ErrorCode::E0758,
658 : "unexpected EOF while looking for end of comment");
659 1 : break;
660 : }
661 :
662 : // if /* found
663 49836 : if (current_char == '/' && peek_input (1) == '*')
664 : {
665 : // skip /* characters
666 49 : skip_input (1);
667 :
668 49 : current_column += 2;
669 :
670 49 : level += 1;
671 49 : continue;
672 : }
673 :
674 : // ignore until */ is found
675 49787 : if (current_char == '*' && peek_input (1) == '/')
676 : {
677 : // skip */ characters
678 964 : skip_input (1);
679 :
680 964 : current_column += 2;
681 :
682 964 : level -= 1;
683 964 : continue;
684 : }
685 :
686 48823 : if (current_char == '\n')
687 : {
688 732 : skip_input ();
689 732 : current_line++;
690 732 : current_column = 1;
691 : // tell line_table that new line starts
692 732 : start_line (current_line, max_column_hint);
693 732 : continue;
694 : }
695 :
696 48091 : skip_input ();
697 48091 : current_column++;
698 : }
699 :
700 : // refresh new token
701 916 : continue;
702 916 : }
703 277 : else if (peek_input () == '*'
704 277 : && (peek_input (1) == '!' || peek_input (1) == '*'))
705 : {
706 : // block doc comment, inner /*! or outer /**
707 117 : bool is_inner = peek_input (1) == '!';
708 117 : skip_input (1);
709 117 : current_column += 3;
710 :
711 117 : std::string str;
712 117 : str.reserve (96);
713 :
714 117 : int level = 1;
715 117 : while (level > 0)
716 : {
717 8069 : current_char = peek_input ();
718 :
719 8069 : if (current_char.is_eof ())
720 : {
721 0 : rust_error_at (
722 : loc, ErrorCode::E0758,
723 : "unexpected EOF while looking for end of comment");
724 0 : break;
725 : }
726 :
727 : // if /* found
728 8069 : if (current_char == '/' && peek_input (1) == '*')
729 : {
730 : // skip /* characters
731 84 : skip_input (1);
732 84 : current_column += 2;
733 :
734 84 : level += 1;
735 84 : str += "/*";
736 84 : continue;
737 : }
738 :
739 : // ignore until */ is found
740 7985 : if (current_char == '*' && peek_input (1) == '/')
741 : {
742 : // skip */ characters
743 201 : skip_input (1);
744 201 : current_column += 2;
745 :
746 201 : level -= 1;
747 201 : if (level > 0)
748 84 : str += "*/";
749 201 : continue;
750 : }
751 :
752 7784 : if (current_char == '\r' && peek_input (1) != '\n')
753 2 : rust_error_at (
754 : loc, "Isolated CR %<\\r%> not allowed in doc comment");
755 :
756 7784 : if (current_char == '\n')
757 : {
758 113 : skip_input ();
759 113 : current_line++;
760 113 : current_column = 1;
761 : // tell line_table that new line starts
762 113 : start_line (current_line, max_column_hint);
763 113 : str += '\n';
764 113 : continue;
765 : }
766 :
767 7671 : str += current_char;
768 7671 : skip_input ();
769 7671 : current_column++;
770 : }
771 :
772 117 : str.shrink_to_fit ();
773 :
774 117 : loc += str.size () - 1;
775 117 : if (is_inner)
776 74 : return Token::make_inner_doc_comment (loc, std::move (str));
777 : else
778 43 : return Token::make_outer_doc_comment (loc, std::move (str));
779 117 : }
780 : else
781 : {
782 : // division
783 160 : current_column++;
784 160 : return Token::make (DIV, loc);
785 : }
786 136 : case '%':
787 136 : if (peek_input () == '=')
788 : {
789 : // modulo-assign
790 9 : skip_input ();
791 9 : current_column += 2;
792 9 : loc += 1;
793 :
794 9 : return Token::make (PERCENT_EQ, loc);
795 : }
796 : else
797 : {
798 : // modulo
799 127 : current_column++;
800 127 : return Token::make (PERCENT, loc);
801 : }
802 189 : case '^':
803 189 : if (peek_input () == '=')
804 : {
805 : // xor-assign?
806 103 : skip_input ();
807 103 : current_column += 2;
808 103 : loc += 1;
809 :
810 103 : return Token::make (CARET_EQ, loc);
811 : }
812 : else
813 : {
814 : // xor?
815 86 : current_column++;
816 86 : return Token::make (CARET, loc);
817 : }
818 15935 : case '<':
819 15935 : if (peek_input () == '<')
820 : {
821 1230 : if (peek_input (1) == '=')
822 : {
823 : // left-shift assign
824 15 : skip_input (1);
825 15 : current_column += 3;
826 15 : loc += 2;
827 :
828 15 : return Token::make (LEFT_SHIFT_EQ, loc);
829 : }
830 : else
831 : {
832 : // left-shift
833 1215 : skip_input ();
834 1215 : current_column += 2;
835 1215 : loc += 1;
836 :
837 1215 : return Token::make (LEFT_SHIFT, loc);
838 : }
839 : }
840 14705 : else if (peek_input () == '=')
841 : {
842 : // smaller than or equal to
843 353 : skip_input ();
844 353 : current_column += 2;
845 353 : loc += 1;
846 :
847 353 : return Token::make (LESS_OR_EQUAL, loc);
848 : }
849 : else
850 : {
851 : // smaller than
852 14352 : current_column++;
853 14352 : return Token::make (LEFT_ANGLE, loc);
854 : }
855 14650 : break;
856 14650 : case '>':
857 14650 : if (peek_input () == '>')
858 : {
859 567 : if (peek_input (1) == '=')
860 : {
861 : // right-shift-assign
862 11 : skip_input (1);
863 11 : current_column += 3;
864 11 : loc += 2;
865 :
866 11 : return Token::make (RIGHT_SHIFT_EQ, loc);
867 : }
868 : else
869 : {
870 : // right-shift
871 556 : skip_input ();
872 556 : current_column += 2;
873 556 : loc += 1;
874 :
875 556 : return Token::make (RIGHT_SHIFT, loc);
876 : }
877 : }
878 14083 : else if (peek_input () == '=')
879 : {
880 : // larger than or equal to
881 383 : skip_input ();
882 383 : current_column += 2;
883 383 : loc += 1;
884 :
885 383 : return Token::make (GREATER_OR_EQUAL, loc);
886 : }
887 : else
888 : {
889 : // larger than
890 13700 : current_column++;
891 13700 : return Token::make (RIGHT_ANGLE, loc);
892 : }
893 50971 : case ':':
894 50971 : if (peek_input () == ':')
895 : {
896 : // scope resolution ::
897 16777 : skip_input ();
898 16777 : current_column += 2;
899 16777 : loc += 1;
900 :
901 16777 : return Token::make (SCOPE_RESOLUTION, loc);
902 : }
903 : else
904 : {
905 : // single colon :
906 34194 : current_column++;
907 34194 : return Token::make (COLON, loc);
908 : }
909 22533 : case '!':
910 : // no special handling for macros in lexer?
911 22533 : if (peek_input () == '=')
912 : {
913 : // not equal boolean operator
914 1127 : skip_input ();
915 1127 : current_column += 2;
916 1127 : loc += 1;
917 :
918 1127 : return Token::make (NOT_EQUAL, loc);
919 : }
920 : else
921 : {
922 : // not equal unary operator
923 21406 : current_column++;
924 :
925 21406 : return Token::make (EXCLAM, loc);
926 : }
927 867 : case '?':
928 867 : current_column++;
929 867 : return Token::make (QUESTION_MARK, loc);
930 37369 : case '#':
931 37369 : current_column++;
932 37369 : return Token::make (HASH, loc);
933 44421 : case '[':
934 44421 : current_column++;
935 44421 : return Token::make (LEFT_SQUARE, loc);
936 44414 : case ']':
937 44414 : current_column++;
938 44414 : return Token::make (RIGHT_SQUARE, loc);
939 51106 : case '{':
940 51106 : current_column++;
941 51106 : return Token::make (LEFT_CURLY, loc);
942 51055 : case '}':
943 51055 : current_column++;
944 51055 : return Token::make (RIGHT_CURLY, loc);
945 46 : case '@':
946 46 : current_column++;
947 46 : return Token::make (PATTERN_BIND, loc);
948 11720 : case '$':
949 11720 : current_column++;
950 11720 : return Token::make (DOLLAR_SIGN, loc);
951 0 : case '~':
952 0 : current_column++;
953 0 : return Token::make (TILDE, loc);
954 0 : case '\\':
955 0 : current_column++;
956 0 : return Token::make (BACKSLASH, loc);
957 0 : case '`':
958 0 : current_column++;
959 0 : return Token::make (BACKTICK, loc);
960 1556 : case '|':
961 1556 : if (peek_input () == '=')
962 : {
963 : // bitwise or-assign?
964 42 : skip_input ();
965 42 : current_column += 2;
966 42 : loc += 1;
967 :
968 42 : return Token::make (PIPE_EQ, loc);
969 : }
970 1514 : else if (peek_input () == '|')
971 : {
972 : // logical or
973 150 : skip_input ();
974 150 : current_column += 2;
975 150 : loc += 1;
976 :
977 150 : return Token::make (OR, loc);
978 : }
979 : else
980 : {
981 : // bitwise or
982 1364 : current_column++;
983 :
984 1364 : return Token::make (PIPE, loc);
985 : }
986 15608 : case '&':
987 15608 : if (peek_input () == '=')
988 : {
989 : // bitwise and-assign?
990 22 : skip_input ();
991 22 : current_column += 2;
992 22 : loc += 1;
993 :
994 22 : return Token::make (AMP_EQ, loc);
995 : }
996 15586 : else if (peek_input () == '&')
997 : {
998 : // logical and
999 512 : skip_input ();
1000 512 : current_column += 2;
1001 512 : loc += 1;
1002 :
1003 512 : return Token::make (LOGICAL_AND, loc);
1004 : }
1005 : else
1006 : {
1007 : // bitwise and/reference
1008 15074 : current_column++;
1009 :
1010 15074 : return Token::make (AMP, loc);
1011 : }
1012 15913 : case '.':
1013 15913 : if (peek_input () == '.')
1014 : {
1015 1788 : if (peek_input (1) == '.')
1016 : {
1017 : // ellipsis
1018 887 : skip_input (1);
1019 887 : current_column += 3;
1020 887 : loc += 2;
1021 :
1022 887 : return Token::make (ELLIPSIS, loc);
1023 : }
1024 901 : else if (peek_input (1) == '=')
1025 : {
1026 : // ..=
1027 121 : skip_input (1);
1028 121 : current_column += 3;
1029 121 : loc += 2;
1030 :
1031 121 : return Token::make (DOT_DOT_EQ, loc);
1032 : }
1033 : else
1034 : {
1035 : // ..
1036 780 : skip_input ();
1037 780 : current_column += 2;
1038 780 : loc += 1;
1039 :
1040 780 : return Token::make (DOT_DOT, loc);
1041 : }
1042 : }
1043 : else /*if (!ISDIGIT (peek_input ()))*/
1044 : {
1045 : // single dot .
1046 : // Only if followed by a non-number - otherwise is float
1047 : // nope, float cannot start with '.'.
1048 14125 : current_column++;
1049 14125 : return Token::make (DOT, loc);
1050 : }
1051 2025276 : }
1052 : // TODO: special handling of _ in the lexer? instead of being identifier
1053 :
1054 : // byte character, byte string and raw byte string literals
1055 629369 : if (current_char == 'b')
1056 : {
1057 18187 : if (peek_input () == '\'')
1058 204 : return parse_byte_char (loc);
1059 17983 : else if (peek_input () == '"')
1060 121 : return parse_byte_string (loc);
1061 17862 : else if (peek_input () == 'r'
1062 17862 : && (peek_input (1) == '#' || peek_input (1) == '"'))
1063 32 : return parse_raw_byte_string (loc);
1064 : }
1065 :
1066 : // C-style strings
1067 611182 : else if (current_char == 'c' && peek_input () == '"')
1068 15 : return parse_c_string (loc);
1069 :
1070 : // raw identifiers and raw strings
1071 628997 : if (current_char == 'r')
1072 : {
1073 12337 : Codepoint peek = peek_input ();
1074 12337 : Codepoint peek1 = peek_input (1);
1075 :
1076 : // TODO (tamaron) parse Unicode ident
1077 12337 : if (peek == '#' && is_identifier_start (peek1.value))
1078 : {
1079 85 : TokenPtr raw_ident_ptr = parse_raw_identifier (loc);
1080 85 : if (raw_ident_ptr != nullptr)
1081 84 : return raw_ident_ptr;
1082 : else
1083 1 : continue; /* input got parsed, it just wasn't valid. An error
1084 : was produced. */
1085 85 : }
1086 : else
1087 : {
1088 12252 : TokenPtr maybe_raw_string_ptr = maybe_parse_raw_string (loc);
1089 12252 : if (maybe_raw_string_ptr != nullptr)
1090 31 : return maybe_raw_string_ptr;
1091 12252 : }
1092 : }
1093 :
1094 : // find identifiers and keywords.
1095 628881 : if (is_identifier_start (current_char.value))
1096 516333 : return parse_identifier_or_keyword (loc);
1097 :
1098 : // int and float literals
1099 112548 : if (ISDIGIT (current_char.value))
1100 : { // _ not allowed as first char
1101 68998 : if (current_char == '0'
1102 68998 : && is_non_decimal_int_literal_separator (peek_input ().value))
1103 : {
1104 : // handle binary, octal, hex literals
1105 4639 : TokenPtr non_dec_int_lit_ptr
1106 4639 : = parse_non_decimal_int_literals (loc);
1107 4639 : if (non_dec_int_lit_ptr != nullptr)
1108 4639 : return non_dec_int_lit_ptr;
1109 4639 : }
1110 : else
1111 : {
1112 : // handle decimals (integer or float)
1113 64359 : TokenPtr decimal_or_float_ptr = parse_decimal_int_or_float (loc);
1114 64359 : if (decimal_or_float_ptr != nullptr)
1115 64359 : return decimal_or_float_ptr;
1116 64359 : }
1117 : }
1118 :
1119 : // string literals
1120 43550 : if (current_char == '"')
1121 29038 : return parse_string (loc);
1122 :
1123 : // char literals and lifetime names
1124 14512 : if (current_char == '\'')
1125 : {
1126 14512 : TokenPtr char_or_lifetime_ptr = parse_char_or_lifetime (loc);
1127 14512 : if (char_or_lifetime_ptr != nullptr)
1128 14512 : return char_or_lifetime_ptr;
1129 14512 : }
1130 :
1131 : // DEBUG: check for specific character problems:
1132 0 : if (current_char == '0')
1133 0 : rust_debug ("'0' uncaught before unexpected character");
1134 0 : else if (current_char == ']')
1135 0 : rust_debug ("']' uncaught before unexpected character");
1136 : else if (current_char == 0x5d)
1137 : rust_debug ("whatever 0x5d is (not '0' or ']') uncaught before "
1138 : "unexpected character");
1139 :
1140 : // didn't match anything so error
1141 0 : rust_error_at (loc, "unexpected character %<%x%>", current_char.value);
1142 0 : current_column++;
1143 : }
1144 : }
1145 :
1146 : // Parses in a suffix
1147 : std::pair<std::string, int>
1148 58688 : Lexer::parse_in_suffix ()
1149 : {
1150 58688 : std::string suffix;
1151 :
1152 58688 : int additional_length_offset = 0;
1153 :
1154 : // get suffix
1155 122123 : while (ISALPHA (current_char.value) || ISDIGIT (current_char.value)
1156 124937 : || current_char == '_')
1157 : {
1158 7560 : additional_length_offset++;
1159 :
1160 7560 : suffix += current_char;
1161 7560 : skip_input ();
1162 7560 : current_char = peek_input ();
1163 : }
1164 :
1165 58688 : return std::make_pair (std::move (suffix), additional_length_offset);
1166 58688 : }
1167 :
1168 : // Parses in the exponent part (if any) of a float literal.
1169 : std::pair<std::string, int>
1170 3544 : Lexer::parse_in_exponent_part ()
1171 : {
1172 3544 : int additional_length_offset = 0;
1173 3544 : std::string str;
1174 3544 : if (current_char == 'E' || current_char == 'e')
1175 : {
1176 : // add exponent to string as strtod works with it
1177 91 : str += current_char;
1178 91 : skip_input ();
1179 91 : current_char = peek_input ();
1180 :
1181 91 : additional_length_offset++;
1182 :
1183 : // special - and + handling
1184 91 : if (current_char == '-' || current_char == '+')
1185 : {
1186 26 : str += current_char;
1187 :
1188 26 : skip_input ();
1189 26 : current_char = peek_input ();
1190 :
1191 26 : additional_length_offset++;
1192 : }
1193 :
1194 : // parse another decimal number for exponent
1195 91 : auto str_length = parse_in_decimal ();
1196 91 : str += std::get<0> (str_length);
1197 91 : additional_length_offset += std::get<1> (str_length);
1198 91 : }
1199 7088 : return std::make_pair (str, additional_length_offset);
1200 3544 : }
1201 :
1202 : // Parses a decimal integer.
1203 : std::tuple<std::string, int, bool>
1204 67937 : Lexer::parse_in_decimal ()
1205 : {
1206 : /* A pure decimal contains only digits. */
1207 67937 : bool pure_decimal = true;
1208 67937 : int additional_length_offset = 0;
1209 67937 : std::string str;
1210 104273 : while (ISDIGIT (current_char.value) || current_char.value == '_')
1211 : {
1212 36336 : if (current_char == '_')
1213 : {
1214 222 : pure_decimal = false;
1215 : }
1216 36336 : additional_length_offset++;
1217 :
1218 36336 : str += current_char;
1219 36336 : skip_input ();
1220 36336 : current_char = peek_input ();
1221 : }
1222 135874 : return std::make_tuple (str, additional_length_offset, pure_decimal);
1223 67937 : }
1224 :
1225 : /* Parses escapes (and string continues) in "byte" strings and characters. Does
1226 : * not support unicode. */
1227 : std::tuple<char, int, bool>
1228 90 : Lexer::parse_escape (char opening_char)
1229 : {
1230 90 : int additional_length_offset = 0;
1231 90 : char output_char = 0;
1232 :
1233 : // skip to actual letter
1234 90 : skip_input ();
1235 90 : current_char = peek_input ();
1236 90 : additional_length_offset++;
1237 :
1238 90 : switch (current_char.value)
1239 : {
1240 22 : case 'x':
1241 22 : {
1242 22 : auto hex_escape_pair = parse_partial_hex_escape ();
1243 22 : long hexLong = hex_escape_pair.first;
1244 22 : additional_length_offset += hex_escape_pair.second;
1245 :
1246 22 : if (hexLong > 255 || hexLong < 0)
1247 0 : rust_error_at (
1248 : get_current_location (),
1249 : "byte \\x escape %<\\x%x%> out of range - allows up to %<\\xFF%>",
1250 : static_cast<unsigned int> (hexLong));
1251 : /* TODO: restore capital for escape output - gcc pretty-printer doesn't
1252 : * support %X directly */
1253 22 : char hexChar = static_cast<char> (hexLong);
1254 :
1255 22 : output_char = hexChar;
1256 : }
1257 22 : break;
1258 : case 'n':
1259 : output_char = '\n';
1260 : break;
1261 3 : case 'r':
1262 3 : output_char = '\r';
1263 3 : break;
1264 3 : case 't':
1265 3 : output_char = '\t';
1266 3 : break;
1267 17 : case '\\':
1268 17 : output_char = '\\';
1269 17 : break;
1270 11 : case '0':
1271 11 : output_char = '\0';
1272 11 : break;
1273 17 : case '\'':
1274 17 : output_char = '\'';
1275 17 : break;
1276 1 : case '"':
1277 1 : output_char = '"';
1278 1 : break;
1279 2 : case 'u':
1280 3 : rust_error_at (get_current_location (),
1281 : "cannot have a unicode escape \\u in a byte %s",
1282 : opening_char == '\'' ? "character" : "string");
1283 : // Try to parse it anyway, just to skip it
1284 2 : parse_partial_unicode_escape ();
1285 2 : return std::make_tuple (output_char, additional_length_offset, false);
1286 4 : case '\r':
1287 4 : case '\n':
1288 : // string continue
1289 4 : return std::make_tuple (0, parse_partial_string_continue (), true);
1290 1 : default:
1291 1 : rust_error_at (get_current_location (),
1292 : "unknown escape sequence %<\\%s%>",
1293 1 : current_char.as_string ().c_str ());
1294 : // returns false if no parsing could be done
1295 : // return false;
1296 1 : return std::make_tuple (output_char, additional_length_offset, false);
1297 83 : break;
1298 : }
1299 : // all non-special cases (string continue) should skip their used char
1300 83 : skip_input ();
1301 83 : current_char = peek_input ();
1302 83 : additional_length_offset++;
1303 :
1304 : // returns true if parsing was successful
1305 : // return true;
1306 83 : return std::make_tuple (output_char, additional_length_offset, false);
1307 : }
1308 :
1309 : /* Parses an escape (or string continue) in a string or character. Supports
1310 : * unicode escapes. */
1311 : std::tuple<Codepoint, int, bool>
1312 14587 : Lexer::parse_utf8_escape ()
1313 : {
1314 14587 : Codepoint output_char;
1315 14587 : int additional_length_offset = 0;
1316 :
1317 : // skip to actual letter
1318 14587 : skip_input ();
1319 14587 : current_char = peek_input ();
1320 14587 : additional_length_offset++;
1321 :
1322 14587 : switch (current_char.value)
1323 : {
1324 30 : case 'x':
1325 30 : {
1326 30 : auto hex_escape_pair = parse_partial_hex_escape ();
1327 30 : long hexLong = hex_escape_pair.first;
1328 30 : additional_length_offset += hex_escape_pair.second;
1329 :
1330 30 : if (hexLong > 127 || hexLong < 0)
1331 4 : rust_error_at (
1332 : get_current_location (),
1333 : "ascii \\x escape %<\\x%x%> out of range - allows up to %<\\x7F%>",
1334 : static_cast<unsigned int> (hexLong));
1335 : /* TODO: restore capital for escape output - gcc pretty-printer doesn't
1336 : * support %X directly */
1337 30 : char hexChar = static_cast<char> (hexLong);
1338 :
1339 30 : output_char = hexChar;
1340 : }
1341 30 : break;
1342 : case 'n':
1343 : output_char = '\n';
1344 : break;
1345 3 : case 'r':
1346 3 : output_char = '\r';
1347 3 : break;
1348 13 : case 't':
1349 13 : output_char = '\t';
1350 13 : break;
1351 8 : case '\\':
1352 8 : output_char = '\\';
1353 8 : break;
1354 1472 : case '0':
1355 1472 : output_char = '\0';
1356 1472 : break;
1357 5 : case '\'':
1358 5 : output_char = '\'';
1359 5 : break;
1360 59 : case '"':
1361 59 : output_char = '"';
1362 59 : break;
1363 11427 : case 'u':
1364 11427 : {
1365 11427 : auto unicode_escape_pair = parse_partial_unicode_escape ();
1366 11427 : output_char = unicode_escape_pair.first;
1367 11427 : additional_length_offset += unicode_escape_pair.second;
1368 :
1369 11427 : return std::make_tuple (output_char, additional_length_offset, false);
1370 : }
1371 178 : break;
1372 178 : case '\r':
1373 178 : case '\n':
1374 : // string continue
1375 178 : return std::make_tuple (0, parse_partial_string_continue (), true);
1376 1 : default:
1377 1 : rust_error_at (get_current_location (),
1378 : "unknown escape sequence %<\\%s%>",
1379 1 : current_char.as_string ().c_str ());
1380 : // returns false if no parsing could be done
1381 : // return false;
1382 1 : return std::make_tuple (output_char, additional_length_offset, false);
1383 2981 : break;
1384 : }
1385 : /* all non-special cases (unicode, string continue) should skip their used
1386 : * char */
1387 2981 : skip_input ();
1388 2981 : current_char = peek_input ();
1389 2981 : additional_length_offset++;
1390 :
1391 : // returns true if parsing was successful
1392 : // return true;
1393 2981 : return std::make_tuple (output_char, additional_length_offset, false);
1394 : }
1395 :
1396 : // Parses the body of a string continue that has been found in an escape.
1397 : int
1398 182 : Lexer::parse_partial_string_continue ()
1399 : {
1400 182 : int additional_length_offset = 1;
1401 :
1402 : // string continue
1403 : // TODO use utf-8 codepoint to skip whitespaces
1404 3952 : while (is_whitespace (current_char.value))
1405 : {
1406 3770 : if (current_char == '\n')
1407 : {
1408 182 : current_line++;
1409 182 : current_column = 1;
1410 : // tell line_table that new line starts
1411 182 : start_line (current_line, max_column_hint);
1412 :
1413 : // reset "length"
1414 182 : additional_length_offset = 1;
1415 :
1416 : // get next char
1417 182 : skip_input ();
1418 182 : current_char = peek_input ();
1419 :
1420 182 : continue;
1421 : }
1422 :
1423 3588 : skip_input ();
1424 3588 : current_char = peek_input ();
1425 3588 : additional_length_offset++;
1426 : }
1427 :
1428 182 : return additional_length_offset;
1429 : }
1430 :
1431 : /* Parses the body of a '\x' escape. Note that it does not check that the number
1432 : * is valid and smaller than 255. */
1433 : std::pair<long, int>
1434 52 : Lexer::parse_partial_hex_escape ()
1435 : {
1436 : // hex char string (null-terminated)
1437 52 : char hexNum[3] = {0, 0, 0};
1438 :
1439 : // first hex char
1440 52 : current_char = peek_input (1);
1441 52 : int additional_length_offset = 1;
1442 :
1443 52 : if (!is_x_digit (current_char.value))
1444 : {
1445 4 : rust_error_at (get_current_location (),
1446 : "invalid character %<\\x%s%> in \\x sequence",
1447 4 : current_char.as_string ().c_str ());
1448 4 : return std::make_pair (0, 0);
1449 : }
1450 48 : hexNum[0] = current_char.value;
1451 :
1452 : // second hex char
1453 48 : skip_input ();
1454 48 : current_char = peek_input (1);
1455 48 : additional_length_offset++;
1456 :
1457 48 : if (!is_x_digit (current_char.value))
1458 : {
1459 2 : rust_error_at (get_current_location (),
1460 2 : "invalid character %<\\x%c%s%> in \\x sequence", hexNum[0],
1461 2 : current_char.as_string ().c_str ());
1462 2 : return std::make_pair (0, 1);
1463 : }
1464 46 : skip_input ();
1465 46 : hexNum[1] = current_char.value;
1466 :
1467 46 : long hexLong = std::strtol (hexNum, nullptr, 16);
1468 :
1469 46 : return std::make_pair (hexLong, additional_length_offset);
1470 : }
1471 :
1472 : // Parses the body of a unicode escape.
1473 : std::pair<Codepoint, int>
1474 11429 : Lexer::parse_partial_unicode_escape ()
1475 : {
1476 11429 : skip_input ();
1477 11429 : current_char = peek_input ();
1478 11429 : int additional_length_offset = 0;
1479 :
1480 11429 : if (current_char != '{')
1481 : {
1482 2 : rust_error_at (get_current_location (),
1483 : "unicode escape should start with %<{%>");
1484 : /* Skip what should probaby have been between brackets. */
1485 10 : while (is_x_digit (current_char.value) || current_char == '_')
1486 : {
1487 6 : skip_input ();
1488 6 : current_char = peek_input ();
1489 6 : additional_length_offset++;
1490 : }
1491 2 : return std::make_pair (Codepoint (0), additional_length_offset);
1492 : }
1493 :
1494 11427 : skip_input ();
1495 11427 : current_char = peek_input ();
1496 11427 : additional_length_offset++;
1497 :
1498 11427 : if (current_char == '_')
1499 : {
1500 2 : rust_error_at (get_current_location (),
1501 : "unicode escape cannot start with %<_%>");
1502 2 : skip_input ();
1503 2 : current_char = peek_input ();
1504 2 : additional_length_offset++;
1505 : // fallthrough and try to parse the rest anyway
1506 : }
1507 :
1508 : // parse unicode escape - 1-6 hex digits
1509 11427 : std::string num_str;
1510 11427 : num_str.reserve (6);
1511 :
1512 : // loop through to add entire hex number to string
1513 50471 : while (is_x_digit (current_char.value) || current_char.value == '_')
1514 : {
1515 27617 : if (current_char == '_')
1516 : {
1517 : // don't add _ to number
1518 24 : skip_input ();
1519 24 : current_char = peek_input ();
1520 :
1521 24 : additional_length_offset++;
1522 :
1523 24 : continue;
1524 : }
1525 :
1526 27593 : additional_length_offset++;
1527 :
1528 : // add raw hex numbers
1529 27593 : num_str += current_char;
1530 :
1531 27593 : skip_input ();
1532 27593 : current_char = peek_input ();
1533 : }
1534 :
1535 11427 : if (current_char == '}')
1536 : {
1537 11425 : skip_input ();
1538 11425 : current_char = peek_input ();
1539 11425 : additional_length_offset++;
1540 : }
1541 : else
1542 : {
1543 : // actually an error, but allow propagation anyway Assume that
1544 : // wrong bracketm whitespace or single/double quotes are wrong
1545 : // termination, otherwise it is a wrong character, then skip to the actual
1546 : // terminator.
1547 : // TODO use utf-8 codepoint to skip whitespaces
1548 2 : if (current_char == '{' || is_whitespace (current_char.value)
1549 4 : || current_char == '\'' || current_char == '"')
1550 : {
1551 0 : rust_error_at (get_current_location (),
1552 : "expected terminating %<}%> in unicode escape");
1553 0 : return std::make_pair (Codepoint (0), additional_length_offset);
1554 : }
1555 : else
1556 : {
1557 2 : rust_error_at (get_current_location (),
1558 : "invalid character %qs in unicode escape",
1559 2 : current_char.as_string ().c_str ());
1560 : // TODO use utf-8 codepoint to skip whitespaces
1561 8 : while (current_char != '}' && current_char != '{'
1562 6 : && !is_whitespace (current_char.value) && current_char != '\''
1563 14 : && current_char != '"')
1564 : {
1565 6 : skip_input ();
1566 6 : current_char = peek_input ();
1567 6 : additional_length_offset++;
1568 : }
1569 : // Consume the actual closing bracket if found
1570 2 : if (current_char == '}')
1571 : {
1572 2 : skip_input ();
1573 2 : current_char = peek_input ();
1574 2 : additional_length_offset++;
1575 : }
1576 2 : return std::make_pair (Codepoint (0), additional_length_offset);
1577 : }
1578 : }
1579 :
1580 : // ensure 1-6 hex characters
1581 11425 : if (num_str.length () > 6 || num_str.length () < 1)
1582 : {
1583 4 : rust_error_at (get_current_location (),
1584 : "unicode escape should be between 1 and 6 hex "
1585 : "characters; it is %lu",
1586 4 : (unsigned long) num_str.length ());
1587 : // return false;
1588 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1589 : }
1590 :
1591 11421 : unsigned long hex_num = std::strtoul (num_str.c_str (), nullptr, 16);
1592 :
1593 11421 : if (hex_num > 0xd7ff && hex_num < 0xe000)
1594 : {
1595 4 : rust_error_at (
1596 : get_current_location (),
1597 : "unicode escape cannot be a surrogate value (D800 to DFFF)");
1598 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1599 : }
1600 :
1601 11417 : if (hex_num > 0x10ffff)
1602 : {
1603 4 : rust_error_at (get_current_location (),
1604 : "unicode escape cannot be larger than 10FFFF");
1605 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1606 : }
1607 :
1608 : // return true;
1609 11413 : return std::make_pair (Codepoint (static_cast<uint32_t> (hex_num)),
1610 : additional_length_offset);
1611 11427 : }
1612 :
1613 : // Parses a byte character.
1614 : TokenPtr
1615 204 : Lexer::parse_byte_char (location_t loc)
1616 : {
1617 204 : skip_input ();
1618 204 : current_column++;
1619 : // make current char the next character
1620 204 : current_char = peek_input ();
1621 :
1622 204 : int length = 1;
1623 :
1624 : // char to save
1625 204 : Codepoint byte_char = 0;
1626 :
1627 : // detect escapes
1628 204 : if (current_char == '\\')
1629 : {
1630 54 : auto escape_length_pair = parse_escape ('\'');
1631 54 : byte_char = std::get<0> (escape_length_pair);
1632 54 : length += std::get<1> (escape_length_pair);
1633 :
1634 54 : current_char = peek_input ();
1635 :
1636 54 : if (current_char != '\'')
1637 : {
1638 0 : rust_error_at (get_current_location (), "unclosed %<byte char%>");
1639 : }
1640 :
1641 54 : skip_input ();
1642 54 : current_char = peek_input ();
1643 54 : length++; // go to next char
1644 : }
1645 150 : else if (current_char != '\'')
1646 : {
1647 : // otherwise, get character from direct input character
1648 150 : byte_char = current_char;
1649 :
1650 150 : if (!byte_char.is_ascii ())
1651 : {
1652 2 : rust_error_at (get_current_location (),
1653 : "non-ASCII character in %<byte char%>");
1654 : }
1655 :
1656 150 : skip_input ();
1657 150 : current_char = peek_input ();
1658 150 : length++;
1659 :
1660 150 : if (current_char != '\'')
1661 : {
1662 0 : rust_error_at (get_current_location (), "unclosed %<byte char%>");
1663 : }
1664 :
1665 150 : skip_input ();
1666 150 : current_char = peek_input ();
1667 150 : length++; // go to next char
1668 : }
1669 : else
1670 : {
1671 0 : rust_error_at (get_current_location (),
1672 : "no character inside %<%> for %<byte char%>");
1673 : }
1674 :
1675 204 : current_column += length;
1676 :
1677 204 : loc += length - 1;
1678 204 : return Token::make_byte_char (loc, byte_char.value);
1679 : }
1680 :
1681 : // Parses a byte string.
1682 : TokenPtr
1683 121 : Lexer::parse_byte_string (location_t loc)
1684 : {
1685 : // byte string
1686 :
1687 : // skip quote character
1688 121 : skip_input ();
1689 121 : current_column++;
1690 :
1691 121 : std::string str;
1692 121 : str.reserve (16); // some sensible default
1693 :
1694 121 : current_char = peek_input ();
1695 :
1696 121 : const location_t string_begin_locus = get_current_location ();
1697 :
1698 1019 : while (current_char != '"' && !current_char.is_eof ())
1699 : {
1700 777 : if (current_char == '\\')
1701 : {
1702 35 : int length = 1;
1703 35 : auto escape_length_pair = parse_escape ('"');
1704 35 : char output_char = std::get<0> (escape_length_pair);
1705 :
1706 35 : if (output_char == 0 && std::get<2> (escape_length_pair))
1707 4 : length = std::get<1> (escape_length_pair) - 1;
1708 : else
1709 31 : length += std::get<1> (escape_length_pair);
1710 :
1711 35 : if (output_char != 0 || !std::get<2> (escape_length_pair))
1712 31 : str += output_char;
1713 :
1714 35 : current_column += length;
1715 :
1716 35 : continue;
1717 35 : }
1718 :
1719 742 : current_column++;
1720 742 : if (current_char.value == '\n')
1721 : {
1722 23 : current_line++;
1723 23 : current_column = 1;
1724 : // tell line_table that new line starts
1725 23 : start_line (current_line, max_column_hint);
1726 : }
1727 :
1728 742 : str += current_char;
1729 742 : skip_input ();
1730 742 : current_char = peek_input ();
1731 : }
1732 :
1733 121 : if (current_char == '"')
1734 : {
1735 114 : current_column++;
1736 :
1737 114 : skip_input ();
1738 114 : current_char = peek_input ();
1739 : }
1740 7 : else if (current_char.is_eof ())
1741 : {
1742 7 : rust_error_at (string_begin_locus, "unended byte string literal");
1743 7 : return Token::make (END_OF_FILE, get_current_location ());
1744 : }
1745 : else
1746 : {
1747 : rust_unreachable ();
1748 : }
1749 :
1750 114 : str.shrink_to_fit ();
1751 114 : loc += str.size () - 1;
1752 :
1753 114 : return Token::make_byte_string (loc, std::move (str));
1754 121 : }
1755 :
1756 : // Parses a C-style string.
1757 : TokenPtr
1758 15 : Lexer::parse_c_string (location_t loc)
1759 : {
1760 15 : skip_input ();
1761 15 : current_column++;
1762 :
1763 : // Mostly same code copied from parse_string...
1764 :
1765 15 : std::string str;
1766 15 : str.reserve (16); // some sensible default
1767 :
1768 15 : current_char = peek_input ();
1769 :
1770 15 : const location_t string_begin_locus = get_current_location ();
1771 :
1772 106 : while (current_char.value != '"' && !current_char.is_eof ())
1773 : {
1774 76 : if (current_char.value == '\\')
1775 : {
1776 1 : int length = 1;
1777 :
1778 1 : auto escape_pair = parse_escape ('"');
1779 1 : current_char = std::get<0> (escape_pair);
1780 :
1781 1 : if (current_char == Codepoint (0) && std::get<2> (escape_pair))
1782 0 : length = std::get<1> (escape_pair) - 1;
1783 : else
1784 1 : length += std::get<1> (escape_pair);
1785 :
1786 1 : if (current_char != Codepoint (0) || !std::get<2> (escape_pair))
1787 2 : str += current_char.as_string ();
1788 :
1789 1 : current_column += length;
1790 :
1791 : // FIXME: parse_escape does not update current_char correctly.
1792 1 : current_char = peek_input ();
1793 1 : continue;
1794 1 : }
1795 :
1796 75 : current_column++;
1797 75 : if (current_char.value == '\n')
1798 : {
1799 0 : current_line++;
1800 0 : current_column = 1;
1801 : // tell line_table that new line starts
1802 0 : start_line (current_line, max_column_hint);
1803 : }
1804 :
1805 75 : str += current_char;
1806 75 : skip_input ();
1807 75 : current_char = peek_input ();
1808 : }
1809 :
1810 15 : if (current_char.value == '"')
1811 : {
1812 15 : current_column++;
1813 :
1814 15 : skip_input ();
1815 15 : current_char = peek_input ();
1816 : }
1817 0 : else if (current_char.is_eof ())
1818 : {
1819 0 : rust_error_at (string_begin_locus, "unended C string literal");
1820 0 : return Token::make (END_OF_FILE, get_current_location ());
1821 : }
1822 : else
1823 : {
1824 : rust_unreachable ();
1825 : }
1826 :
1827 15 : str.shrink_to_fit ();
1828 :
1829 15 : return Token::make_c_string (loc, std::move (str));
1830 15 : }
1831 :
1832 : // Parses a raw byte string.
1833 : TokenPtr
1834 32 : Lexer::parse_raw_byte_string (location_t loc)
1835 : {
1836 : // raw byte string literals
1837 32 : std::string str;
1838 32 : str.reserve (16); // some sensible default
1839 :
1840 32 : int length = 1;
1841 32 : int hash_count = 0;
1842 :
1843 32 : const location_t string_begin_locus = get_current_location ();
1844 :
1845 : // get hash count at beginnning
1846 32 : skip_input ();
1847 32 : current_char = peek_input ();
1848 32 : length++;
1849 32 : current_column++;
1850 54 : while (current_char == '#')
1851 : {
1852 22 : hash_count++;
1853 22 : length++;
1854 22 : current_column++;
1855 :
1856 22 : skip_input ();
1857 22 : current_char = peek_input ();
1858 : }
1859 :
1860 32 : if (current_char != '"')
1861 : {
1862 0 : rust_error_at (get_current_location (),
1863 : "raw byte string has no opening %<\"%>");
1864 : }
1865 :
1866 32 : skip_input ();
1867 32 : current_char = peek_input ();
1868 32 : length++;
1869 32 : current_column++;
1870 :
1871 330 : while (true)
1872 : {
1873 181 : if (current_char == '"')
1874 : {
1875 51 : bool enough_hashes = true;
1876 :
1877 51 : for (int i = 0; i < hash_count; i++)
1878 : {
1879 26 : if (peek_input (i + 1) != '#')
1880 : {
1881 : enough_hashes = false;
1882 : break;
1883 : }
1884 : }
1885 :
1886 35 : if (enough_hashes)
1887 : {
1888 : // skip enough input and peek enough input
1889 25 : skip_input (hash_count);
1890 25 : current_char = peek_input ();
1891 25 : length += hash_count + 1;
1892 25 : current_column += hash_count + 1;
1893 25 : break;
1894 : }
1895 : }
1896 146 : else if (current_char.is_eof ())
1897 : {
1898 7 : rust_error_at (string_begin_locus, "unended raw byte string literal");
1899 7 : return Token::make (END_OF_FILE, get_current_location ());
1900 : }
1901 139 : else if (current_char.value > 127)
1902 : {
1903 1 : rust_error_at (get_current_location (),
1904 : "character %qs in raw byte string out of range",
1905 1 : current_char.as_string ().c_str ());
1906 1 : current_char = 0;
1907 : }
1908 :
1909 149 : length++;
1910 149 : current_column++;
1911 149 : if (current_char == '\n')
1912 : {
1913 22 : current_line++;
1914 22 : current_column = 1;
1915 22 : start_line (current_line, max_column_hint);
1916 : }
1917 :
1918 149 : str += current_char;
1919 149 : skip_input ();
1920 149 : current_char = peek_input ();
1921 149 : }
1922 :
1923 25 : loc += length - 1;
1924 :
1925 25 : str.shrink_to_fit ();
1926 :
1927 25 : return Token::make_byte_string (loc, std::move (str));
1928 32 : }
1929 :
1930 : // Parses a raw identifier.
1931 : TokenPtr
1932 85 : Lexer::parse_raw_identifier (location_t loc)
1933 : {
1934 : // raw identifier
1935 85 : std::string str;
1936 85 : str.reserve (16); // default
1937 :
1938 85 : skip_input ();
1939 85 : current_char = peek_input ();
1940 :
1941 85 : current_column += 2;
1942 :
1943 85 : bool first_is_underscore = current_char == '_';
1944 :
1945 85 : int length = 0;
1946 85 : current_char = peek_input ();
1947 : // loop through entire name
1948 495 : while (is_identifier_continue (current_char.value))
1949 : {
1950 325 : length++;
1951 :
1952 325 : str += current_char;
1953 325 : skip_input ();
1954 325 : current_char = peek_input ();
1955 : }
1956 :
1957 85 : current_column += length;
1958 :
1959 85 : rust_debug ("raw ident: %s", str.c_str ());
1960 :
1961 : // if just a single underscore, not an identifier
1962 85 : if (first_is_underscore && length == 1)
1963 1 : rust_error_at (get_current_location (),
1964 : "%<_%> is not a valid raw identifier");
1965 :
1966 85 : using namespace Rust::Values;
1967 85 : std::set<std::string> invalid{
1968 85 : Keywords::CRATE, Keywords::EXTERN_KW, Keywords::SELF,
1969 85 : Keywords::SUPER, Keywords::SELF_ALIAS,
1970 510 : };
1971 :
1972 85 : if (invalid.find (str) != invalid.end ())
1973 : {
1974 1 : rust_error_at (get_current_location (),
1975 : "%qs is a forbidden raw identifier", str.c_str ());
1976 :
1977 1 : return nullptr;
1978 : }
1979 : else
1980 : {
1981 84 : str.shrink_to_fit ();
1982 84 : loc += length - 1;
1983 :
1984 84 : return Token::make_identifier (loc, std::move (str));
1985 : }
1986 85 : }
1987 :
1988 : // skip broken string input (unterminated strings)
1989 : void
1990 0 : Lexer::skip_broken_string_input (Codepoint current_char)
1991 : {
1992 0 : while (current_char != '"' && !current_char.is_eof ())
1993 : {
1994 0 : if (current_char == '\n')
1995 : {
1996 0 : current_line++;
1997 0 : current_column = 1;
1998 : }
1999 : else
2000 : {
2001 0 : current_column++;
2002 : }
2003 0 : skip_input ();
2004 0 : current_char = peek_input ();
2005 : }
2006 0 : if (current_char == '"')
2007 : {
2008 0 : current_column++;
2009 :
2010 0 : skip_input ();
2011 0 : current_char = peek_input ();
2012 : }
2013 0 : rust_debug ("skipped to %d:%d due to bad quotes", current_line,
2014 : current_column);
2015 0 : }
2016 :
2017 : // Parses a string.
2018 : TokenPtr
2019 29038 : Lexer::parse_string (location_t loc)
2020 : {
2021 29038 : std::string str;
2022 29038 : str.reserve (16); // some sensible default
2023 :
2024 29038 : current_char = peek_input ();
2025 :
2026 29038 : const location_t string_begin_locus = get_current_location ();
2027 :
2028 : // FIXME: This fails if the input ends. How do we check for EOF?
2029 352678 : while (current_char.value != '"' && !current_char.is_eof ())
2030 : {
2031 294602 : if (current_char.value == '\\')
2032 : {
2033 3143 : int length = 1;
2034 :
2035 : // parse escape
2036 3143 : auto utf8_escape_pair = parse_utf8_escape ();
2037 3143 : current_char = std::get<0> (utf8_escape_pair);
2038 :
2039 3143 : if (current_char == Codepoint (0) && std::get<2> (utf8_escape_pair))
2040 178 : length = std::get<1> (utf8_escape_pair) - 1;
2041 : else
2042 2965 : length += std::get<1> (utf8_escape_pair);
2043 :
2044 3143 : if (current_char != Codepoint (0) || !std::get<2> (utf8_escape_pair))
2045 5930 : str += current_char.as_string ();
2046 :
2047 3143 : current_column += length;
2048 :
2049 : // FIXME: should remove this but can't.
2050 : // `parse_utf8_escape` does not update `current_char` correctly.
2051 3143 : current_char = peek_input ();
2052 3143 : continue;
2053 3143 : }
2054 :
2055 291459 : current_column++;
2056 291459 : if (current_char.value == '\n')
2057 : {
2058 2775 : current_line++;
2059 2775 : current_column = 1;
2060 : // tell line_table that new line starts
2061 2775 : start_line (current_line, max_column_hint);
2062 : }
2063 :
2064 291459 : str += current_char;
2065 291459 : skip_input ();
2066 291459 : current_char = peek_input ();
2067 : }
2068 :
2069 29038 : if (current_char.value == '"')
2070 : {
2071 29024 : current_column++;
2072 :
2073 29024 : skip_input ();
2074 29024 : current_char = peek_input ();
2075 : }
2076 14 : else if (current_char.is_eof ())
2077 : {
2078 14 : rust_error_at (string_begin_locus, "unended string literal");
2079 14 : return Token::make (END_OF_FILE, get_current_location ());
2080 : }
2081 : else
2082 : {
2083 : rust_unreachable ();
2084 : }
2085 :
2086 29024 : str.shrink_to_fit ();
2087 :
2088 29024 : return Token::make_string (loc, std::move (str));
2089 29038 : }
2090 :
2091 : // Parses an identifier or keyword.
2092 : TokenPtr
2093 516333 : Lexer::parse_identifier_or_keyword (location_t loc)
2094 : {
2095 516333 : std::string str;
2096 516333 : str.reserve (16); // default
2097 1032666 : str += current_char.as_string ();
2098 :
2099 516333 : bool first_is_underscore = current_char == '_';
2100 :
2101 516333 : int length = 1;
2102 516333 : current_char = peek_input ();
2103 :
2104 : // loop through entire name
2105 2948990 : while (is_identifier_continue (current_char.value))
2106 : {
2107 1916324 : auto s = current_char.as_string ();
2108 1916324 : length++;
2109 :
2110 3832648 : str += current_char.as_string ();
2111 1916324 : skip_input ();
2112 1916324 : current_char = peek_input ();
2113 1916324 : }
2114 :
2115 516333 : current_column += length;
2116 :
2117 : // if just a single underscore, not an identifier
2118 516333 : if (first_is_underscore && length == 1)
2119 2079 : return Token::make (UNDERSCORE, loc);
2120 :
2121 514254 : str.shrink_to_fit ();
2122 :
2123 514254 : loc += length - 1;
2124 :
2125 514254 : TokenId keyword = classify_keyword (str);
2126 514254 : if (keyword == IDENTIFIER)
2127 365705 : return Token::make_identifier (loc, std::move (str));
2128 : else
2129 148549 : return Token::make (keyword, loc);
2130 516333 : }
2131 :
2132 : // Possibly returns a raw string token if it exists - otherwise returns null.
2133 : TokenPtr
2134 12252 : Lexer::maybe_parse_raw_string (location_t loc)
2135 : {
2136 12252 : int peek_index = 0;
2137 12267 : while (peek_input (peek_index) == '#')
2138 15 : peek_index++;
2139 :
2140 12252 : if (peek_input (peek_index) == '"')
2141 31 : return parse_raw_string (loc, peek_index);
2142 : else
2143 12221 : return nullptr;
2144 : }
2145 :
2146 : // Returns a raw string token.
2147 : TokenPtr
2148 31 : Lexer::parse_raw_string (location_t loc, int initial_hash_count)
2149 : {
2150 : // raw string literals
2151 31 : std::string str;
2152 31 : str.reserve (16); // some sensible default
2153 :
2154 31 : int length = 1 + initial_hash_count;
2155 31 : current_column += length;
2156 :
2157 31 : const location_t string_begin_locus = get_current_location ();
2158 :
2159 31 : if (initial_hash_count > 0)
2160 13 : skip_input (initial_hash_count - 1);
2161 :
2162 31 : current_char = peek_input ();
2163 :
2164 31 : if (current_char != '"')
2165 0 : rust_error_at (get_current_location (), "raw string has no opening %<\"%>");
2166 :
2167 31 : length++;
2168 31 : current_column++;
2169 31 : skip_input ();
2170 31 : current_char = peek_input ();
2171 :
2172 2501 : while (true)
2173 : {
2174 1266 : if (current_char.value == '"')
2175 : {
2176 50 : bool enough_hashes = true;
2177 :
2178 50 : for (int i = 0; i < initial_hash_count; i++)
2179 : {
2180 19 : if (peek_input (i + 1) != '#')
2181 : {
2182 : enough_hashes = false;
2183 : break;
2184 : }
2185 : }
2186 :
2187 34 : if (enough_hashes)
2188 : {
2189 : // skip enough input and peek enough input
2190 31 : skip_input (initial_hash_count);
2191 31 : current_char = peek_input ();
2192 31 : length += initial_hash_count + 1;
2193 31 : current_column += initial_hash_count + 1;
2194 31 : break;
2195 : }
2196 : }
2197 1232 : else if (current_char.is_eof ())
2198 : {
2199 0 : rust_error_at (string_begin_locus, "unended raw string literal");
2200 0 : return Token::make (END_OF_FILE, get_current_location ());
2201 : }
2202 :
2203 1235 : length++;
2204 1235 : current_column++;
2205 1235 : if (current_char == '\n')
2206 : {
2207 31 : current_line++;
2208 31 : current_column = 1;
2209 31 : start_line (current_line, max_column_hint);
2210 : }
2211 :
2212 2470 : str += current_char.as_string ();
2213 1235 : skip_input ();
2214 1235 : current_char = peek_input ();
2215 1235 : }
2216 :
2217 31 : loc += length - 1;
2218 :
2219 31 : str.shrink_to_fit ();
2220 :
2221 31 : return Token::make_raw_string (loc, std::move (str));
2222 31 : }
2223 :
2224 : template <typename IsDigitFunc>
2225 : TokenPtr
2226 4639 : Lexer::parse_non_decimal_int_literal (location_t loc, IsDigitFunc is_digit_func,
2227 : IntegerLiteralBase base)
2228 : {
2229 4639 : std::string raw_str = "0";
2230 4639 : raw_str += current_char; // x, o, b
2231 4639 : skip_input ();
2232 :
2233 4639 : int length = 2;
2234 4639 : bool has_valid_digit = false;
2235 :
2236 4639 : current_char = peek_input ();
2237 :
2238 : // loop through to add entire number to string
2239 38150 : while (true)
2240 : {
2241 42789 : if (is_digit_func (current_char.value))
2242 : {
2243 : has_valid_digit = true;
2244 : }
2245 5474 : else if (current_char != '_')
2246 : {
2247 : break;
2248 : }
2249 38150 : length++;
2250 :
2251 38150 : raw_str += current_char;
2252 38150 : skip_input ();
2253 38150 : current_char = peek_input ();
2254 : }
2255 :
2256 4639 : int suffix_start = raw_str.length ();
2257 :
2258 : // parse in suffix if it exists
2259 4639 : auto suffix_pair = parse_in_suffix ();
2260 4639 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2261 4639 : raw_str += suffix_pair.first;
2262 4639 : length += suffix_pair.second;
2263 :
2264 4639 : current_column += length;
2265 :
2266 4639 : if (!has_valid_digit)
2267 : {
2268 5 : rust_error_at (loc, ErrorCode::E0768, "no valid digits found for number");
2269 : }
2270 :
2271 4639 : loc += length - 1;
2272 :
2273 4639 : return Token::make_int (loc, std::move (raw_str), suffix_start, base,
2274 4639 : type_hint);
2275 4639 : }
2276 :
2277 : // Parses a hex, binary or octal int literal.
2278 : TokenPtr
2279 4639 : Lexer::parse_non_decimal_int_literals (location_t loc)
2280 : {
2281 4639 : current_char = peek_input ();
2282 :
2283 4639 : if (current_char == 'x')
2284 : {
2285 : // hex (integer only)
2286 3334 : return parse_non_decimal_int_literal (loc, is_x_digit,
2287 3334 : IntegerLiteralBase::Hex);
2288 : }
2289 1305 : else if (current_char == 'o')
2290 : {
2291 : // octal (integer only)
2292 19 : return parse_non_decimal_int_literal (loc, is_octal_digit,
2293 19 : IntegerLiteralBase::Octal);
2294 : }
2295 1286 : else if (current_char == 'b')
2296 : {
2297 : // binary (integer only)
2298 1286 : return parse_non_decimal_int_literal (loc, is_bin_digit,
2299 1286 : IntegerLiteralBase::Binary);
2300 : }
2301 : else
2302 : {
2303 0 : return nullptr;
2304 : }
2305 : }
2306 :
2307 : // Parses a decimal-based int literal or float literal.
2308 : TokenPtr
2309 64359 : Lexer::parse_decimal_int_or_float (location_t loc)
2310 : {
2311 64359 : std::string str;
2312 64359 : str.reserve (16); // some sensible default
2313 64359 : str += current_char;
2314 :
2315 64359 : int length = 1;
2316 64359 : bool first_zero = current_char == '0';
2317 :
2318 64359 : current_char = peek_input ();
2319 :
2320 : // parse initial decimal integer (or first integer part of float) literal
2321 64359 : auto initial_decimal = parse_in_decimal ();
2322 64359 : str += std::get<0> (initial_decimal);
2323 64359 : length += std::get<1> (initial_decimal);
2324 :
2325 : // detect float literal
2326 : //
2327 : // Note:
2328 : //
2329 : // We should not use is_float_digit () for this verification but instead
2330 : // directly ISDIGIT because rust does not support non digit values right after
2331 : // a dot.
2332 : // The following value is not legal in rust:
2333 : // let a = 3.e1;
2334 : // A `0` should be put between the dot and the exponent to be valid
2335 : // (eg. 3.0e1).
2336 64359 : if (current_char == '.' && ISDIGIT (peek_input (1).value))
2337 : {
2338 : // float with a '.', parse another decimal into it
2339 :
2340 : // add . to str
2341 3487 : str += current_char;
2342 3487 : skip_input ();
2343 3487 : current_char = peek_input ();
2344 3487 : length++;
2345 :
2346 : // parse another decimal number for float
2347 3487 : auto second_decimal = parse_in_decimal ();
2348 3487 : str += std::get<0> (second_decimal);
2349 3487 : length += std::get<1> (second_decimal);
2350 :
2351 : // parse in exponent part if it exists
2352 3487 : auto exponent_pair = parse_in_exponent_part ();
2353 3487 : str += exponent_pair.first;
2354 3487 : length += exponent_pair.second;
2355 :
2356 3487 : int suffix_start = str.length ();
2357 :
2358 : // parse in type suffix if it exists
2359 3487 : auto suffix_pair = parse_in_suffix ();
2360 3487 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2361 3487 : str += suffix_pair.first;
2362 3487 : length += suffix_pair.second;
2363 :
2364 3487 : current_column += length;
2365 :
2366 3487 : loc += length - 1;
2367 :
2368 3487 : str.shrink_to_fit ();
2369 3487 : return Token::make_float (loc, std::move (str), suffix_start, type_hint);
2370 3487 : }
2371 60872 : else if (current_char == '.'
2372 60872 : && check_valid_float_dot_end (peek_input (1).value))
2373 : {
2374 : // float that is just an integer with a terminating '.' character
2375 :
2376 : // add . to str
2377 10310 : str += current_char;
2378 10310 : skip_input ();
2379 10310 : current_char = peek_input ();
2380 10310 : length++;
2381 :
2382 : // type hint not allowed
2383 :
2384 10310 : current_column += length;
2385 :
2386 10310 : loc += length - 1;
2387 :
2388 10310 : str.shrink_to_fit ();
2389 10310 : auto suffix_start = str.length ();
2390 10310 : return Token::make_float (loc, std::move (str), suffix_start,
2391 10310 : CORETYPE_UNKNOWN);
2392 : }
2393 50562 : else if (current_char == 'E' || current_char == 'e')
2394 : {
2395 : // exponent float with no '.' character
2396 :
2397 : // parse exponent part
2398 57 : auto exponent_pair = parse_in_exponent_part ();
2399 57 : str += exponent_pair.first;
2400 57 : length += exponent_pair.second;
2401 :
2402 57 : int suffix_start = str.length ();
2403 :
2404 : // parse in type suffix if it exists
2405 57 : auto suffix_pair = parse_in_suffix ();
2406 57 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2407 57 : str += suffix_pair.first;
2408 57 : length += suffix_pair.second;
2409 :
2410 57 : current_column += length;
2411 :
2412 57 : loc += length - 1;
2413 :
2414 57 : str.shrink_to_fit ();
2415 57 : return Token::make_float (loc, std::move (str), suffix_start, type_hint);
2416 57 : }
2417 : else
2418 : {
2419 : // is an integer
2420 :
2421 50505 : int suffix_start = str.length ();
2422 :
2423 : // parse in type suffix if it exists
2424 50505 : auto suffix_pair = parse_in_suffix ();
2425 50505 : str += suffix_pair.first;
2426 :
2427 50505 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2428 :
2429 : /* A "real" pure decimal doesn't have a suffix and no zero prefix. */
2430 50505 : bool pure_decimal = std::get<2> (initial_decimal);
2431 50401 : if (pure_decimal && (!first_zero || suffix_start == 1)
2432 100904 : && suffix_pair.first.empty ())
2433 : type_hint = CORETYPE_PURE_DECIMAL;
2434 :
2435 50505 : length += suffix_pair.second;
2436 :
2437 50505 : current_column += length;
2438 :
2439 50505 : loc += length - 1;
2440 :
2441 50505 : str.shrink_to_fit ();
2442 50505 : return Token::make_int (loc, std::move (str), suffix_start,
2443 50505 : IntegerLiteralBase::Decimal, type_hint);
2444 50505 : }
2445 64359 : }
2446 :
2447 : TokenPtr
2448 14512 : Lexer::parse_char_or_lifetime (location_t loc)
2449 : {
2450 14512 : int length = 1;
2451 :
2452 14512 : current_char = peek_input ();
2453 14512 : if (current_char.is_eof ())
2454 0 : return nullptr;
2455 :
2456 : // parse escaped char literal
2457 14512 : if (current_char.value == '\\')
2458 : {
2459 : // parse escape
2460 11444 : auto utf8_escape_pair = parse_utf8_escape ();
2461 11444 : Codepoint escaped_char = std::get<0> (utf8_escape_pair);
2462 11444 : length += std::get<1> (utf8_escape_pair);
2463 :
2464 11444 : if (peek_input ().value != '\'')
2465 : {
2466 0 : rust_error_at (get_current_location (), "unended character literal");
2467 : }
2468 : else
2469 : {
2470 11444 : skip_input ();
2471 11444 : current_char = peek_input ();
2472 11444 : length++;
2473 : }
2474 :
2475 11444 : current_column += length;
2476 :
2477 11444 : loc += length - 1;
2478 :
2479 11444 : return Token::make_char (loc, escaped_char);
2480 : }
2481 : else
2482 : {
2483 3068 : skip_input ();
2484 :
2485 3068 : if (peek_input ().value == '\'')
2486 : {
2487 : // parse non-escaped char literal
2488 413 : Codepoint non_escaped_char = current_char;
2489 :
2490 : // skip the ' character
2491 413 : skip_input ();
2492 413 : current_char = peek_input ();
2493 :
2494 : // TODO fix due to different widths of utf-8 chars?
2495 413 : current_column += 3;
2496 :
2497 413 : loc += 2;
2498 :
2499 413 : return Token::make_char (loc, non_escaped_char);
2500 : }
2501 2655 : else if (is_identifier_start (current_char.value))
2502 : {
2503 : // parse lifetime name
2504 2655 : std::string str;
2505 5310 : str += current_char.as_string ();
2506 2655 : length++;
2507 :
2508 2655 : current_char = peek_input ();
2509 6394 : while (is_identifier_continue (current_char.value))
2510 : {
2511 2168 : str += current_char.as_string ();
2512 1084 : skip_input ();
2513 1084 : current_char = peek_input ();
2514 1084 : length++;
2515 : }
2516 :
2517 2655 : current_column += length;
2518 :
2519 2655 : loc += length - 1;
2520 :
2521 : // TODO some keywords cannot be used for a lifetime label #2306
2522 : // https://doc.rust-lang.org/reference/tokens.html
2523 :
2524 2655 : str.shrink_to_fit ();
2525 2655 : return Token::make_lifetime (loc, std::move (str));
2526 2655 : }
2527 : else
2528 : {
2529 0 : rust_error_at (
2530 : get_current_location (),
2531 : "expected %' after character constant in character literal");
2532 0 : return nullptr;
2533 : }
2534 : }
2535 : }
2536 :
2537 : void
2538 313 : Lexer::split_current_token (TokenId new_left, TokenId new_right)
2539 : {
2540 : /* TODO: assert that this TokenId is a "simple token" like punctuation and not
2541 : * like "IDENTIFIER"? */
2542 313 : location_t current_loc = peek_token ()->get_locus ();
2543 313 : TokenPtr new_left_tok = Token::make (new_left, current_loc);
2544 313 : TokenPtr new_right_tok = Token::make (new_right, current_loc + 1);
2545 :
2546 313 : token_queue.replace_current_value (std::move (new_left_tok));
2547 313 : token_queue.insert (1, std::move (new_right_tok));
2548 313 : }
2549 :
2550 : void
2551 2 : Lexer::split_current_token (std::vector<TokenPtr> new_tokens)
2552 : {
2553 2 : rust_assert (new_tokens.size () > 0);
2554 4 : token_queue.replace_current_value (new_tokens[0]);
2555 :
2556 5 : for (size_t i = 1; i < new_tokens.size (); i++)
2557 : {
2558 6 : token_queue.insert (i, new_tokens[i]);
2559 : }
2560 2 : }
2561 :
2562 : void
2563 321803 : Lexer::start_line (int current_line, int current_column)
2564 : {
2565 321803 : if (line_map)
2566 321803 : linemap_line_start (line_table, current_line, current_column);
2567 321803 : }
2568 :
2569 : } // namespace Rust
2570 :
2571 : #if CHECKING_P
2572 :
2573 : namespace selftest {
2574 :
2575 : // Checks if `src` has the same contents as the given characters
2576 : static void
2577 6 : assert_source_content (Rust::InputSource &src,
2578 : const std::vector<uint32_t> &expected)
2579 : {
2580 6 : Rust::Codepoint src_char = src.next ();
2581 41 : for (auto expected_char : expected)
2582 : {
2583 : // Make sure that `src` is not shorter than `expected`
2584 35 : ASSERT_FALSE (src_char.is_eof ());
2585 : // Checks skipped character is expeceted one.
2586 35 : ASSERT_EQ (src_char.value, expected_char);
2587 35 : src_char = src.next ();
2588 : }
2589 : // Checks if `src` and `chars` has the same length.
2590 6 : ASSERT_TRUE (src_char.is_eof ());
2591 6 : }
2592 :
2593 : static void
2594 4 : test_buffer_input_source (std::string str,
2595 : const std::vector<uint32_t> &expected)
2596 : {
2597 4 : Rust::BufferInputSource source (str, 0);
2598 4 : assert_source_content (source, expected);
2599 4 : }
2600 :
2601 : static void
2602 2 : test_file_input_source (std::string str, const std::vector<uint32_t> &expected)
2603 : {
2604 2 : FILE *tmpf = tmpfile ();
2605 : // Moves to the first character
2606 2 : fputs (str.c_str (), tmpf);
2607 2 : std::rewind (tmpf);
2608 2 : Rust::FileInputSource source (tmpf);
2609 2 : assert_source_content (source, expected);
2610 2 : }
2611 :
2612 : void
2613 1 : rust_input_source_test ()
2614 : {
2615 : // ASCII
2616 1 : std::string src = (const char *) u8"_abcde\tXYZ\v\f";
2617 1 : std::vector<uint32_t> expected = {u'_', u'a', u'b', u'c', u'd', u'e',
2618 1 : u'\t', u'X', u'Y', u'Z', u'\v', u'\f'};
2619 2 : test_buffer_input_source (src, expected);
2620 :
2621 : // BOM
2622 1 : src = (const char *) u8"\xef\xbb\xbfOK";
2623 1 : expected = {u'O', u'K'};
2624 2 : test_buffer_input_source (src, expected);
2625 :
2626 : // Russian
2627 1 : src = (const char *) u8"приве́т";
2628 1 : expected = {u'п',
2629 : u'р',
2630 : u'и',
2631 : u'в',
2632 : 0x0435 /* CYRILLIC SMALL LETTER IE е */,
2633 : 0x301 /* COMBINING ACUTE ACCENT ́ */,
2634 1 : u'т'};
2635 2 : test_buffer_input_source (src, expected);
2636 :
2637 1 : src = (const char *) u8"❤️🦀";
2638 1 : expected = {0x2764 /* HEAVY BLACK HEART */,
2639 1 : 0xfe0f /* VARIATION SELECTOR-16 */, U'🦀'};
2640 2 : test_buffer_input_source (src, expected);
2641 :
2642 1 : src = (const char *) u8"こんにちは";
2643 1 : expected = {u'こ', u'ん', u'に', u'ち', u'は'};
2644 2 : test_file_input_source (src, expected);
2645 :
2646 1 : src = (const char *) u8"👮♂👩⚕";
2647 1 : expected
2648 : = {0x1f46e /* POLICE OFFICER */, 0x200d /* ZERO WIDTH JOINER */,
2649 : 0x2642 /* MALE SIGN */, 0x1f469 /* WOMAN */,
2650 1 : 0x200d /* ZERO WIDTH JOINER */, 0x2695 /* STAFF OF AESCULAPIUS */};
2651 2 : test_file_input_source (src, expected);
2652 1 : }
2653 :
2654 : } // namespace selftest
2655 :
2656 : #endif // CHECKING_P
|
