Line data Source code
1 : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
2 :
3 : // This file is part of GCC.
4 :
5 : // GCC is free software; you can redistribute it and/or modify it under
6 : // the terms of the GNU General Public License as published by the Free
7 : // Software Foundation; either version 3, or (at your option) any later
8 : // version.
9 :
10 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 : // for more details.
14 :
15 : // You should have received a copy of the GNU General Public License
16 : // along with GCC; see the file COPYING3. If not see
17 : // <http://www.gnu.org/licenses/>.
18 :
19 : #include "rust-codepoint.h"
20 : #include "rust-system.h"
21 : #include "rust-lex.h"
22 : #include "rust-diagnostics.h"
23 : #include "rust-linemap.h"
24 : #include "rust-edition.h"
25 : #include "safe-ctype.h"
26 : #include "cpplib.h"
27 : #include "rust-keyword-values.h"
28 :
29 : namespace Rust {
30 : // TODO: move to separate compilation unit?
31 : // overload += for uint32_t to allow 32-bit encoded utf-8 to be added
32 : std::string &
33 48394467 : operator+= (std::string &str, Codepoint char32)
34 : {
35 48394467 : if (char32.value < 0x80)
36 : {
37 48344071 : str += static_cast<char> (char32.value);
38 : }
39 50396 : else if (char32.value < (0x1F + 1) << (1 * 6))
40 : {
41 17907 : str += static_cast<char> (0xC0 | ((char32.value >> 6) & 0x1F));
42 17907 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
43 : }
44 32489 : else if (char32.value < (0x0F + 1) << (2 * 6))
45 : {
46 24804 : str += static_cast<char> (0xE0 | ((char32.value >> 12) & 0x0F));
47 24804 : str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
48 24804 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
49 : }
50 7685 : else if (char32.value < (0x07 + 1) << (3 * 6))
51 : {
52 7681 : str += static_cast<char> (0xF0 | ((char32.value >> 18) & 0x07));
53 7681 : str += static_cast<char> (0x80 | ((char32.value >> 12) & 0x3F));
54 7681 : str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
55 7681 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
56 : }
57 : else
58 : {
59 4 : rust_debug ("Invalid unicode codepoint found: '%u' ", char32.value);
60 : }
61 48394467 : return str;
62 : }
63 :
64 : std::string
65 31670036 : Codepoint::as_string ()
66 : {
67 31670036 : std::string str;
68 :
69 : // str += Codepoint (value);
70 31670036 : str += *this;
71 :
72 31670036 : return str;
73 : }
74 :
75 : /* Includes all allowable float digits EXCEPT _ and . as that needs lookahead
76 : * for handling. */
77 : bool
78 0 : is_float_digit (uint32_t number)
79 : {
80 0 : return ISDIGIT (number) || number == 'E' || number == 'e';
81 : }
82 :
83 : /* Basically ISXDIGIT from safe-ctype but may change if Rust's encoding or
84 : * whatever is different */
85 : bool
86 483224 : is_x_digit (uint32_t number)
87 : {
88 483224 : return ISXDIGIT (number);
89 : }
90 :
91 : bool
92 87 : is_octal_digit (uint32_t number)
93 : {
94 87 : return number >= '0' && number <= '7';
95 : }
96 :
97 : bool
98 153740 : is_bin_digit (uint32_t number)
99 : {
100 153740 : return number == '0' || number == '1';
101 : }
102 :
103 : bool
104 84900 : check_valid_float_dot_end (uint32_t character)
105 : {
106 84900 : return character != '.' && character != '_' && !ISALPHA (character);
107 : }
108 :
109 : bool
110 33046 : is_whitespace (uint32_t character)
111 : {
112 : // https://doc.rust-lang.org/reference/whitespace.html
113 33046 : switch (character)
114 : {
115 : case '\t':
116 : case '\n':
117 : case '\v':
118 : case '\f':
119 : case '\r':
120 : case ' ':
121 : case 0x0085: // next line
122 : case 0x200e: // left-to-right mark
123 : case 0x200f: // right-to-left mark
124 : case 0x2028: // line separator
125 : case 0x2029: // paragraph separator
126 : return true;
127 5100 : default:
128 5100 : return false;
129 : }
130 : }
131 :
132 : bool
133 106787 : is_non_decimal_int_literal_separator (uint32_t character)
134 : {
135 106787 : return character == 'x' || character == 'o' || character == 'b';
136 : }
137 :
138 : bool
139 2850210 : is_identifier_start (uint32_t codepoint)
140 : {
141 2850210 : return (cpp_check_xid_property (codepoint) & CPP_XID_START)
142 2850210 : || codepoint == '_';
143 : }
144 :
145 : bool
146 11437634 : is_identifier_continue (uint32_t codepoint)
147 : {
148 11437634 : return cpp_check_xid_property (codepoint) & CPP_XID_CONTINUE;
149 : }
150 :
151 164 : Lexer::Lexer (const std::string &input, Linemap *linemap)
152 164 : : input (RAIIFile::create_error ()), current_line (1), current_column (1),
153 164 : line_map (linemap), dump_lex_out ({}),
154 164 : raw_input_source (new BufferInputSource (input, 0)),
155 164 : input_queue{*raw_input_source}, token_queue (TokenSource (this))
156 164 : {}
157 :
158 7108 : Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap,
159 : tl::optional<std::ofstream &> dump_lex_opt)
160 7108 : : input (std::move (file_input)), current_line (1), current_column (1),
161 7108 : line_map (linemap), dump_lex_out (dump_lex_opt),
162 14216 : raw_input_source (new FileInputSource (input.get_raw ())),
163 14216 : input_queue{*raw_input_source}, token_queue (TokenSource (this))
164 : {
165 : // inform line_table that file is being entered and is in line 1
166 : // TODO: filename needs to live ~forever, is there a better way than xstrdup?
167 : // ^ (as long as the linemap does)
168 7108 : if (linemap)
169 7108 : line_map->start_file (xstrdup (filename), current_line);
170 7108 : }
171 :
172 7249 : Lexer::~Lexer ()
173 : {
174 : /* ok apparently stop (which is equivalent of original code in destructor) is
175 : * meant to be called after all files have finished parsing, for cleanup. On
176 : * the other hand, actual code that it calls to leave a certain line map is
177 : * mentioned in GCC docs as being useful for "just leaving an included header"
178 : * and stuff like that, so this line mapping functionality may need fixing.
179 : * FIXME: find out whether this occurs. */
180 :
181 : // line_map->stop();
182 7249 : }
183 :
184 : bool
185 5410 : Lexer::input_source_is_valid_utf8 ()
186 : {
187 5410 : return raw_input_source->is_valid ();
188 : }
189 :
190 : location_t
191 16137398 : Lexer::get_current_location ()
192 : {
193 16137398 : if (line_map)
194 16137169 : return linemap_position_for_column (line_table, current_column);
195 : else
196 : // If we have no linemap, we're lexing something without proper locations
197 : return UNDEF_LOCATION;
198 : }
199 :
200 : Codepoint
201 54802437 : Lexer::peek_input (int n)
202 : {
203 54802437 : return input_queue.peek (n);
204 : }
205 :
206 : Codepoint
207 51693125 : Lexer::peek_input ()
208 : {
209 51693125 : return peek_input (0);
210 : }
211 :
212 : void
213 45789773 : Lexer::skip_input (int n)
214 : {
215 45789773 : input_queue.skip (n);
216 45789773 : }
217 :
218 : void
219 45411005 : Lexer::skip_input ()
220 : {
221 45411005 : skip_input (0);
222 45411005 : }
223 :
224 : void
225 7028607 : Lexer::skip_token (int n)
226 : {
227 : // dump tokens if dump-lex option is enabled
228 7028607 : if (dump_lex_out.has_value ())
229 55 : dump_and_skip (n);
230 : else
231 7028552 : token_queue.skip (n);
232 7028607 : }
233 :
234 : void
235 55 : Lexer::dump_and_skip (int n)
236 : {
237 55 : std::ofstream &out = dump_lex_out.value ();
238 55 : bool found_eof = false;
239 55 : const_TokenPtr tok;
240 110 : for (int i = 0; i < n + 1; i++)
241 : {
242 55 : if (!found_eof)
243 : {
244 55 : tok = peek_token ();
245 55 : found_eof |= tok->get_id () == Rust::END_OF_FILE;
246 :
247 55 : location_t loc = tok->get_locus ();
248 :
249 55 : out << "<id=";
250 55 : out << tok->token_id_to_str ();
251 55 : out << (tok->should_have_str ()
252 167 : ? (std::string (", text=") + tok->get_str ()
253 167 : + std::string (", typehint=")
254 93 : + std::string (tok->get_type_hint_str ()))
255 55 : : "")
256 110 : << " ";
257 110 : out << Linemap::location_to_string (loc) << '\n';
258 : }
259 :
260 55 : token_queue.skip (0);
261 : }
262 55 : }
263 :
264 : void
265 0 : Lexer::replace_current_token (TokenPtr replacement)
266 : {
267 0 : token_queue.replace_current_value (replacement);
268 :
269 0 : rust_debug ("called 'replace_current_token' - this is deprecated");
270 0 : }
271 :
272 : /* Determines whether the string passed in is a keyword or not. If it is, it
273 : * returns the keyword name. */
274 : TokenId
275 2138174 : Lexer::classify_keyword (const std::string &str)
276 : {
277 2138174 : auto &keywords = Rust::Values::Keywords::keywords_tokens;
278 2138174 : auto keyword = keywords.find (str);
279 :
280 2138174 : if (keyword == keywords.end ())
281 : return IDENTIFIER;
282 :
283 522171 : auto id = keyword->second;
284 :
285 : // We now have the expected token ID of the reserved keyword. However, some
286 : // keywords are reserved starting in certain editions. For example, `try` is
287 : // only a reserved keyword in editions >=2018. The language might gain new
288 : // reserved keywords in the future.
289 : //
290 : // https://doc.rust-lang.org/reference/keywords.html#reserved-keywords
291 :
292 : // `try` is not a reserved keyword before 2018
293 522171 : if (get_rust_edition () == Edition::E2015 && id == TRY)
294 1 : return IDENTIFIER;
295 :
296 : return id;
297 : }
298 :
299 : TokenPtr
300 7052698 : Lexer::build_token ()
301 : {
302 : // loop to go through multiple characters to build a single token
303 15993388 : while (true)
304 : {
305 15993388 : location_t loc = get_current_location ();
306 :
307 15993388 : current_char = peek_input ();
308 15993388 : skip_input ();
309 :
310 : // detect shebang
311 : // Must be the first thing on the first line, starting with #!
312 : // But since an attribute can also start with an #! we don't count it as a
313 : // shebang line when after any whitespace or comments there is a [. If it
314 : // is a shebang line we simple drop the line. Otherwise we don't consume
315 : // any characters and fall through to the real tokenizer.
316 46100 : if (current_line == 1 && current_column == 1 && current_char == '#'
317 16039488 : && peek_input () == '!')
318 : {
319 : int n = 1;
320 3844 : while (true)
321 : {
322 3830 : Codepoint next_char = peek_input (n);
323 3830 : if (is_whitespace (next_char.value))
324 7 : n++;
325 3823 : else if ((next_char == '/' && peek_input (n + 1) == '/'
326 7 : && peek_input (n + 2) != '!'
327 7 : && peek_input (n + 2) != '/')
328 3844 : || (next_char == '/' && peek_input (n + 1) == '/'
329 0 : && peek_input (n + 2) == '/'
330 7 : && peek_input (n + 3) == '/'))
331 : {
332 : // two // or four ////
333 : // A single line comment
334 : // (but not an inner or outer doc comment)
335 7 : n += 2;
336 7 : next_char = peek_input (n);
337 119 : while (next_char != '\n' && !next_char.is_eof ())
338 : {
339 112 : n++;
340 112 : next_char = peek_input (n);
341 : }
342 7 : if (next_char == '\n')
343 7 : n++;
344 : }
345 3816 : else if (next_char == '/' && peek_input (n + 1) == '*'
346 0 : && peek_input (n + 2) == '*'
347 3816 : && peek_input (n + 3) == '/')
348 : {
349 : /**/
350 0 : n += 4;
351 : }
352 3816 : else if (next_char == '/' && peek_input (n + 1) == '*'
353 0 : && peek_input (n + 2) == '*' && peek_input (n + 3) == '*'
354 3816 : && peek_input (n + 4) == '/')
355 : {
356 : /***/
357 0 : n += 5;
358 : }
359 3816 : else if ((next_char == '/' && peek_input (n + 1) == '*'
360 0 : && peek_input (n + 2) != '*'
361 0 : && peek_input (n + 2) != '!')
362 3837 : || (next_char == '/' && peek_input (n + 1) == '*'
363 0 : && peek_input (n + 2) == '*'
364 0 : && peek_input (n + 3) == '*'))
365 : {
366 : // one /* or three /***
367 : // Start of a block comment
368 : // (but not an inner or outer doc comment)
369 0 : n += 2;
370 0 : int level = 1;
371 0 : while (level > 0)
372 : {
373 0 : if (peek_input (n).is_eof ())
374 : break;
375 0 : else if (peek_input (n) == '/'
376 0 : && peek_input (n + 1) == '*')
377 : {
378 0 : n += 2;
379 0 : level += 1;
380 : }
381 0 : else if (peek_input (n) == '*'
382 0 : && peek_input (n + 1) == '/')
383 : {
384 0 : n += 2;
385 0 : level -= 1;
386 : }
387 : else
388 0 : n++;
389 : }
390 : }
391 3816 : else if (next_char != '[')
392 : {
393 : // definitely shebang, ignore the first line
394 518 : while (current_char != '\n' && !current_char.is_eof ())
395 : {
396 490 : current_char = peek_input ();
397 490 : skip_input ();
398 : }
399 :
400 : // newline
401 28 : current_line++;
402 28 : current_column = 1;
403 : // tell line_table that new line starts
404 28 : start_line (current_line, max_column_hint);
405 28 : break;
406 : }
407 : else
408 : break; /* Definitely not a shebang line. */
409 14 : }
410 : }
411 :
412 : // return end of file token if end of file
413 15993388 : if (current_char.is_eof ())
414 7564 : return Token::make (END_OF_FILE, loc);
415 :
416 : // if not end of file, start tokenising
417 15985824 : switch (current_char.value)
418 : {
419 : /* ignore whitespace characters for tokens but continue updating
420 : * location */
421 973526 : case '\n': // newline
422 973526 : case 0x0085: // next line
423 973526 : case 0x2028: // line separator
424 973526 : case 0x2029: // paragraph separator
425 973526 : current_line++;
426 973526 : current_column = 1;
427 : // tell line_table that new line starts
428 973526 : start_line (current_line, max_column_hint);
429 973526 : continue;
430 252 : case '\r': // cr
431 : // Ignore, we expect a newline (lf) soon.
432 252 : continue;
433 7917870 : case ' ': // space
434 7917870 : current_column++;
435 7917870 : continue;
436 114 : case '\t': // horizontal tab
437 : // width of a tab is not well-defined, assume 8 spaces
438 114 : current_column += 8;
439 114 : continue;
440 28 : case '\v': // vertical tab
441 28 : case 0x000c: // form feed
442 28 : case 0x200e: // left-to-right mark
443 28 : case 0x200f: // right-to-left mark
444 : // Ignored.
445 28 : continue;
446 :
447 : // punctuation - actual tokens
448 271622 : case '=':
449 271622 : if (peek_input () == '>')
450 : {
451 : // match arm arrow
452 31696 : skip_input ();
453 31696 : current_column += 2;
454 31696 : loc += 1;
455 :
456 31696 : return Token::make (MATCH_ARROW, loc);
457 : }
458 239926 : else if (peek_input () == '=')
459 : {
460 : // equality operator
461 3383 : skip_input ();
462 3383 : current_column += 2;
463 3383 : loc += 1;
464 :
465 3383 : return Token::make (EQUAL_EQUAL, loc);
466 : }
467 : else
468 : {
469 : // assignment operator
470 236543 : current_column++;
471 236543 : return Token::make (EQUAL, loc);
472 : }
473 495234 : case '(':
474 495234 : current_column++;
475 495234 : return Token::make (LEFT_PAREN, loc);
476 91719 : case '-':
477 91719 : if (peek_input () == '>')
478 : {
479 : // return type specifier
480 61251 : skip_input ();
481 61251 : current_column += 2;
482 61251 : loc += 1;
483 :
484 61251 : return Token::make (RETURN_TYPE, loc);
485 : }
486 30468 : else if (peek_input () == '=')
487 : {
488 : // minus-assign
489 869 : skip_input ();
490 869 : current_column += 2;
491 869 : loc += 1;
492 :
493 869 : return Token::make (MINUS_EQ, loc);
494 : }
495 : else
496 : {
497 : // minus
498 29599 : current_column++;
499 29599 : return Token::make (MINUS, loc);
500 : }
501 9536 : case '+':
502 9536 : if (peek_input () == '=')
503 : {
504 : // add-assign
505 1343 : skip_input ();
506 1343 : current_column += 2;
507 1343 : loc += 1;
508 :
509 1343 : return Token::make (PLUS_EQ, loc);
510 : }
511 : else
512 : {
513 : // add
514 8193 : current_column++;
515 8193 : return Token::make (PLUS, loc);
516 : }
517 495215 : case ')':
518 495215 : current_column++;
519 495215 : return Token::make (RIGHT_PAREN, loc);
520 182374 : case ';':
521 182374 : current_column++;
522 182374 : return Token::make (SEMICOLON, loc);
523 33068 : case '*':
524 33068 : if (peek_input () == '=')
525 : {
526 : // multiplication-assign
527 63 : skip_input ();
528 63 : current_column += 2;
529 63 : loc += 1;
530 :
531 63 : return Token::make (ASTERISK_EQ, loc);
532 : }
533 : else
534 : {
535 : // multiplication
536 33005 : current_column++;
537 33005 : return Token::make (ASTERISK, loc);
538 : }
539 752268 : case ',':
540 752268 : current_column++;
541 752268 : return Token::make (COMMA, loc);
542 425088 : case '/':
543 425088 : if (peek_input () == '=')
544 : {
545 : // division-assign
546 215 : skip_input ();
547 215 : current_column += 2;
548 215 : loc += 1;
549 :
550 215 : return Token::make (DIV_EQ, loc);
551 : }
552 424873 : else if ((peek_input () == '/' && peek_input (1) != '!'
553 400647 : && peek_input (1) != '/')
554 800290 : || (peek_input () == '/' && peek_input (1) == '/'
555 400647 : && peek_input (2) == '/'))
556 : {
557 : // two // or four ////
558 : // single line comment
559 : // (but not an inner or outer doc comment)
560 47325 : skip_input ();
561 47325 : current_column += 2;
562 47325 : current_char = peek_input ();
563 :
564 : // basically ignore until line finishes
565 2476246 : while (current_char != '\n' && !current_char.is_eof ())
566 : {
567 2381596 : skip_input ();
568 2381596 : current_column++; // not used
569 2381596 : current_char = peek_input ();
570 : }
571 47325 : continue;
572 : }
573 377548 : else if (peek_input () == '/'
574 377548 : && (peek_input (1) == '!' || peek_input (1) == '/'))
575 : {
576 : /* single line doc comment, inner or outer. */
577 374816 : bool is_inner = peek_input (1) == '!';
578 374816 : skip_input (1);
579 374816 : current_column += 3;
580 :
581 374816 : std::string str;
582 374816 : str.reserve (32);
583 374816 : current_char = peek_input ();
584 14304334 : while (current_char != '\n')
585 : {
586 13554751 : skip_input ();
587 13554751 : if (current_char == '\r')
588 : {
589 51 : Codepoint next_char = peek_input ();
590 51 : if (next_char == '\n')
591 : {
592 49 : current_char = '\n';
593 49 : break;
594 : }
595 2 : rust_error_at (
596 : loc, "Isolated CR %<\\r%> not allowed in doc comment");
597 2 : current_char = next_char;
598 2 : continue;
599 2 : }
600 13554700 : if (current_char.is_eof ())
601 : {
602 0 : rust_error_at (
603 : loc, ErrorCode::E0758,
604 : "unexpected EOF while looking for end of comment");
605 0 : break;
606 : }
607 13554700 : str += current_char;
608 13554700 : current_char = peek_input ();
609 : }
610 374816 : skip_input ();
611 374816 : current_line++;
612 374816 : current_column = 1;
613 : // tell line_table that new line starts
614 374816 : start_line (current_line, max_column_hint);
615 :
616 374816 : str.shrink_to_fit ();
617 :
618 374816 : loc += str.size () - 1;
619 374816 : if (is_inner)
620 21494 : return Token::make_inner_doc_comment (loc, std::move (str));
621 : else
622 353322 : return Token::make_outer_doc_comment (loc, std::move (str));
623 374816 : }
624 2732 : else if (peek_input () == '*' && peek_input (1) == '*'
625 2817 : && peek_input (2) == '/')
626 : {
627 : /**/
628 14 : skip_input (2);
629 14 : current_column += 4;
630 14 : continue;
631 : }
632 2718 : else if (peek_input () == '*' && peek_input (1) == '*'
633 2789 : && peek_input (2) == '*' && peek_input (3) == '/')
634 : {
635 : /***/
636 14 : skip_input (3);
637 14 : current_column += 5;
638 14 : continue;
639 : }
640 2704 : else if ((peek_input () == '*' && peek_input (1) != '!'
641 1589 : && peek_input (1) != '*')
642 2842 : || (peek_input () == '*' && peek_input (1) == '*'
643 1589 : && peek_input (2) == '*'))
644 : {
645 : // one /* or three /***
646 : // block comment
647 : // (but not an inner or outer doc comment)
648 1546 : skip_input ();
649 1546 : current_column += 2;
650 :
651 1546 : int level = 1;
652 144338 : while (level > 0)
653 : {
654 142793 : current_char = peek_input ();
655 :
656 142793 : if (current_char.is_eof ())
657 : {
658 1 : rust_error_at (
659 : loc, ErrorCode::E0758,
660 : "unexpected EOF while looking for end of comment");
661 1 : break;
662 : }
663 :
664 : // if /* found
665 142792 : if (current_char == '/' && peek_input (1) == '*')
666 : {
667 : // skip /* characters
668 49 : skip_input (1);
669 :
670 49 : current_column += 2;
671 :
672 49 : level += 1;
673 49 : continue;
674 : }
675 :
676 : // ignore until */ is found
677 142743 : if (current_char == '*' && peek_input (1) == '/')
678 : {
679 : // skip */ characters
680 1594 : skip_input (1);
681 :
682 1594 : current_column += 2;
683 :
684 1594 : level -= 1;
685 1594 : continue;
686 : }
687 :
688 141149 : if (current_char == '\n')
689 : {
690 2981 : skip_input ();
691 2981 : current_line++;
692 2981 : current_column = 1;
693 : // tell line_table that new line starts
694 2981 : start_line (current_line, max_column_hint);
695 2981 : continue;
696 : }
697 :
698 138168 : skip_input ();
699 138168 : current_column++;
700 : }
701 :
702 : // refresh new token
703 1546 : continue;
704 1546 : }
705 1158 : else if (peek_input () == '*'
706 1158 : && (peek_input (1) == '!' || peek_input (1) == '*'))
707 : {
708 : // block doc comment, inner /*! or outer /**
709 124 : bool is_inner = peek_input (1) == '!';
710 124 : skip_input (1);
711 124 : current_column += 3;
712 :
713 124 : std::string str;
714 124 : str.reserve (96);
715 :
716 124 : int level = 1;
717 124 : while (level > 0)
718 : {
719 45757 : current_char = peek_input ();
720 :
721 45757 : if (current_char.is_eof ())
722 : {
723 0 : rust_error_at (
724 : loc, ErrorCode::E0758,
725 : "unexpected EOF while looking for end of comment");
726 0 : break;
727 : }
728 :
729 : // if /* found
730 45757 : if (current_char == '/' && peek_input (1) == '*')
731 : {
732 : // skip /* characters
733 84 : skip_input (1);
734 84 : current_column += 2;
735 :
736 84 : level += 1;
737 84 : str += "/*";
738 84 : continue;
739 : }
740 :
741 : // ignore until */ is found
742 45673 : if (current_char == '*' && peek_input (1) == '/')
743 : {
744 : // skip */ characters
745 208 : skip_input (1);
746 208 : current_column += 2;
747 :
748 208 : level -= 1;
749 208 : if (level > 0)
750 84 : str += "*/";
751 208 : continue;
752 : }
753 :
754 45465 : if (current_char == '\r' && peek_input (1) != '\n')
755 2 : rust_error_at (
756 : loc, "Isolated CR %<\\r%> not allowed in doc comment");
757 :
758 45465 : if (current_char == '\n')
759 : {
760 904 : skip_input ();
761 904 : current_line++;
762 904 : current_column = 1;
763 : // tell line_table that new line starts
764 904 : start_line (current_line, max_column_hint);
765 904 : str += '\n';
766 904 : continue;
767 : }
768 :
769 44561 : str += current_char;
770 44561 : skip_input ();
771 44561 : current_column++;
772 : }
773 :
774 124 : str.shrink_to_fit ();
775 :
776 124 : loc += str.size () - 1;
777 124 : if (is_inner)
778 81 : return Token::make_inner_doc_comment (loc, std::move (str));
779 : else
780 43 : return Token::make_outer_doc_comment (loc, std::move (str));
781 124 : }
782 : else
783 : {
784 : // division
785 1034 : current_column++;
786 1034 : return Token::make (DIV, loc);
787 : }
788 768 : case '%':
789 768 : if (peek_input () == '=')
790 : {
791 : // modulo-assign
792 23 : skip_input ();
793 23 : current_column += 2;
794 23 : loc += 1;
795 :
796 23 : return Token::make (PERCENT_EQ, loc);
797 : }
798 : else
799 : {
800 : // modulo
801 745 : current_column++;
802 745 : return Token::make (PERCENT, loc);
803 : }
804 487 : case '^':
805 487 : if (peek_input () == '=')
806 : {
807 : // xor-assign?
808 236 : skip_input ();
809 236 : current_column += 2;
810 236 : loc += 1;
811 :
812 236 : return Token::make (CARET_EQ, loc);
813 : }
814 : else
815 : {
816 : // xor?
817 251 : current_column++;
818 251 : return Token::make (CARET, loc);
819 : }
820 71101 : case '<':
821 71101 : if (peek_input () == '<')
822 : {
823 9383 : if (peek_input (1) == '=')
824 : {
825 : // left-shift assign
826 71 : skip_input (1);
827 71 : current_column += 3;
828 71 : loc += 2;
829 :
830 71 : return Token::make (LEFT_SHIFT_EQ, loc);
831 : }
832 : else
833 : {
834 : // left-shift
835 9312 : skip_input ();
836 9312 : current_column += 2;
837 9312 : loc += 1;
838 :
839 9312 : return Token::make (LEFT_SHIFT, loc);
840 : }
841 : }
842 61718 : else if (peek_input () == '=')
843 : {
844 : // smaller than or equal to
845 1321 : skip_input ();
846 1321 : current_column += 2;
847 1321 : loc += 1;
848 :
849 1321 : return Token::make (LESS_OR_EQUAL, loc);
850 : }
851 : else
852 : {
853 : // smaller than
854 60397 : current_column++;
855 60397 : return Token::make (LEFT_ANGLE, loc);
856 : }
857 61441 : break;
858 61441 : case '>':
859 61441 : if (peek_input () == '>')
860 : {
861 3918 : if (peek_input (1) == '=')
862 : {
863 : // right-shift-assign
864 41 : skip_input (1);
865 41 : current_column += 3;
866 41 : loc += 2;
867 :
868 41 : return Token::make (RIGHT_SHIFT_EQ, loc);
869 : }
870 : else
871 : {
872 : // right-shift
873 3877 : skip_input ();
874 3877 : current_column += 2;
875 3877 : loc += 1;
876 :
877 3877 : return Token::make (RIGHT_SHIFT, loc);
878 : }
879 : }
880 57523 : else if (peek_input () == '=')
881 : {
882 : // larger than or equal to
883 1595 : skip_input ();
884 1595 : current_column += 2;
885 1595 : loc += 1;
886 :
887 1595 : return Token::make (GREATER_OR_EQUAL, loc);
888 : }
889 : else
890 : {
891 : // larger than
892 55928 : current_column++;
893 55928 : return Token::make (RIGHT_ANGLE, loc);
894 : }
895 211626 : case ':':
896 211626 : if (peek_input () == ':')
897 : {
898 : // scope resolution ::
899 69191 : skip_input ();
900 69191 : current_column += 2;
901 69191 : loc += 1;
902 :
903 69191 : return Token::make (SCOPE_RESOLUTION, loc);
904 : }
905 : else
906 : {
907 : // single colon :
908 142435 : current_column++;
909 142435 : return Token::make (COLON, loc);
910 : }
911 66933 : case '!':
912 : // no special handling for macros in lexer?
913 66933 : if (peek_input () == '=')
914 : {
915 : // not equal boolean operator
916 3176 : skip_input ();
917 3176 : current_column += 2;
918 3176 : loc += 1;
919 :
920 3176 : return Token::make (NOT_EQUAL, loc);
921 : }
922 : else
923 : {
924 : // not equal unary operator
925 63757 : current_column++;
926 :
927 63757 : return Token::make (EXCLAM, loc);
928 : }
929 4596 : case '?':
930 4596 : current_column++;
931 4596 : return Token::make (QUESTION_MARK, loc);
932 154837 : case '#':
933 154837 : current_column++;
934 154837 : return Token::make (HASH, loc);
935 196911 : case '[':
936 196911 : current_column++;
937 196911 : return Token::make (LEFT_SQUARE, loc);
938 196904 : case ']':
939 196904 : current_column++;
940 196904 : return Token::make (RIGHT_SQUARE, loc);
941 158729 : case '{':
942 158729 : current_column++;
943 158729 : return Token::make (LEFT_CURLY, loc);
944 158676 : case '}':
945 158676 : current_column++;
946 158676 : return Token::make (RIGHT_CURLY, loc);
947 203 : case '@':
948 203 : current_column++;
949 203 : return Token::make (PATTERN_BIND, loc);
950 69503 : case '$':
951 69503 : current_column++;
952 69503 : return Token::make (DOLLAR_SIGN, loc);
953 0 : case '~':
954 0 : current_column++;
955 0 : return Token::make (TILDE, loc);
956 0 : case '\\':
957 0 : current_column++;
958 0 : return Token::make (BACKSLASH, loc);
959 0 : case '`':
960 0 : current_column++;
961 0 : return Token::make (BACKTICK, loc);
962 9986 : case '|':
963 9986 : if (peek_input () == '=')
964 : {
965 : // bitwise or-assign?
966 140 : skip_input ();
967 140 : current_column += 2;
968 140 : loc += 1;
969 :
970 140 : return Token::make (PIPE_EQ, loc);
971 : }
972 9846 : else if (peek_input () == '|')
973 : {
974 : // logical or
975 1289 : skip_input ();
976 1289 : current_column += 2;
977 1289 : loc += 1;
978 :
979 1289 : return Token::make (OR, loc);
980 : }
981 : else
982 : {
983 : // bitwise or
984 8557 : current_column++;
985 :
986 8557 : return Token::make (PIPE, loc);
987 : }
988 57285 : case '&':
989 57285 : if (peek_input () == '=')
990 : {
991 : // bitwise and-assign?
992 29 : skip_input ();
993 29 : current_column += 2;
994 29 : loc += 1;
995 :
996 29 : return Token::make (AMP_EQ, loc);
997 : }
998 57256 : else if (peek_input () == '&')
999 : {
1000 : // logical and
1001 1947 : skip_input ();
1002 1947 : current_column += 2;
1003 1947 : loc += 1;
1004 :
1005 1947 : return Token::make (LOGICAL_AND, loc);
1006 : }
1007 : else
1008 : {
1009 : // bitwise and/reference
1010 55309 : current_column++;
1011 :
1012 55309 : return Token::make (AMP, loc);
1013 : }
1014 84386 : case '.':
1015 84386 : if (peek_input () == '.')
1016 : {
1017 5801 : if (peek_input (1) == '.')
1018 : {
1019 : // ellipsis
1020 921 : skip_input (1);
1021 921 : current_column += 3;
1022 921 : loc += 2;
1023 :
1024 921 : return Token::make (ELLIPSIS, loc);
1025 : }
1026 4880 : else if (peek_input (1) == '=')
1027 : {
1028 : // ..=
1029 679 : skip_input (1);
1030 679 : current_column += 3;
1031 679 : loc += 2;
1032 :
1033 679 : return Token::make (DOT_DOT_EQ, loc);
1034 : }
1035 : else
1036 : {
1037 : // ..
1038 4201 : skip_input ();
1039 4201 : current_column += 2;
1040 4201 : loc += 1;
1041 :
1042 4201 : return Token::make (DOT_DOT, loc);
1043 : }
1044 : }
1045 : else /*if (!ISDIGIT (peek_input ()))*/
1046 : {
1047 : // single dot .
1048 : // Only if followed by a non-number - otherwise is float
1049 : // nope, float cannot start with '.'.
1050 78585 : current_column++;
1051 78585 : return Token::make (DOT, loc);
1052 : }
1053 8891790 : }
1054 : // TODO: special handling of _ in the lexer? instead of being identifier
1055 :
1056 : // byte character, byte string and raw byte string literals
1057 2833538 : if (current_char == 'b')
1058 : {
1059 71145 : if (peek_input () == '\'')
1060 1086 : return parse_byte_char (loc);
1061 70059 : else if (peek_input () == '"')
1062 520 : return parse_byte_string (loc);
1063 69539 : else if (peek_input () == 'r'
1064 69539 : && (peek_input (1) == '#' || peek_input (1) == '"'))
1065 32 : return parse_raw_byte_string (loc);
1066 : }
1067 :
1068 : // C-style strings
1069 2762393 : else if (current_char == 'c' && peek_input () == '"')
1070 15 : return parse_c_string (loc);
1071 :
1072 : // raw identifiers and raw strings
1073 2831885 : if (current_char == 'r')
1074 : {
1075 72588 : Codepoint peek = peek_input ();
1076 72588 : Codepoint peek1 = peek_input (1);
1077 :
1078 : // TODO (tamaron) parse Unicode ident
1079 72588 : if (peek == '#' && is_identifier_start (peek1.value))
1080 : {
1081 116 : TokenPtr raw_ident_ptr = parse_raw_identifier (loc);
1082 116 : if (raw_ident_ptr != nullptr)
1083 115 : return raw_ident_ptr;
1084 : else
1085 1 : continue; /* input got parsed, it just wasn't valid. An error
1086 : was produced. */
1087 116 : }
1088 : else
1089 : {
1090 72472 : TokenPtr maybe_raw_string_ptr = maybe_parse_raw_string (loc);
1091 72472 : if (maybe_raw_string_ptr != nullptr)
1092 73 : return maybe_raw_string_ptr;
1093 72472 : }
1094 : }
1095 :
1096 : // find identifiers and keywords.
1097 2831696 : if (is_identifier_start (current_char.value))
1098 2145390 : return parse_identifier_or_keyword (loc);
1099 :
1100 : // int and float literals
1101 686306 : if (ISDIGIT (current_char.value))
1102 : { // _ not allowed as first char
1103 431372 : if (current_char == '0'
1104 431372 : && is_non_decimal_int_literal_separator (peek_input ().value))
1105 : {
1106 : // handle binary, octal, hex literals
1107 35180 : TokenPtr non_dec_int_lit_ptr
1108 35180 : = parse_non_decimal_int_literals (loc);
1109 35180 : if (non_dec_int_lit_ptr != nullptr)
1110 35180 : return non_dec_int_lit_ptr;
1111 35180 : }
1112 : else
1113 : {
1114 : // handle decimals (integer or float)
1115 396192 : TokenPtr decimal_or_float_ptr = parse_decimal_int_or_float (loc);
1116 396192 : if (decimal_or_float_ptr != nullptr)
1117 396192 : return decimal_or_float_ptr;
1118 396192 : }
1119 : }
1120 :
1121 : // string literals
1122 254934 : if (current_char == '"')
1123 143305 : return parse_string (loc);
1124 :
1125 : // char literals and lifetime names
1126 111629 : if (current_char == '\'')
1127 : {
1128 111629 : TokenPtr char_or_lifetime_ptr = parse_char_or_lifetime (loc);
1129 111629 : if (char_or_lifetime_ptr != nullptr)
1130 111629 : return char_or_lifetime_ptr;
1131 111629 : }
1132 :
1133 : // DEBUG: check for specific character problems:
1134 0 : if (current_char == '0')
1135 0 : rust_debug ("'0' uncaught before unexpected character");
1136 0 : else if (current_char == ']')
1137 0 : rust_debug ("']' uncaught before unexpected character");
1138 : else if (current_char == 0x5d)
1139 : rust_debug ("whatever 0x5d is (not '0' or ']') uncaught before "
1140 : "unexpected character");
1141 :
1142 : // didn't match anything so error
1143 0 : rust_error_at (loc, "unexpected character %<%x%>", current_char.value);
1144 0 : current_column++;
1145 : }
1146 : }
1147 :
1148 : // Parses in a suffix
1149 : std::pair<std::string, int>
1150 348913 : Lexer::parse_in_suffix ()
1151 : {
1152 348913 : std::string suffix;
1153 :
1154 348913 : int additional_length_offset = 0;
1155 :
1156 : // get suffix
1157 710911 : while (ISALPHA (current_char.value) || ISDIGIT (current_char.value)
1158 718780 : || current_char == '_')
1159 : {
1160 20953 : additional_length_offset++;
1161 :
1162 20953 : suffix += current_char;
1163 20953 : skip_input ();
1164 20953 : current_char = peek_input ();
1165 : }
1166 :
1167 348913 : return std::make_pair (std::move (suffix), additional_length_offset);
1168 348913 : }
1169 :
1170 : // Parses in the exponent part (if any) of a float literal.
1171 : std::pair<std::string, int>
1172 25821 : Lexer::parse_in_exponent_part ()
1173 : {
1174 25821 : int additional_length_offset = 0;
1175 25821 : std::string str;
1176 25821 : if (current_char == 'E' || current_char == 'e')
1177 : {
1178 : // add exponent to string as strtod works with it
1179 672 : str += current_char;
1180 672 : skip_input ();
1181 672 : current_char = peek_input ();
1182 :
1183 672 : additional_length_offset++;
1184 :
1185 : // special - and + handling
1186 672 : if (current_char == '-' || current_char == '+')
1187 : {
1188 152 : str += current_char;
1189 :
1190 152 : skip_input ();
1191 152 : current_char = peek_input ();
1192 :
1193 152 : additional_length_offset++;
1194 : }
1195 :
1196 : // parse another decimal number for exponent
1197 672 : auto str_length = parse_in_decimal ();
1198 672 : str += std::get<0> (str_length);
1199 672 : additional_length_offset += std::get<1> (str_length);
1200 672 : }
1201 51642 : return std::make_pair (str, additional_length_offset);
1202 25821 : }
1203 :
1204 : // Parses a decimal integer.
1205 : std::tuple<std::string, int, bool>
1206 422229 : Lexer::parse_in_decimal ()
1207 : {
1208 : /* A pure decimal contains only digits. */
1209 422229 : bool pure_decimal = true;
1210 422229 : int additional_length_offset = 0;
1211 422229 : std::string str;
1212 660799 : while (ISDIGIT (current_char.value) || current_char.value == '_')
1213 : {
1214 238570 : if (current_char == '_')
1215 : {
1216 1630 : pure_decimal = false;
1217 : }
1218 238570 : additional_length_offset++;
1219 :
1220 238570 : str += current_char;
1221 238570 : skip_input ();
1222 238570 : current_char = peek_input ();
1223 : }
1224 844458 : return std::make_tuple (str, additional_length_offset, pure_decimal);
1225 422229 : }
1226 :
1227 : /* Parses escapes (and string continues) in "byte" strings and characters. Does
1228 : * not support unicode. */
1229 : std::tuple<char, int, bool>
1230 286 : Lexer::parse_escape (char opening_char)
1231 : {
1232 286 : int additional_length_offset = 0;
1233 286 : char output_char = 0;
1234 :
1235 : // skip to actual letter
1236 286 : skip_input ();
1237 286 : current_char = peek_input ();
1238 286 : additional_length_offset++;
1239 :
1240 286 : switch (current_char.value)
1241 : {
1242 57 : case 'x':
1243 57 : {
1244 57 : auto hex_escape_pair = parse_partial_hex_escape ();
1245 57 : long hexLong = hex_escape_pair.first;
1246 57 : additional_length_offset += hex_escape_pair.second;
1247 :
1248 57 : if (hexLong > 255 || hexLong < 0)
1249 0 : rust_error_at (
1250 : get_current_location (),
1251 : "byte \\x escape %<\\x%x%> out of range - allows up to %<\\xFF%>",
1252 : static_cast<unsigned int> (hexLong));
1253 : /* TODO: restore capital for escape output - gcc pretty-printer doesn't
1254 : * support %X directly */
1255 57 : char hexChar = static_cast<char> (hexLong);
1256 :
1257 57 : output_char = hexChar;
1258 : }
1259 57 : break;
1260 : case 'n':
1261 : output_char = '\n';
1262 : break;
1263 24 : case 'r':
1264 24 : output_char = '\r';
1265 24 : break;
1266 17 : case 't':
1267 17 : output_char = '\t';
1268 17 : break;
1269 80 : case '\\':
1270 80 : output_char = '\\';
1271 80 : break;
1272 18 : case '0':
1273 18 : output_char = '\0';
1274 18 : break;
1275 31 : case '\'':
1276 31 : output_char = '\'';
1277 31 : break;
1278 1 : case '"':
1279 1 : output_char = '"';
1280 1 : break;
1281 2 : case 'u':
1282 3 : rust_error_at (get_current_location (),
1283 : "cannot have a unicode escape \\u in a byte %s",
1284 : opening_char == '\'' ? "character" : "string");
1285 : // Try to parse it anyway, just to skip it
1286 2 : parse_partial_unicode_escape ();
1287 2 : return std::make_tuple (output_char, additional_length_offset, false);
1288 32 : case '\r':
1289 32 : case '\n':
1290 : // string continue
1291 32 : return std::make_tuple (0, parse_partial_string_continue (), true);
1292 1 : default:
1293 1 : rust_error_at (get_current_location (),
1294 : "unknown escape sequence %<\\%s%>",
1295 1 : current_char.as_string ().c_str ());
1296 : // returns false if no parsing could be done
1297 : // return false;
1298 1 : return std::make_tuple (output_char, additional_length_offset, false);
1299 251 : break;
1300 : }
1301 : // all non-special cases (string continue) should skip their used char
1302 251 : skip_input ();
1303 251 : current_char = peek_input ();
1304 251 : additional_length_offset++;
1305 :
1306 : // returns true if parsing was successful
1307 : // return true;
1308 251 : return std::make_tuple (output_char, additional_length_offset, false);
1309 : }
1310 :
1311 : /* Parses an escape (or string continue) in a string or character. Supports
1312 : * unicode escapes. */
1313 : std::tuple<Codepoint, int, bool>
1314 96324 : Lexer::parse_utf8_escape ()
1315 : {
1316 96324 : Codepoint output_char;
1317 96324 : int additional_length_offset = 0;
1318 :
1319 : // skip to actual letter
1320 96324 : skip_input ();
1321 96324 : current_char = peek_input ();
1322 96324 : additional_length_offset++;
1323 :
1324 96324 : switch (current_char.value)
1325 : {
1326 121 : case 'x':
1327 121 : {
1328 121 : auto hex_escape_pair = parse_partial_hex_escape ();
1329 121 : long hexLong = hex_escape_pair.first;
1330 121 : additional_length_offset += hex_escape_pair.second;
1331 :
1332 121 : if (hexLong > 127 || hexLong < 0)
1333 4 : rust_error_at (
1334 : get_current_location (),
1335 : "ascii \\x escape %<\\x%x%> out of range - allows up to %<\\x7F%>",
1336 : static_cast<unsigned int> (hexLong));
1337 : /* TODO: restore capital for escape output - gcc pretty-printer doesn't
1338 : * support %X directly */
1339 121 : char hexChar = static_cast<char> (hexLong);
1340 :
1341 121 : output_char = hexChar;
1342 : }
1343 121 : break;
1344 : case 'n':
1345 : output_char = '\n';
1346 : break;
1347 24 : case 'r':
1348 24 : output_char = '\r';
1349 24 : break;
1350 90 : case 't':
1351 90 : output_char = '\t';
1352 90 : break;
1353 57 : case '\\':
1354 57 : output_char = '\\';
1355 57 : break;
1356 1572 : case '0':
1357 1572 : output_char = '\0';
1358 1572 : break;
1359 33 : case '\'':
1360 33 : output_char = '\'';
1361 33 : break;
1362 465 : case '"':
1363 465 : output_char = '"';
1364 465 : break;
1365 91096 : case 'u':
1366 91096 : {
1367 91096 : auto unicode_escape_pair = parse_partial_unicode_escape ();
1368 91096 : output_char = unicode_escape_pair.first;
1369 91096 : additional_length_offset += unicode_escape_pair.second;
1370 :
1371 91096 : return std::make_tuple (output_char, additional_length_offset, false);
1372 : }
1373 1237 : break;
1374 1237 : case '\r':
1375 1237 : case '\n':
1376 : // string continue
1377 1237 : return std::make_tuple (0, parse_partial_string_continue (), true);
1378 1 : default:
1379 1 : rust_error_at (get_current_location (),
1380 : "unknown escape sequence %<\\%s%>",
1381 1 : current_char.as_string ().c_str ());
1382 : // returns false if no parsing could be done
1383 : // return false;
1384 1 : return std::make_tuple (output_char, additional_length_offset, false);
1385 3990 : break;
1386 : }
1387 : /* all non-special cases (unicode, string continue) should skip their used
1388 : * char */
1389 3990 : skip_input ();
1390 3990 : current_char = peek_input ();
1391 3990 : additional_length_offset++;
1392 :
1393 : // returns true if parsing was successful
1394 : // return true;
1395 3990 : return std::make_tuple (output_char, additional_length_offset, false);
1396 : }
1397 :
1398 : // Parses the body of a string continue that has been found in an escape.
1399 : int
1400 1269 : Lexer::parse_partial_string_continue ()
1401 : {
1402 1269 : int additional_length_offset = 1;
1403 :
1404 : // string continue
1405 : // TODO use utf-8 codepoint to skip whitespaces
1406 29208 : while (is_whitespace (current_char.value))
1407 : {
1408 27939 : if (current_char == '\n')
1409 : {
1410 1269 : current_line++;
1411 1269 : current_column = 1;
1412 : // tell line_table that new line starts
1413 1269 : start_line (current_line, max_column_hint);
1414 :
1415 : // reset "length"
1416 1269 : additional_length_offset = 1;
1417 :
1418 : // get next char
1419 1269 : skip_input ();
1420 1269 : current_char = peek_input ();
1421 :
1422 1269 : continue;
1423 : }
1424 :
1425 26670 : skip_input ();
1426 26670 : current_char = peek_input ();
1427 26670 : additional_length_offset++;
1428 : }
1429 :
1430 1269 : return additional_length_offset;
1431 : }
1432 :
1433 : /* Parses the body of a '\x' escape. Note that it does not check that the number
1434 : * is valid and smaller than 255. */
1435 : std::pair<long, int>
1436 178 : Lexer::parse_partial_hex_escape ()
1437 : {
1438 : // hex char string (null-terminated)
1439 178 : char hexNum[3] = {0, 0, 0};
1440 :
1441 : // first hex char
1442 178 : current_char = peek_input (1);
1443 178 : int additional_length_offset = 1;
1444 :
1445 178 : if (!is_x_digit (current_char.value))
1446 : {
1447 4 : rust_error_at (get_current_location (),
1448 : "invalid character %<\\x%s%> in \\x sequence",
1449 4 : current_char.as_string ().c_str ());
1450 4 : return std::make_pair (0, 0);
1451 : }
1452 174 : hexNum[0] = current_char.value;
1453 :
1454 : // second hex char
1455 174 : skip_input ();
1456 174 : current_char = peek_input (1);
1457 174 : additional_length_offset++;
1458 :
1459 174 : if (!is_x_digit (current_char.value))
1460 : {
1461 2 : rust_error_at (get_current_location (),
1462 2 : "invalid character %<\\x%c%s%> in \\x sequence", hexNum[0],
1463 2 : current_char.as_string ().c_str ());
1464 2 : return std::make_pair (0, 1);
1465 : }
1466 172 : skip_input ();
1467 172 : hexNum[1] = current_char.value;
1468 :
1469 172 : long hexLong = std::strtol (hexNum, nullptr, 16);
1470 :
1471 172 : return std::make_pair (hexLong, additional_length_offset);
1472 : }
1473 :
1474 : // Parses the body of a unicode escape.
1475 : std::pair<Codepoint, int>
1476 91098 : Lexer::parse_partial_unicode_escape ()
1477 : {
1478 91098 : skip_input ();
1479 91098 : current_char = peek_input ();
1480 91098 : int additional_length_offset = 0;
1481 :
1482 91098 : if (current_char != '{')
1483 : {
1484 2 : rust_error_at (get_current_location (),
1485 : "unicode escape should start with %<{%>");
1486 : /* Skip what should probaby have been between brackets. */
1487 10 : while (is_x_digit (current_char.value) || current_char == '_')
1488 : {
1489 6 : skip_input ();
1490 6 : current_char = peek_input ();
1491 6 : additional_length_offset++;
1492 : }
1493 2 : return std::make_pair (Codepoint (0), additional_length_offset);
1494 : }
1495 :
1496 91096 : skip_input ();
1497 91096 : current_char = peek_input ();
1498 91096 : additional_length_offset++;
1499 :
1500 91096 : if (current_char == '_')
1501 : {
1502 2 : rust_error_at (get_current_location (),
1503 : "unicode escape cannot start with %<_%>");
1504 2 : skip_input ();
1505 2 : current_char = peek_input ();
1506 2 : additional_length_offset++;
1507 : // fallthrough and try to parse the rest anyway
1508 : }
1509 :
1510 : // parse unicode escape - 1-6 hex digits
1511 91096 : std::string num_str;
1512 91096 : num_str.reserve (6);
1513 :
1514 : // loop through to add entire hex number to string
1515 401652 : while (is_x_digit (current_char.value) || current_char.value == '_')
1516 : {
1517 219460 : if (current_char == '_')
1518 : {
1519 : // don't add _ to number
1520 24 : skip_input ();
1521 24 : current_char = peek_input ();
1522 :
1523 24 : additional_length_offset++;
1524 :
1525 24 : continue;
1526 : }
1527 :
1528 219436 : additional_length_offset++;
1529 :
1530 : // add raw hex numbers
1531 219436 : num_str += current_char;
1532 :
1533 219436 : skip_input ();
1534 219436 : current_char = peek_input ();
1535 : }
1536 :
1537 91096 : if (current_char == '}')
1538 : {
1539 91094 : skip_input ();
1540 91094 : current_char = peek_input ();
1541 91094 : additional_length_offset++;
1542 : }
1543 : else
1544 : {
1545 : // actually an error, but allow propagation anyway Assume that
1546 : // wrong bracketm whitespace or single/double quotes are wrong
1547 : // termination, otherwise it is a wrong character, then skip to the actual
1548 : // terminator.
1549 : // TODO use utf-8 codepoint to skip whitespaces
1550 2 : if (current_char == '{' || is_whitespace (current_char.value)
1551 4 : || current_char == '\'' || current_char == '"')
1552 : {
1553 0 : rust_error_at (get_current_location (),
1554 : "expected terminating %<}%> in unicode escape");
1555 0 : return std::make_pair (Codepoint (0), additional_length_offset);
1556 : }
1557 : else
1558 : {
1559 2 : rust_error_at (get_current_location (),
1560 : "invalid character %qs in unicode escape",
1561 2 : current_char.as_string ().c_str ());
1562 : // TODO use utf-8 codepoint to skip whitespaces
1563 8 : while (current_char != '}' && current_char != '{'
1564 6 : && !is_whitespace (current_char.value) && current_char != '\''
1565 14 : && current_char != '"')
1566 : {
1567 6 : skip_input ();
1568 6 : current_char = peek_input ();
1569 6 : additional_length_offset++;
1570 : }
1571 : // Consume the actual closing bracket if found
1572 2 : if (current_char == '}')
1573 : {
1574 2 : skip_input ();
1575 2 : current_char = peek_input ();
1576 2 : additional_length_offset++;
1577 : }
1578 2 : return std::make_pair (Codepoint (0), additional_length_offset);
1579 : }
1580 : }
1581 :
1582 : // ensure 1-6 hex characters
1583 91094 : if (num_str.length () > 6 || num_str.length () < 1)
1584 : {
1585 4 : rust_error_at (get_current_location (),
1586 : "unicode escape should be between 1 and 6 hex "
1587 : "characters; it is %lu",
1588 4 : (unsigned long) num_str.length ());
1589 : // return false;
1590 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1591 : }
1592 :
1593 91090 : unsigned long hex_num = std::strtoul (num_str.c_str (), nullptr, 16);
1594 :
1595 91090 : if (hex_num > 0xd7ff && hex_num < 0xe000)
1596 : {
1597 4 : rust_error_at (
1598 : get_current_location (),
1599 : "unicode escape cannot be a surrogate value (D800 to DFFF)");
1600 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1601 : }
1602 :
1603 91086 : if (hex_num > 0x10ffff)
1604 : {
1605 4 : rust_error_at (get_current_location (),
1606 : "unicode escape cannot be larger than 10FFFF");
1607 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1608 : }
1609 :
1610 : // return true;
1611 91082 : return std::make_pair (Codepoint (static_cast<uint32_t> (hex_num)),
1612 : additional_length_offset);
1613 91096 : }
1614 :
1615 : // Parses a byte character.
1616 : TokenPtr
1617 1086 : Lexer::parse_byte_char (location_t loc)
1618 : {
1619 1086 : skip_input ();
1620 1086 : current_column++;
1621 : // make current char the next character
1622 1086 : current_char = peek_input ();
1623 :
1624 1086 : int length = 1;
1625 :
1626 : // char to save
1627 1086 : Codepoint byte_char = 0;
1628 :
1629 : // detect escapes
1630 1086 : if (current_char == '\\')
1631 : {
1632 222 : auto escape_length_pair = parse_escape ('\'');
1633 222 : byte_char = std::get<0> (escape_length_pair);
1634 222 : length += std::get<1> (escape_length_pair);
1635 :
1636 222 : current_char = peek_input ();
1637 :
1638 222 : if (current_char != '\'')
1639 : {
1640 0 : rust_error_at (get_current_location (), "unclosed %<byte char%>");
1641 : }
1642 :
1643 222 : skip_input ();
1644 222 : current_char = peek_input ();
1645 222 : length++; // go to next char
1646 : }
1647 864 : else if (current_char != '\'')
1648 : {
1649 : // otherwise, get character from direct input character
1650 864 : byte_char = current_char;
1651 :
1652 864 : if (!byte_char.is_ascii ())
1653 : {
1654 2 : rust_error_at (get_current_location (),
1655 : "non-ASCII character in %<byte char%>");
1656 : }
1657 :
1658 864 : skip_input ();
1659 864 : current_char = peek_input ();
1660 864 : length++;
1661 :
1662 864 : if (current_char != '\'')
1663 : {
1664 0 : rust_error_at (get_current_location (), "unclosed %<byte char%>");
1665 : }
1666 :
1667 864 : skip_input ();
1668 864 : current_char = peek_input ();
1669 864 : length++; // go to next char
1670 : }
1671 : else
1672 : {
1673 0 : rust_error_at (get_current_location (),
1674 : "no character inside %<%> for %<byte char%>");
1675 : }
1676 :
1677 1086 : current_column += length;
1678 :
1679 1086 : loc += length - 1;
1680 1086 : return Token::make_byte_char (loc, byte_char.value);
1681 : }
1682 :
1683 : // Parses a byte string.
1684 : TokenPtr
1685 520 : Lexer::parse_byte_string (location_t loc)
1686 : {
1687 : // byte string
1688 :
1689 : // skip quote character
1690 520 : skip_input ();
1691 520 : current_column++;
1692 :
1693 520 : std::string str;
1694 520 : str.reserve (16); // some sensible default
1695 :
1696 520 : current_char = peek_input ();
1697 :
1698 520 : const location_t string_begin_locus = get_current_location ();
1699 :
1700 5086 : while (current_char != '"' && !current_char.is_eof ())
1701 : {
1702 4046 : if (current_char == '\\')
1703 : {
1704 63 : int length = 1;
1705 63 : auto escape_length_pair = parse_escape ('"');
1706 63 : char output_char = std::get<0> (escape_length_pair);
1707 :
1708 63 : if (output_char == 0 && std::get<2> (escape_length_pair))
1709 32 : length = std::get<1> (escape_length_pair) - 1;
1710 : else
1711 31 : length += std::get<1> (escape_length_pair);
1712 :
1713 63 : if (output_char != 0 || !std::get<2> (escape_length_pair))
1714 31 : str += output_char;
1715 :
1716 63 : current_column += length;
1717 :
1718 63 : continue;
1719 63 : }
1720 :
1721 3983 : current_column++;
1722 3983 : if (current_char.value == '\n')
1723 : {
1724 23 : current_line++;
1725 23 : current_column = 1;
1726 : // tell line_table that new line starts
1727 23 : start_line (current_line, max_column_hint);
1728 : }
1729 :
1730 3983 : str += current_char;
1731 3983 : skip_input ();
1732 3983 : current_char = peek_input ();
1733 : }
1734 :
1735 520 : if (current_char == '"')
1736 : {
1737 513 : current_column++;
1738 :
1739 513 : skip_input ();
1740 513 : current_char = peek_input ();
1741 : }
1742 7 : else if (current_char.is_eof ())
1743 : {
1744 7 : rust_error_at (string_begin_locus, "unended byte string literal");
1745 7 : return Token::make (END_OF_FILE, get_current_location ());
1746 : }
1747 : else
1748 : {
1749 : rust_unreachable ();
1750 : }
1751 :
1752 513 : str.shrink_to_fit ();
1753 513 : loc += str.size () - 1;
1754 :
1755 513 : return Token::make_byte_string (loc, std::move (str));
1756 520 : }
1757 :
1758 : // Parses a C-style string.
1759 : TokenPtr
1760 15 : Lexer::parse_c_string (location_t loc)
1761 : {
1762 15 : skip_input ();
1763 15 : current_column++;
1764 :
1765 : // Mostly same code copied from parse_string...
1766 :
1767 15 : std::string str;
1768 15 : str.reserve (16); // some sensible default
1769 :
1770 15 : current_char = peek_input ();
1771 :
1772 15 : const location_t string_begin_locus = get_current_location ();
1773 :
1774 106 : while (current_char.value != '"' && !current_char.is_eof ())
1775 : {
1776 76 : if (current_char.value == '\\')
1777 : {
1778 1 : int length = 1;
1779 :
1780 1 : auto escape_pair = parse_escape ('"');
1781 1 : current_char = std::get<0> (escape_pair);
1782 :
1783 1 : if (current_char == Codepoint (0) && std::get<2> (escape_pair))
1784 0 : length = std::get<1> (escape_pair) - 1;
1785 : else
1786 1 : length += std::get<1> (escape_pair);
1787 :
1788 1 : if (current_char != Codepoint (0) || !std::get<2> (escape_pair))
1789 2 : str += current_char.as_string ();
1790 :
1791 1 : current_column += length;
1792 :
1793 : // FIXME: parse_escape does not update current_char correctly.
1794 1 : current_char = peek_input ();
1795 1 : continue;
1796 1 : }
1797 :
1798 75 : current_column++;
1799 75 : if (current_char.value == '\n')
1800 : {
1801 0 : current_line++;
1802 0 : current_column = 1;
1803 : // tell line_table that new line starts
1804 0 : start_line (current_line, max_column_hint);
1805 : }
1806 :
1807 75 : str += current_char;
1808 75 : skip_input ();
1809 75 : current_char = peek_input ();
1810 : }
1811 :
1812 15 : if (current_char.value == '"')
1813 : {
1814 15 : current_column++;
1815 :
1816 15 : skip_input ();
1817 15 : current_char = peek_input ();
1818 : }
1819 0 : else if (current_char.is_eof ())
1820 : {
1821 0 : rust_error_at (string_begin_locus, "unended C string literal");
1822 0 : return Token::make (END_OF_FILE, get_current_location ());
1823 : }
1824 : else
1825 : {
1826 : rust_unreachable ();
1827 : }
1828 :
1829 15 : str.shrink_to_fit ();
1830 :
1831 15 : return Token::make_c_string (loc, std::move (str));
1832 15 : }
1833 :
1834 : // Parses a raw byte string.
1835 : TokenPtr
1836 32 : Lexer::parse_raw_byte_string (location_t loc)
1837 : {
1838 : // raw byte string literals
1839 32 : std::string str;
1840 32 : str.reserve (16); // some sensible default
1841 :
1842 32 : int length = 1;
1843 32 : int hash_count = 0;
1844 :
1845 32 : const location_t string_begin_locus = get_current_location ();
1846 :
1847 : // get hash count at beginnning
1848 32 : skip_input ();
1849 32 : current_char = peek_input ();
1850 32 : length++;
1851 32 : current_column++;
1852 54 : while (current_char == '#')
1853 : {
1854 22 : hash_count++;
1855 22 : length++;
1856 22 : current_column++;
1857 :
1858 22 : skip_input ();
1859 22 : current_char = peek_input ();
1860 : }
1861 :
1862 32 : if (current_char != '"')
1863 : {
1864 0 : rust_error_at (get_current_location (),
1865 : "raw byte string has no opening %<\"%>");
1866 : }
1867 :
1868 32 : skip_input ();
1869 32 : current_char = peek_input ();
1870 32 : length++;
1871 32 : current_column++;
1872 :
1873 330 : while (true)
1874 : {
1875 181 : if (current_char == '"')
1876 : {
1877 51 : bool enough_hashes = true;
1878 :
1879 51 : for (int i = 0; i < hash_count; i++)
1880 : {
1881 26 : if (peek_input (i + 1) != '#')
1882 : {
1883 : enough_hashes = false;
1884 : break;
1885 : }
1886 : }
1887 :
1888 35 : if (enough_hashes)
1889 : {
1890 : // skip enough input and peek enough input
1891 25 : skip_input (hash_count);
1892 25 : current_char = peek_input ();
1893 25 : length += hash_count + 1;
1894 25 : current_column += hash_count + 1;
1895 25 : break;
1896 : }
1897 : }
1898 146 : else if (current_char.is_eof ())
1899 : {
1900 7 : rust_error_at (string_begin_locus, "unended raw byte string literal");
1901 7 : return Token::make (END_OF_FILE, get_current_location ());
1902 : }
1903 139 : else if (current_char.value > 127)
1904 : {
1905 1 : rust_error_at (get_current_location (),
1906 : "character %qs in raw byte string out of range",
1907 1 : current_char.as_string ().c_str ());
1908 1 : current_char = 0;
1909 : }
1910 :
1911 149 : length++;
1912 149 : current_column++;
1913 149 : if (current_char == '\n')
1914 : {
1915 22 : current_line++;
1916 22 : current_column = 1;
1917 22 : start_line (current_line, max_column_hint);
1918 : }
1919 :
1920 149 : str += current_char;
1921 149 : skip_input ();
1922 149 : current_char = peek_input ();
1923 149 : }
1924 :
1925 25 : loc += length - 1;
1926 :
1927 25 : str.shrink_to_fit ();
1928 :
1929 25 : return Token::make_byte_string (loc, std::move (str));
1930 32 : }
1931 :
1932 : // Parses a raw identifier.
1933 : TokenPtr
1934 116 : Lexer::parse_raw_identifier (location_t loc)
1935 : {
1936 : // raw identifier
1937 116 : std::string str;
1938 116 : str.reserve (16); // default
1939 :
1940 116 : skip_input ();
1941 116 : current_char = peek_input ();
1942 :
1943 116 : current_column += 2;
1944 :
1945 116 : bool first_is_underscore = current_char == '_';
1946 :
1947 116 : int length = 0;
1948 116 : current_char = peek_input ();
1949 : // loop through entire name
1950 650 : while (is_identifier_continue (current_char.value))
1951 : {
1952 418 : length++;
1953 :
1954 418 : str += current_char;
1955 418 : skip_input ();
1956 418 : current_char = peek_input ();
1957 : }
1958 :
1959 116 : current_column += length;
1960 :
1961 116 : rust_debug ("raw ident: %s", str.c_str ());
1962 :
1963 : // if just a single underscore, not an identifier
1964 116 : if (first_is_underscore && length == 1)
1965 1 : rust_error_at (get_current_location (),
1966 : "%<_%> is not a valid raw identifier");
1967 :
1968 116 : using namespace Rust::Values;
1969 116 : std::set<std::string> invalid{
1970 116 : Keywords::CRATE, Keywords::EXTERN_KW, Keywords::SELF,
1971 116 : Keywords::SUPER, Keywords::SELF_ALIAS,
1972 696 : };
1973 :
1974 116 : if (invalid.find (str) != invalid.end ())
1975 : {
1976 1 : rust_error_at (get_current_location (),
1977 : "%qs is a forbidden raw identifier", str.c_str ());
1978 :
1979 1 : return nullptr;
1980 : }
1981 : else
1982 : {
1983 115 : str.shrink_to_fit ();
1984 115 : loc += length - 1;
1985 :
1986 115 : return Token::make_identifier (loc, std::move (str));
1987 : }
1988 116 : }
1989 :
1990 : // skip broken string input (unterminated strings)
1991 : void
1992 0 : Lexer::skip_broken_string_input (Codepoint current_char)
1993 : {
1994 0 : while (current_char != '"' && !current_char.is_eof ())
1995 : {
1996 0 : if (current_char == '\n')
1997 : {
1998 0 : current_line++;
1999 0 : current_column = 1;
2000 : }
2001 : else
2002 : {
2003 0 : current_column++;
2004 : }
2005 0 : skip_input ();
2006 0 : current_char = peek_input ();
2007 : }
2008 0 : if (current_char == '"')
2009 : {
2010 0 : current_column++;
2011 :
2012 0 : skip_input ();
2013 0 : current_char = peek_input ();
2014 : }
2015 0 : rust_debug ("skipped to %d:%d due to bad quotes", current_line,
2016 : current_column);
2017 0 : }
2018 :
2019 : // Parses a string.
2020 : TokenPtr
2021 143305 : Lexer::parse_string (location_t loc)
2022 : {
2023 143305 : std::string str;
2024 143305 : str.reserve (16); // some sensible default
2025 :
2026 143305 : current_char = peek_input ();
2027 :
2028 143305 : const location_t string_begin_locus = get_current_location ();
2029 :
2030 : // FIXME: This fails if the input ends. How do we check for EOF?
2031 2102144 : while (current_char.value != '"' && !current_char.is_eof ())
2032 : {
2033 1815534 : if (current_char.value == '\\')
2034 : {
2035 4923 : int length = 1;
2036 :
2037 : // parse escape
2038 4923 : auto utf8_escape_pair = parse_utf8_escape ();
2039 4923 : current_char = std::get<0> (utf8_escape_pair);
2040 :
2041 4923 : if (current_char == Codepoint (0) && std::get<2> (utf8_escape_pair))
2042 1237 : length = std::get<1> (utf8_escape_pair) - 1;
2043 : else
2044 3686 : length += std::get<1> (utf8_escape_pair);
2045 :
2046 4923 : if (current_char != Codepoint (0) || !std::get<2> (utf8_escape_pair))
2047 7372 : str += current_char.as_string ();
2048 :
2049 4923 : current_column += length;
2050 :
2051 : // FIXME: should remove this but can't.
2052 : // `parse_utf8_escape` does not update `current_char` correctly.
2053 4923 : current_char = peek_input ();
2054 4923 : continue;
2055 4923 : }
2056 :
2057 1810611 : current_column++;
2058 1810611 : if (current_char.value == '\n')
2059 : {
2060 21731 : current_line++;
2061 21731 : current_column = 1;
2062 : // tell line_table that new line starts
2063 21731 : start_line (current_line, max_column_hint);
2064 : }
2065 :
2066 1810611 : str += current_char;
2067 1810611 : skip_input ();
2068 1810611 : current_char = peek_input ();
2069 : }
2070 :
2071 143305 : if (current_char.value == '"')
2072 : {
2073 143291 : current_column++;
2074 :
2075 143291 : skip_input ();
2076 143291 : current_char = peek_input ();
2077 : }
2078 14 : else if (current_char.is_eof ())
2079 : {
2080 14 : rust_error_at (string_begin_locus, "unended string literal");
2081 14 : return Token::make (END_OF_FILE, get_current_location ());
2082 : }
2083 : else
2084 : {
2085 : rust_unreachable ();
2086 : }
2087 :
2088 143291 : str.shrink_to_fit ();
2089 :
2090 143291 : return Token::make_string (loc, std::move (str));
2091 143305 : }
2092 :
2093 : // Parses an identifier or keyword.
2094 : TokenPtr
2095 2145390 : Lexer::parse_identifier_or_keyword (location_t loc)
2096 : {
2097 2145390 : std::string str;
2098 2145390 : str.reserve (16); // default
2099 4290780 : str += current_char.as_string ();
2100 :
2101 2145390 : bool first_is_underscore = current_char == '_';
2102 :
2103 2145390 : int length = 1;
2104 2145390 : current_char = peek_input ();
2105 :
2106 : // loop through entire name
2107 13561031 : while (is_identifier_continue (current_char.value))
2108 : {
2109 9270251 : auto s = current_char.as_string ();
2110 9270251 : length++;
2111 :
2112 18540502 : str += current_char.as_string ();
2113 9270251 : skip_input ();
2114 9270251 : current_char = peek_input ();
2115 9270251 : }
2116 :
2117 2145390 : current_column += length;
2118 :
2119 : // if just a single underscore, not an identifier
2120 2145390 : if (first_is_underscore && length == 1)
2121 7216 : return Token::make (UNDERSCORE, loc);
2122 :
2123 2138174 : str.shrink_to_fit ();
2124 :
2125 2138174 : loc += length - 1;
2126 :
2127 2138174 : TokenId keyword = classify_keyword (str);
2128 2138174 : if (keyword == IDENTIFIER)
2129 1616004 : return Token::make_identifier (loc, std::move (str));
2130 : else
2131 522170 : return Token::make (keyword, loc);
2132 2145390 : }
2133 :
2134 : // Possibly returns a raw string token if it exists - otherwise returns null.
2135 : TokenPtr
2136 72472 : Lexer::maybe_parse_raw_string (location_t loc)
2137 : {
2138 72472 : int peek_index = 0;
2139 72529 : while (peek_input (peek_index) == '#')
2140 57 : peek_index++;
2141 :
2142 72472 : if (peek_input (peek_index) == '"')
2143 73 : return parse_raw_string (loc, peek_index);
2144 : else
2145 72399 : return nullptr;
2146 : }
2147 :
2148 : // Returns a raw string token.
2149 : TokenPtr
2150 73 : Lexer::parse_raw_string (location_t loc, int initial_hash_count)
2151 : {
2152 : // raw string literals
2153 73 : std::string str;
2154 73 : str.reserve (16); // some sensible default
2155 :
2156 73 : int length = 1 + initial_hash_count;
2157 73 : current_column += length;
2158 :
2159 73 : const location_t string_begin_locus = get_current_location ();
2160 :
2161 73 : if (initial_hash_count > 0)
2162 55 : skip_input (initial_hash_count - 1);
2163 :
2164 73 : current_char = peek_input ();
2165 :
2166 73 : if (current_char != '"')
2167 0 : rust_error_at (get_current_location (), "raw string has no opening %<\"%>");
2168 :
2169 73 : length++;
2170 73 : current_column++;
2171 73 : skip_input ();
2172 73 : current_char = peek_input ();
2173 :
2174 18741 : while (true)
2175 : {
2176 9407 : if (current_char.value == '"')
2177 : {
2178 134 : bool enough_hashes = true;
2179 :
2180 134 : for (int i = 0; i < initial_hash_count; i++)
2181 : {
2182 61 : if (peek_input (i + 1) != '#')
2183 : {
2184 : enough_hashes = false;
2185 : break;
2186 : }
2187 : }
2188 :
2189 76 : if (enough_hashes)
2190 : {
2191 : // skip enough input and peek enough input
2192 73 : skip_input (initial_hash_count);
2193 73 : current_char = peek_input ();
2194 73 : length += initial_hash_count + 1;
2195 73 : current_column += initial_hash_count + 1;
2196 73 : break;
2197 : }
2198 : }
2199 9331 : else if (current_char.is_eof ())
2200 : {
2201 0 : rust_error_at (string_begin_locus, "unended raw string literal");
2202 0 : return Token::make (END_OF_FILE, get_current_location ());
2203 : }
2204 :
2205 9334 : length++;
2206 9334 : current_column++;
2207 9334 : if (current_char == '\n')
2208 : {
2209 241 : current_line++;
2210 241 : current_column = 1;
2211 241 : start_line (current_line, max_column_hint);
2212 : }
2213 :
2214 18668 : str += current_char.as_string ();
2215 9334 : skip_input ();
2216 9334 : current_char = peek_input ();
2217 9334 : }
2218 :
2219 73 : loc += length - 1;
2220 :
2221 73 : str.shrink_to_fit ();
2222 :
2223 73 : return Token::make_raw_string (loc, std::move (str));
2224 73 : }
2225 :
2226 : template <typename IsDigitFunc>
2227 : TokenPtr
2228 35180 : Lexer::parse_non_decimal_int_literal (location_t loc, IsDigitFunc is_digit_func,
2229 : IntegerLiteralBase base)
2230 : {
2231 35180 : std::string raw_str = "0";
2232 35180 : raw_str += current_char; // x, o, b
2233 35180 : skip_input ();
2234 :
2235 35180 : int length = 2;
2236 35180 : bool has_valid_digit = false;
2237 :
2238 35180 : current_char = peek_input ();
2239 :
2240 : // loop through to add entire number to string
2241 290955 : while (true)
2242 : {
2243 326135 : if (is_digit_func (current_char.value))
2244 : {
2245 : has_valid_digit = true;
2246 : }
2247 41468 : else if (current_char != '_')
2248 : {
2249 : break;
2250 : }
2251 290955 : length++;
2252 :
2253 290955 : raw_str += current_char;
2254 290955 : skip_input ();
2255 290955 : current_char = peek_input ();
2256 : }
2257 :
2258 35180 : int suffix_start = raw_str.length ();
2259 :
2260 : // parse in suffix if it exists
2261 35180 : auto suffix_pair = parse_in_suffix ();
2262 35180 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2263 35180 : raw_str += suffix_pair.first;
2264 35180 : length += suffix_pair.second;
2265 :
2266 35180 : current_column += length;
2267 :
2268 35180 : if (!has_valid_digit)
2269 : {
2270 5 : rust_error_at (loc, ErrorCode::E0768, "no valid digits found for number");
2271 : }
2272 :
2273 35180 : loc += length - 1;
2274 :
2275 35180 : return Token::make_int (loc, std::move (raw_str), suffix_start, base,
2276 35180 : type_hint);
2277 35180 : }
2278 :
2279 : // Parses a hex, binary or octal int literal.
2280 : TokenPtr
2281 35180 : Lexer::parse_non_decimal_int_literals (location_t loc)
2282 : {
2283 35180 : current_char = peek_input ();
2284 :
2285 35180 : if (current_char == 'x')
2286 : {
2287 : // hex (integer only)
2288 24593 : return parse_non_decimal_int_literal (loc, is_x_digit,
2289 24593 : IntegerLiteralBase::Hex);
2290 : }
2291 10587 : else if (current_char == 'o')
2292 : {
2293 : // octal (integer only)
2294 19 : return parse_non_decimal_int_literal (loc, is_octal_digit,
2295 19 : IntegerLiteralBase::Octal);
2296 : }
2297 10568 : else if (current_char == 'b')
2298 : {
2299 : // binary (integer only)
2300 10568 : return parse_non_decimal_int_literal (loc, is_bin_digit,
2301 10568 : IntegerLiteralBase::Binary);
2302 : }
2303 : else
2304 : {
2305 0 : return nullptr;
2306 : }
2307 : }
2308 :
2309 : // Parses a decimal-based int literal or float literal.
2310 : TokenPtr
2311 396192 : Lexer::parse_decimal_int_or_float (location_t loc)
2312 : {
2313 396192 : std::string str;
2314 396192 : str.reserve (16); // some sensible default
2315 396192 : str += current_char;
2316 :
2317 396192 : int length = 1;
2318 396192 : bool first_zero = current_char == '0';
2319 :
2320 396192 : current_char = peek_input ();
2321 :
2322 : // parse initial decimal integer (or first integer part of float) literal
2323 396192 : auto initial_decimal = parse_in_decimal ();
2324 396192 : str += std::get<0> (initial_decimal);
2325 396192 : length += std::get<1> (initial_decimal);
2326 :
2327 : // detect float literal
2328 : //
2329 : // Note:
2330 : //
2331 : // We should not use is_float_digit () for this verification but instead
2332 : // directly ISDIGIT because rust does not support non digit values right after
2333 : // a dot.
2334 : // The following value is not legal in rust:
2335 : // let a = 3.e1;
2336 : // A `0` should be put between the dot and the exponent to be valid
2337 : // (eg. 3.0e1).
2338 396192 : if (current_char == '.' && ISDIGIT (peek_input (1).value))
2339 : {
2340 : // float with a '.', parse another decimal into it
2341 :
2342 : // add . to str
2343 25365 : str += current_char;
2344 25365 : skip_input ();
2345 25365 : current_char = peek_input ();
2346 25365 : length++;
2347 :
2348 : // parse another decimal number for float
2349 25365 : auto second_decimal = parse_in_decimal ();
2350 25365 : str += std::get<0> (second_decimal);
2351 25365 : length += std::get<1> (second_decimal);
2352 :
2353 : // parse in exponent part if it exists
2354 25365 : auto exponent_pair = parse_in_exponent_part ();
2355 25365 : str += exponent_pair.first;
2356 25365 : length += exponent_pair.second;
2357 :
2358 25365 : int suffix_start = str.length ();
2359 :
2360 : // parse in type suffix if it exists
2361 25365 : auto suffix_pair = parse_in_suffix ();
2362 25365 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2363 25365 : str += suffix_pair.first;
2364 25365 : length += suffix_pair.second;
2365 :
2366 25365 : current_column += length;
2367 :
2368 25365 : loc += length - 1;
2369 :
2370 25365 : str.shrink_to_fit ();
2371 25365 : return Token::make_float (loc, std::move (str), suffix_start, type_hint);
2372 25365 : }
2373 370827 : else if (current_char == '.'
2374 370827 : && check_valid_float_dot_end (peek_input (1).value))
2375 : {
2376 : // float that is just an integer with a terminating '.' character
2377 :
2378 : // add . to str
2379 82459 : str += current_char;
2380 82459 : skip_input ();
2381 82459 : current_char = peek_input ();
2382 82459 : length++;
2383 :
2384 : // type hint not allowed
2385 :
2386 82459 : current_column += length;
2387 :
2388 82459 : loc += length - 1;
2389 :
2390 82459 : str.shrink_to_fit ();
2391 82459 : auto suffix_start = str.length ();
2392 82459 : return Token::make_float (loc, std::move (str), suffix_start,
2393 82459 : CORETYPE_UNKNOWN);
2394 : }
2395 288368 : else if (current_char == 'E' || current_char == 'e')
2396 : {
2397 : // exponent float with no '.' character
2398 :
2399 : // parse exponent part
2400 456 : auto exponent_pair = parse_in_exponent_part ();
2401 456 : str += exponent_pair.first;
2402 456 : length += exponent_pair.second;
2403 :
2404 456 : int suffix_start = str.length ();
2405 :
2406 : // parse in type suffix if it exists
2407 456 : auto suffix_pair = parse_in_suffix ();
2408 456 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2409 456 : str += suffix_pair.first;
2410 456 : length += suffix_pair.second;
2411 :
2412 456 : current_column += length;
2413 :
2414 456 : loc += length - 1;
2415 :
2416 456 : str.shrink_to_fit ();
2417 456 : return Token::make_float (loc, std::move (str), suffix_start, type_hint);
2418 456 : }
2419 : else
2420 : {
2421 : // is an integer
2422 :
2423 287912 : int suffix_start = str.length ();
2424 :
2425 : // parse in type suffix if it exists
2426 287912 : auto suffix_pair = parse_in_suffix ();
2427 287912 : str += suffix_pair.first;
2428 :
2429 287912 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2430 :
2431 : /* A "real" pure decimal doesn't have a suffix and no zero prefix. */
2432 287912 : bool pure_decimal = std::get<2> (initial_decimal);
2433 287184 : if (pure_decimal && (!first_zero || suffix_start == 1)
2434 575094 : && suffix_pair.first.empty ())
2435 : type_hint = CORETYPE_PURE_DECIMAL;
2436 :
2437 287912 : length += suffix_pair.second;
2438 :
2439 287912 : current_column += length;
2440 :
2441 287912 : loc += length - 1;
2442 :
2443 287912 : str.shrink_to_fit ();
2444 287912 : return Token::make_int (loc, std::move (str), suffix_start,
2445 287912 : IntegerLiteralBase::Decimal, type_hint);
2446 287912 : }
2447 396192 : }
2448 :
2449 : TokenPtr
2450 111629 : Lexer::parse_char_or_lifetime (location_t loc)
2451 : {
2452 111629 : int length = 1;
2453 :
2454 111629 : current_char = peek_input ();
2455 111629 : if (current_char.is_eof ())
2456 0 : return nullptr;
2457 :
2458 : // parse escaped char literal
2459 111629 : if (current_char.value == '\\')
2460 : {
2461 : // parse escape
2462 91401 : auto utf8_escape_pair = parse_utf8_escape ();
2463 91401 : Codepoint escaped_char = std::get<0> (utf8_escape_pair);
2464 91401 : length += std::get<1> (utf8_escape_pair);
2465 :
2466 91401 : if (peek_input ().value != '\'')
2467 : {
2468 0 : rust_error_at (get_current_location (), "unended character literal");
2469 : }
2470 : else
2471 : {
2472 91401 : skip_input ();
2473 91401 : current_char = peek_input ();
2474 91401 : length++;
2475 : }
2476 :
2477 91401 : current_column += length;
2478 :
2479 91401 : loc += length - 1;
2480 :
2481 91401 : return Token::make_char (loc, escaped_char);
2482 : }
2483 : else
2484 : {
2485 20228 : skip_input ();
2486 :
2487 20228 : if (peek_input ().value == '\'')
2488 : {
2489 : // parse non-escaped char literal
2490 1885 : Codepoint non_escaped_char = current_char;
2491 :
2492 : // skip the ' character
2493 1885 : skip_input ();
2494 1885 : current_char = peek_input ();
2495 :
2496 : // TODO fix due to different widths of utf-8 chars?
2497 1885 : current_column += 3;
2498 :
2499 1885 : loc += 2;
2500 :
2501 1885 : return Token::make_char (loc, non_escaped_char);
2502 : }
2503 18343 : else if (is_identifier_start (current_char.value))
2504 : {
2505 : // parse lifetime name
2506 18343 : std::string str;
2507 36686 : str += current_char.as_string ();
2508 18343 : length++;
2509 :
2510 18343 : current_char = peek_input ();
2511 39802 : while (is_identifier_continue (current_char.value))
2512 : {
2513 6232 : str += current_char.as_string ();
2514 3116 : skip_input ();
2515 3116 : current_char = peek_input ();
2516 3116 : length++;
2517 : }
2518 :
2519 18343 : current_column += length;
2520 :
2521 18343 : loc += length - 1;
2522 :
2523 : // TODO some keywords cannot be used for a lifetime label #2306
2524 : // https://doc.rust-lang.org/reference/tokens.html
2525 :
2526 18343 : str.shrink_to_fit ();
2527 18343 : return Token::make_lifetime (loc, std::move (str));
2528 18343 : }
2529 : else
2530 : {
2531 0 : rust_error_at (
2532 : get_current_location (),
2533 : "expected %' after character constant in character literal");
2534 0 : return nullptr;
2535 : }
2536 : }
2537 : }
2538 :
2539 : void
2540 2086 : Lexer::split_current_token (TokenId new_left, TokenId new_right)
2541 : {
2542 : /* TODO: assert that this TokenId is a "simple token" like punctuation and not
2543 : * like "IDENTIFIER"? */
2544 2086 : location_t current_loc = peek_token ()->get_locus ();
2545 2086 : TokenPtr new_left_tok = Token::make (new_left, current_loc);
2546 2086 : TokenPtr new_right_tok = Token::make (new_right, current_loc + 1);
2547 :
2548 2086 : token_queue.replace_current_value (std::move (new_left_tok));
2549 2086 : token_queue.insert (1, std::move (new_right_tok));
2550 2086 : }
2551 :
2552 : void
2553 2 : Lexer::split_current_token (std::vector<TokenPtr> new_tokens)
2554 : {
2555 2 : rust_assert (new_tokens.size () > 0);
2556 4 : token_queue.replace_current_value (new_tokens[0]);
2557 :
2558 5 : for (size_t i = 1; i < new_tokens.size (); i++)
2559 : {
2560 6 : token_queue.insert (i, new_tokens[i]);
2561 : }
2562 2 : }
2563 :
2564 : void
2565 1375541 : Lexer::start_line (int current_line, int current_column)
2566 : {
2567 1375541 : if (line_map)
2568 1375541 : linemap_line_start (line_table, current_line, current_column);
2569 1375541 : }
2570 :
2571 : } // namespace Rust
2572 :
2573 : #if CHECKING_P
2574 :
2575 : namespace selftest {
2576 :
2577 : // Checks if `src` has the same contents as the given characters
2578 : static void
2579 6 : assert_source_content (Rust::InputSource &src,
2580 : const std::vector<uint32_t> &expected)
2581 : {
2582 6 : Rust::Codepoint src_char = src.next ();
2583 41 : for (auto expected_char : expected)
2584 : {
2585 : // Make sure that `src` is not shorter than `expected`
2586 35 : ASSERT_FALSE (src_char.is_eof ());
2587 : // Checks skipped character is expeceted one.
2588 35 : ASSERT_EQ (src_char.value, expected_char);
2589 35 : src_char = src.next ();
2590 : }
2591 : // Checks if `src` and `chars` has the same length.
2592 6 : ASSERT_TRUE (src_char.is_eof ());
2593 6 : }
2594 :
2595 : static void
2596 4 : test_buffer_input_source (std::string str,
2597 : const std::vector<uint32_t> &expected)
2598 : {
2599 4 : Rust::BufferInputSource source (str, 0);
2600 4 : assert_source_content (source, expected);
2601 4 : }
2602 :
2603 : static void
2604 2 : test_file_input_source (std::string str, const std::vector<uint32_t> &expected)
2605 : {
2606 2 : FILE *tmpf = tmpfile ();
2607 : // Moves to the first character
2608 2 : fputs (str.c_str (), tmpf);
2609 2 : std::rewind (tmpf);
2610 2 : Rust::FileInputSource source (tmpf);
2611 2 : assert_source_content (source, expected);
2612 2 : }
2613 :
2614 : void
2615 1 : rust_input_source_test ()
2616 : {
2617 : // ASCII
2618 1 : std::string src = (const char *) u8"_abcde\tXYZ\v\f";
2619 1 : std::vector<uint32_t> expected = {u'_', u'a', u'b', u'c', u'd', u'e',
2620 1 : u'\t', u'X', u'Y', u'Z', u'\v', u'\f'};
2621 2 : test_buffer_input_source (src, expected);
2622 :
2623 : // BOM
2624 1 : src = (const char *) u8"\xef\xbb\xbfOK";
2625 1 : expected = {u'O', u'K'};
2626 2 : test_buffer_input_source (src, expected);
2627 :
2628 : // Russian
2629 1 : src = (const char *) u8"приве́т";
2630 1 : expected = {u'п',
2631 : u'р',
2632 : u'и',
2633 : u'в',
2634 : 0x0435 /* CYRILLIC SMALL LETTER IE е */,
2635 : 0x301 /* COMBINING ACUTE ACCENT ́ */,
2636 1 : u'т'};
2637 2 : test_buffer_input_source (src, expected);
2638 :
2639 1 : src = (const char *) u8"❤️🦀";
2640 1 : expected = {0x2764 /* HEAVY BLACK HEART */,
2641 1 : 0xfe0f /* VARIATION SELECTOR-16 */, U'🦀'};
2642 2 : test_buffer_input_source (src, expected);
2643 :
2644 1 : src = (const char *) u8"こんにちは";
2645 1 : expected = {u'こ', u'ん', u'に', u'ち', u'は'};
2646 2 : test_file_input_source (src, expected);
2647 :
2648 1 : src = (const char *) u8"👮♂👩⚕";
2649 1 : expected
2650 : = {0x1f46e /* POLICE OFFICER */, 0x200d /* ZERO WIDTH JOINER */,
2651 : 0x2642 /* MALE SIGN */, 0x1f469 /* WOMAN */,
2652 1 : 0x200d /* ZERO WIDTH JOINER */, 0x2695 /* STAFF OF AESCULAPIUS */};
2653 2 : test_file_input_source (src, expected);
2654 1 : }
2655 :
2656 : } // namespace selftest
2657 :
2658 : #endif // CHECKING_P
|