Line data Source code
1 : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
2 :
3 : // This file is part of GCC.
4 :
5 : // GCC is free software; you can redistribute it and/or modify it under
6 : // the terms of the GNU General Public License as published by the Free
7 : // Software Foundation; either version 3, or (at your option) any later
8 : // version.
9 :
10 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 : // for more details.
14 :
15 : // You should have received a copy of the GNU General Public License
16 : // along with GCC; see the file COPYING3. If not see
17 : // <http://www.gnu.org/licenses/>.
18 :
19 : #include "rust-codepoint.h"
20 : #include "rust-system.h"
21 : #include "rust-lex.h"
22 : #include "rust-diagnostics.h"
23 : #include "rust-linemap.h"
24 : #include "rust-edition.h"
25 : #include "safe-ctype.h"
26 : #include "cpplib.h"
27 : #include "rust-keyword-values.h"
28 :
29 : namespace Rust {
30 : // TODO: move to separate compilation unit?
31 : // overload += for uint32_t to allow 32-bit encoded utf-8 to be added
32 : std::string &
33 3324685 : operator+= (std::string &str, Codepoint char32)
34 : {
35 3324685 : if (char32.value < 0x80)
36 : {
37 3323672 : str += static_cast<char> (char32.value);
38 : }
39 1013 : else if (char32.value < (0x1F + 1) << (1 * 6))
40 : {
41 674 : str += static_cast<char> (0xC0 | ((char32.value >> 6) & 0x1F));
42 674 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
43 : }
44 339 : else if (char32.value < (0x0F + 1) << (2 * 6))
45 : {
46 329 : str += static_cast<char> (0xE0 | ((char32.value >> 12) & 0x0F));
47 329 : str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
48 329 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
49 : }
50 10 : else if (char32.value < (0x07 + 1) << (3 * 6))
51 : {
52 6 : str += static_cast<char> (0xF0 | ((char32.value >> 18) & 0x07));
53 6 : str += static_cast<char> (0x80 | ((char32.value >> 12) & 0x3F));
54 6 : str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
55 6 : str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
56 : }
57 : else
58 : {
59 4 : rust_debug ("Invalid unicode codepoint found: '%u' ", char32.value);
60 : }
61 3324685 : return str;
62 : }
63 :
64 : std::string
65 3033896 : Codepoint::as_string ()
66 : {
67 3033896 : std::string str;
68 :
69 : // str += Codepoint (value);
70 3033896 : str += *this;
71 :
72 3033896 : return str;
73 : }
74 :
75 : /* Includes all allowable float digits EXCEPT _ and . as that needs lookahead
76 : * for handling. */
77 : bool
78 0 : is_float_digit (uint32_t number)
79 : {
80 0 : return ISDIGIT (number) || number == 'E' || number == 'e';
81 : }
82 :
83 : /* Basically ISXDIGIT from safe-ctype but may change if Rust's encoding or
84 : * whatever is different */
85 : bool
86 2695 : is_x_digit (uint32_t number)
87 : {
88 2695 : return ISXDIGIT (number);
89 : }
90 :
91 : bool
92 87 : is_octal_digit (uint32_t number)
93 : {
94 87 : return number >= '0' && number <= '7';
95 : }
96 :
97 : bool
98 276 : is_bin_digit (uint32_t number)
99 : {
100 276 : return number == '0' || number == '1';
101 : }
102 :
103 : bool
104 141 : check_valid_float_dot_end (uint32_t character)
105 : {
106 141 : return character != '.' && character != '_' && !ISALPHA (character);
107 : }
108 :
109 : bool
110 3706 : is_whitespace (uint32_t character)
111 : {
112 : // https://doc.rust-lang.org/reference/whitespace.html
113 3706 : switch (character)
114 : {
115 : case '\t':
116 : case '\n':
117 : case '\v':
118 : case '\f':
119 : case '\r':
120 : case ' ':
121 : case 0x0085: // next line
122 : case 0x200e: // left-to-right mark
123 : case 0x200f: // right-to-left mark
124 : case 0x2028: // line separator
125 : case 0x2029: // paragraph separator
126 : return true;
127 3363 : default:
128 3363 : return false;
129 : }
130 : }
131 :
132 : bool
133 4447 : is_non_decimal_int_literal_separator (uint32_t character)
134 : {
135 4447 : return character == 'x' || character == 'o' || character == 'b';
136 : }
137 :
138 : bool
139 326378 : is_identifier_start (uint32_t codepoint)
140 : {
141 326378 : return (cpp_check_xid_property (codepoint) & CPP_XID_START)
142 326378 : || codepoint == '_';
143 : }
144 :
145 : bool
146 1201520 : is_identifier_continue (uint32_t codepoint)
147 : {
148 1201520 : return cpp_check_xid_property (codepoint) & CPP_XID_CONTINUE;
149 : }
150 :
151 104 : Lexer::Lexer (const std::string &input, Linemap *linemap)
152 104 : : input (RAIIFile::create_error ()), current_line (1), current_column (1),
153 104 : line_map (linemap), dump_lex_out ({}),
154 104 : raw_input_source (new BufferInputSource (input, 0)),
155 104 : input_queue{*raw_input_source}, token_queue (TokenSource (this))
156 104 : {}
157 :
158 4964 : Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap,
159 4964 : tl::optional<std::ofstream &> dump_lex_opt)
160 4964 : : input (std::move (file_input)), current_line (1), current_column (1),
161 4964 : line_map (linemap), dump_lex_out (dump_lex_opt),
162 9928 : raw_input_source (new FileInputSource (input.get_raw ())),
163 9928 : input_queue{*raw_input_source}, token_queue (TokenSource (this))
164 : {
165 : // inform line_table that file is being entered and is in line 1
166 4964 : if (linemap)
167 4964 : line_map->start_file (filename, current_line);
168 4964 : }
169 :
170 5066 : Lexer::~Lexer ()
171 : {
172 : /* ok apparently stop (which is equivalent of original code in destructor) is
173 : * meant to be called after all files have finished parsing, for cleanup. On
174 : * the other hand, actual code that it calls to leave a certain line map is
175 : * mentioned in GCC docs as being useful for "just leaving an included header"
176 : * and stuff like that, so this line mapping functionality may need fixing.
177 : * FIXME: find out whether this occurs. */
178 :
179 : // line_map->stop();
180 5066 : }
181 :
182 : bool
183 4909 : Lexer::input_source_is_valid_utf8 ()
184 : {
185 4909 : return raw_input_source->is_valid ();
186 : }
187 :
188 : location_t
189 1922227 : Lexer::get_current_location ()
190 : {
191 1922227 : if (line_map)
192 1922058 : return linemap_position_for_column (line_table, current_column);
193 : else
194 : // If we have no linemap, we're lexing something without proper locations
195 : return UNDEF_LOCATION;
196 : }
197 :
198 : Codepoint
199 4327206 : Lexer::peek_input (int n)
200 : {
201 4327206 : return input_queue.peek (n);
202 : }
203 :
204 : Codepoint
205 4228535 : Lexer::peek_input ()
206 : {
207 4228535 : return peek_input (0);
208 : }
209 :
210 : void
211 3629503 : Lexer::skip_input (int n)
212 : {
213 3629503 : input_queue.skip (n);
214 3629503 : }
215 :
216 : void
217 3619184 : Lexer::skip_input ()
218 : {
219 3619184 : skip_input (0);
220 3619184 : }
221 :
222 : void
223 761298 : Lexer::skip_token (int n)
224 : {
225 : // dump tokens if dump-lex option is enabled
226 761298 : if (dump_lex_out.has_value ())
227 55 : dump_and_skip (n);
228 : else
229 761243 : token_queue.skip (n);
230 761298 : }
231 :
232 : void
233 55 : Lexer::dump_and_skip (int n)
234 : {
235 55 : std::ofstream &out = dump_lex_out.value ();
236 55 : bool found_eof = false;
237 55 : const_TokenPtr tok;
238 110 : for (int i = 0; i < n + 1; i++)
239 : {
240 55 : if (!found_eof)
241 : {
242 55 : tok = peek_token ();
243 55 : found_eof |= tok->get_id () == Rust::END_OF_FILE;
244 :
245 55 : location_t loc = tok->get_locus ();
246 :
247 55 : out << "<id=";
248 55 : out << tok->token_id_to_str ();
249 55 : out << (tok->should_have_str ()
250 167 : ? (std::string (", text=") + tok->get_str ()
251 167 : + std::string (", typehint=")
252 93 : + std::string (tok->get_type_hint_str ()))
253 91 : : "")
254 110 : << " ";
255 110 : out << Linemap::location_to_string (loc) << '\n';
256 : }
257 :
258 55 : token_queue.skip (0);
259 : }
260 55 : }
261 :
262 : void
263 0 : Lexer::replace_current_token (TokenPtr replacement)
264 : {
265 0 : token_queue.replace_current_value (replacement);
266 :
267 0 : rust_debug ("called 'replace_current_token' - this is deprecated");
268 0 : }
269 :
270 : /* Determines whether the string passed in is a keyword or not. If it is, it
271 : * returns the keyword name. */
272 : TokenId
273 292835 : Lexer::classify_keyword (const std::string &str)
274 : {
275 292835 : auto &keywords = Rust::Values::Keywords::keywords_tokens;
276 292835 : auto keyword = keywords.find (str);
277 :
278 292835 : if (keyword == keywords.end ())
279 : return IDENTIFIER;
280 :
281 98251 : auto id = keyword->second;
282 :
283 : // We now have the expected token ID of the reserved keyword. However, some
284 : // keywords are reserved starting in certain editions. For example, `try` is
285 : // only a reserved keyword in editions >=2018. The language might gain new
286 : // reserved keywords in the future.
287 : //
288 : // https://doc.rust-lang.org/reference/keywords.html#reserved-keywords
289 :
290 : // `try` is not a reserved keyword before 2018
291 98251 : if (get_rust_edition () == Edition::E2015 && id == TRY)
292 : return IDENTIFIER;
293 :
294 : return id;
295 : }
296 :
297 : TokenPtr
298 769702 : Lexer::build_token ()
299 : {
300 : // loop to go through multiple characters to build a single token
301 1909133 : while (true)
302 : {
303 1909133 : location_t loc = get_current_location ();
304 :
305 1909133 : current_char = peek_input ();
306 1909133 : skip_input ();
307 :
308 : // detect shebang
309 : // Must be the first thing on the first line, starting with #!
310 : // But since an attribute can also start with an #! we don't count it as a
311 : // shebang line when after any whitespace or comments there is a [. If it
312 : // is a shebang line we simple drop the line. Otherwise we don't consume
313 : // any characters and fall through to the real tokenizer.
314 33416 : if (current_line == 1 && current_column == 1 && current_char == '#'
315 1942549 : && peek_input () == '!')
316 : {
317 : int n = 1;
318 3334 : while (true)
319 : {
320 3334 : Codepoint next_char = peek_input (n);
321 3334 : if (is_whitespace (next_char.value))
322 7 : n++;
323 3327 : else if ((next_char == '/' && peek_input (n + 1) == '/'
324 7 : && peek_input (n + 2) != '!'
325 7 : && peek_input (n + 2) != '/')
326 3348 : || (next_char == '/' && peek_input (n + 1) == '/'
327 0 : && peek_input (n + 2) == '/'
328 0 : && peek_input (n + 3) == '/'))
329 : {
330 : // two // or four ////
331 : // A single line comment
332 : // (but not an inner or outer doc comment)
333 7 : n += 2;
334 7 : next_char = peek_input (n);
335 119 : while (next_char != '\n' && !next_char.is_eof ())
336 : {
337 112 : n++;
338 112 : next_char = peek_input (n);
339 : }
340 7 : if (next_char == '\n')
341 7 : n++;
342 : }
343 3320 : else if (next_char == '/' && peek_input (n + 1) == '*'
344 0 : && peek_input (n + 2) == '*'
345 3320 : && peek_input (n + 3) == '/')
346 : {
347 : /**/
348 0 : n += 4;
349 : }
350 3320 : else if (next_char == '/' && peek_input (n + 1) == '*'
351 0 : && peek_input (n + 2) == '*' && peek_input (n + 3) == '*'
352 3320 : && peek_input (n + 4) == '/')
353 : {
354 : /***/
355 0 : n += 5;
356 : }
357 3320 : else if ((next_char == '/' && peek_input (n + 1) == '*'
358 0 : && peek_input (n + 2) != '*'
359 0 : && peek_input (n + 2) != '!')
360 3341 : || (next_char == '/' && peek_input (n + 1) == '*'
361 0 : && peek_input (n + 2) == '*'
362 0 : && peek_input (n + 3) == '*'))
363 : {
364 : // one /* or three /***
365 : // Start of a block comment
366 : // (but not an inner or outer doc comment)
367 0 : n += 2;
368 0 : int level = 1;
369 0 : while (level > 0)
370 : {
371 0 : if (peek_input (n).is_eof ())
372 : break;
373 0 : else if (peek_input (n) == '/'
374 0 : && peek_input (n + 1) == '*')
375 : {
376 0 : n += 2;
377 0 : level += 1;
378 : }
379 0 : else if (peek_input (n) == '*'
380 0 : && peek_input (n + 1) == '/')
381 : {
382 0 : n += 2;
383 0 : level -= 1;
384 : }
385 : else
386 0 : n++;
387 : }
388 : }
389 3320 : else if (next_char != '[')
390 : {
391 : // definitely shebang, ignore the first line
392 518 : while (current_char != '\n' && !current_char.is_eof ())
393 : {
394 490 : current_char = peek_input ();
395 490 : skip_input ();
396 : }
397 :
398 : // newline
399 28 : current_line++;
400 28 : current_column = 1;
401 : // tell line_table that new line starts
402 28 : start_line (current_line, max_column_hint);
403 28 : break;
404 : }
405 : else
406 : break; /* Definitely not a shebang line. */
407 : }
408 : }
409 :
410 : // return end of file token if end of file
411 1909133 : if (current_char.is_eof ())
412 5367 : return Token::make (END_OF_FILE, loc);
413 :
414 : // if not end of file, start tokenising
415 1903766 : switch (current_char.value)
416 : {
417 : /* ignore whitespace characters for tokens but continue updating
418 : * location */
419 168829 : case '\n': // newline
420 168829 : case 0x0085: // next line
421 168829 : case 0x2028: // line separator
422 168829 : case 0x2029: // paragraph separator
423 168829 : current_line++;
424 168829 : current_column = 1;
425 : // tell line_table that new line starts
426 168829 : start_line (current_line, max_column_hint);
427 168829 : continue;
428 252 : case '\r': // cr
429 : // Ignore, we expect a newline (lf) soon.
430 252 : continue;
431 960273 : case ' ': // space
432 960273 : current_column++;
433 960273 : continue;
434 114 : case '\t': // horizontal tab
435 : // width of a tab is not well-defined, assume 8 spaces
436 114 : current_column += 8;
437 114 : continue;
438 28 : case '\v': // vertical tab
439 28 : case 0x000c: // form feed
440 28 : case 0x200e: // left-to-right mark
441 28 : case 0x200f: // right-to-left mark
442 : // Ignored.
443 28 : continue;
444 :
445 : // punctuation - actual tokens
446 28546 : case '=':
447 28546 : if (peek_input () == '>')
448 : {
449 : // match arm arrow
450 3314 : skip_input ();
451 3314 : current_column += 2;
452 3314 : loc += 1;
453 :
454 3314 : return Token::make (MATCH_ARROW, loc);
455 : }
456 25232 : else if (peek_input () == '=')
457 : {
458 : // equality operator
459 676 : skip_input ();
460 676 : current_column += 2;
461 676 : loc += 1;
462 :
463 676 : return Token::make (EQUAL_EQUAL, loc);
464 : }
465 : else
466 : {
467 : // assignment operator
468 24556 : current_column++;
469 24556 : return Token::make (EQUAL, loc);
470 : }
471 48243 : case '(':
472 48243 : current_column++;
473 48243 : return Token::make (LEFT_PAREN, loc);
474 12287 : case '-':
475 12287 : if (peek_input () == '>')
476 : {
477 : // return type specifier
478 10840 : skip_input ();
479 10840 : current_column += 2;
480 10840 : loc += 1;
481 :
482 10840 : return Token::make (RETURN_TYPE, loc);
483 : }
484 1447 : else if (peek_input () == '=')
485 : {
486 : // minus-assign
487 105 : skip_input ();
488 105 : current_column += 2;
489 105 : loc += 1;
490 :
491 105 : return Token::make (MINUS_EQ, loc);
492 : }
493 : else
494 : {
495 : // minus
496 1342 : current_column++;
497 1342 : return Token::make (MINUS, loc);
498 : }
499 1761 : case '+':
500 1761 : if (peek_input () == '=')
501 : {
502 : // add-assign
503 155 : skip_input ();
504 155 : current_column += 2;
505 155 : loc += 1;
506 :
507 155 : return Token::make (PLUS_EQ, loc);
508 : }
509 : else
510 : {
511 : // add
512 1606 : current_column++;
513 1606 : return Token::make (PLUS, loc);
514 : }
515 48224 : case ')':
516 48224 : current_column++;
517 48224 : return Token::make (RIGHT_PAREN, loc);
518 31288 : case ';':
519 31288 : current_column++;
520 31288 : return Token::make (SEMICOLON, loc);
521 11052 : case '*':
522 11052 : if (peek_input () == '=')
523 : {
524 : // multiplication-assign
525 7 : skip_input ();
526 7 : current_column += 2;
527 7 : loc += 1;
528 :
529 7 : return Token::make (ASTERISK_EQ, loc);
530 : }
531 : else
532 : {
533 : // multiplication
534 11045 : current_column++;
535 11045 : return Token::make (ASTERISK, loc);
536 : }
537 23175 : case ',':
538 23175 : current_column++;
539 23175 : return Token::make (COMMA, loc);
540 18078 : case '/':
541 18078 : if (peek_input () == '=')
542 : {
543 : // division-assign
544 7 : skip_input ();
545 7 : current_column += 2;
546 7 : loc += 1;
547 :
548 7 : return Token::make (DIV_EQ, loc);
549 : }
550 18071 : else if ((peek_input () == '/' && peek_input (1) != '!'
551 16953 : && peek_input (1) != '/')
552 26071 : || (peek_input () == '/' && peek_input (1) == '/'
553 7900 : && peek_input (2) == '/'))
554 : {
555 : // two // or four ////
556 : // single line comment
557 : // (but not an inner or outer doc comment)
558 9068 : skip_input ();
559 9068 : current_column += 2;
560 9068 : current_char = peek_input ();
561 :
562 : // basically ignore until line finishes
563 445641 : while (current_char != '\n' && !current_char.is_eof ())
564 : {
565 427505 : skip_input ();
566 427505 : current_column++; // not used
567 427505 : current_char = peek_input ();
568 : }
569 9068 : continue;
570 : }
571 9003 : else if (peek_input () == '/'
572 9003 : && (peek_input (1) == '!' || peek_input (1) == '/'))
573 : {
574 : /* single line doc comment, inner or outer. */
575 7985 : bool is_inner = peek_input (1) == '!';
576 7985 : skip_input (1);
577 7985 : current_column += 3;
578 :
579 7985 : std::string str;
580 7985 : str.reserve (32);
581 7985 : current_char = peek_input ();
582 192537 : while (current_char != '\n')
583 : {
584 176616 : skip_input ();
585 176616 : if (current_char == '\r')
586 : {
587 51 : Codepoint next_char = peek_input ();
588 51 : if (next_char == '\n')
589 : {
590 49 : current_char = '\n';
591 49 : break;
592 : }
593 2 : rust_error_at (
594 : loc, "Isolated CR %<\\r%> not allowed in doc comment");
595 2 : current_char = next_char;
596 2 : continue;
597 2 : }
598 176565 : if (current_char.is_eof ())
599 : {
600 0 : rust_error_at (
601 : loc, ErrorCode::E0758,
602 : "unexpected EOF while looking for end of comment");
603 0 : break;
604 : }
605 176565 : str += current_char;
606 176565 : current_char = peek_input ();
607 : }
608 7985 : skip_input ();
609 7985 : current_line++;
610 7985 : current_column = 1;
611 : // tell line_table that new line starts
612 7985 : start_line (current_line, max_column_hint);
613 :
614 7985 : str.shrink_to_fit ();
615 :
616 7985 : loc += str.size () - 1;
617 7985 : if (is_inner)
618 100 : return Token::make_inner_doc_comment (loc, std::move (str));
619 : else
620 7885 : return Token::make_outer_doc_comment (loc, std::move (str));
621 7985 : }
622 1018 : else if (peek_input () == '*' && peek_input (1) == '*'
623 1103 : && peek_input (2) == '/')
624 : {
625 : /**/
626 14 : skip_input (2);
627 14 : current_column += 4;
628 14 : continue;
629 : }
630 1004 : else if (peek_input () == '*' && peek_input (1) == '*'
631 1075 : && peek_input (2) == '*' && peek_input (3) == '/')
632 : {
633 : /***/
634 14 : skip_input (3);
635 14 : current_column += 5;
636 14 : continue;
637 : }
638 990 : else if ((peek_input () == '*' && peek_input (1) != '!'
639 881 : && peek_input (1) != '*')
640 1120 : || (peek_input () == '*' && peek_input (1) == '*'
641 57 : && peek_input (2) == '*'))
642 : {
643 : // one /* or three /***
644 : // block comment
645 : // (but not an inner or outer doc comment)
646 838 : skip_input ();
647 838 : current_column += 2;
648 :
649 838 : int level = 1;
650 37904 : while (level > 0)
651 : {
652 37067 : current_char = peek_input ();
653 :
654 37067 : if (current_char.is_eof ())
655 : {
656 1 : rust_error_at (
657 : loc, ErrorCode::E0758,
658 : "unexpected EOF while looking for end of comment");
659 1 : break;
660 : }
661 :
662 : // if /* found
663 37066 : if (current_char == '/' && peek_input (1) == '*')
664 : {
665 : // skip /* characters
666 49 : skip_input (1);
667 :
668 49 : current_column += 2;
669 :
670 49 : level += 1;
671 49 : continue;
672 : }
673 :
674 : // ignore until */ is found
675 37017 : if (current_char == '*' && peek_input (1) == '/')
676 : {
677 : // skip */ characters
678 886 : skip_input (1);
679 :
680 886 : current_column += 2;
681 :
682 886 : level -= 1;
683 886 : continue;
684 : }
685 :
686 36131 : if (current_char == '\n')
687 : {
688 414 : skip_input ();
689 414 : current_line++;
690 414 : current_column = 1;
691 : // tell line_table that new line starts
692 414 : start_line (current_line, max_column_hint);
693 414 : continue;
694 : }
695 :
696 35717 : skip_input ();
697 35717 : current_column++;
698 : }
699 :
700 : // refresh new token
701 838 : continue;
702 838 : }
703 152 : else if (peek_input () == '*'
704 152 : && (peek_input (1) == '!' || peek_input (1) == '*'))
705 : {
706 : // block doc comment, inner /*! or outer /**
707 116 : bool is_inner = peek_input (1) == '!';
708 116 : skip_input (1);
709 116 : current_column += 3;
710 :
711 116 : std::string str;
712 116 : str.reserve (96);
713 :
714 116 : int level = 1;
715 116 : while (level > 0)
716 : {
717 2685 : current_char = peek_input ();
718 :
719 2685 : if (current_char.is_eof ())
720 : {
721 0 : rust_error_at (
722 : loc, ErrorCode::E0758,
723 : "unexpected EOF while looking for end of comment");
724 0 : break;
725 : }
726 :
727 : // if /* found
728 2685 : if (current_char == '/' && peek_input (1) == '*')
729 : {
730 : // skip /* characters
731 84 : skip_input (1);
732 84 : current_column += 2;
733 :
734 84 : level += 1;
735 84 : str += "/*";
736 84 : continue;
737 : }
738 :
739 : // ignore until */ is found
740 2601 : if (current_char == '*' && peek_input (1) == '/')
741 : {
742 : // skip */ characters
743 200 : skip_input (1);
744 200 : current_column += 2;
745 :
746 200 : level -= 1;
747 200 : if (level > 0)
748 84 : str += "*/";
749 200 : continue;
750 : }
751 :
752 2401 : if (current_char == '\r' && peek_input (1) != '\n')
753 2 : rust_error_at (
754 : loc, "Isolated CR %<\\r%> not allowed in doc comment");
755 :
756 2401 : if (current_char == '\n')
757 : {
758 0 : skip_input ();
759 0 : current_line++;
760 0 : current_column = 1;
761 : // tell line_table that new line starts
762 0 : start_line (current_line, max_column_hint);
763 0 : str += '\n';
764 0 : continue;
765 : }
766 :
767 2401 : str += current_char;
768 2401 : skip_input ();
769 2401 : current_column++;
770 : }
771 :
772 116 : str.shrink_to_fit ();
773 :
774 116 : loc += str.size () - 1;
775 116 : if (is_inner)
776 73 : return Token::make_inner_doc_comment (loc, std::move (str));
777 : else
778 43 : return Token::make_outer_doc_comment (loc, std::move (str));
779 116 : }
780 : else
781 : {
782 : // division
783 36 : current_column++;
784 36 : return Token::make (DIV, loc);
785 : }
786 43 : case '%':
787 43 : if (peek_input () == '=')
788 : {
789 : // modulo-assign
790 7 : skip_input ();
791 7 : current_column += 2;
792 7 : loc += 1;
793 :
794 7 : return Token::make (PERCENT_EQ, loc);
795 : }
796 : else
797 : {
798 : // modulo
799 36 : current_column++;
800 36 : return Token::make (PERCENT, loc);
801 : }
802 147 : case '^':
803 147 : if (peek_input () == '=')
804 : {
805 : // xor-assign?
806 84 : skip_input ();
807 84 : current_column += 2;
808 84 : loc += 1;
809 :
810 84 : return Token::make (CARET_EQ, loc);
811 : }
812 : else
813 : {
814 : // xor?
815 63 : current_column++;
816 63 : return Token::make (CARET, loc);
817 : }
818 8835 : case '<':
819 8835 : if (peek_input () == '<')
820 : {
821 66 : if (peek_input (1) == '=')
822 : {
823 : // left-shift assign
824 7 : skip_input (1);
825 7 : current_column += 3;
826 7 : loc += 2;
827 :
828 7 : return Token::make (LEFT_SHIFT_EQ, loc);
829 : }
830 : else
831 : {
832 : // left-shift
833 59 : skip_input ();
834 59 : current_column += 2;
835 59 : loc += 1;
836 :
837 59 : return Token::make (LEFT_SHIFT, loc);
838 : }
839 : }
840 8769 : else if (peek_input () == '=')
841 : {
842 : // smaller than or equal to
843 224 : skip_input ();
844 224 : current_column += 2;
845 224 : loc += 1;
846 :
847 224 : return Token::make (LESS_OR_EQUAL, loc);
848 : }
849 : else
850 : {
851 : // smaller than
852 8545 : current_column++;
853 8545 : return Token::make (LEFT_ANGLE, loc);
854 : }
855 8686 : break;
856 8686 : case '>':
857 8686 : if (peek_input () == '>')
858 : {
859 131 : if (peek_input (1) == '=')
860 : {
861 : // right-shift-assign
862 7 : skip_input (1);
863 7 : current_column += 3;
864 7 : loc += 2;
865 :
866 7 : return Token::make (RIGHT_SHIFT_EQ, loc);
867 : }
868 : else
869 : {
870 : // right-shift
871 124 : skip_input ();
872 124 : current_column += 2;
873 124 : loc += 1;
874 :
875 124 : return Token::make (RIGHT_SHIFT, loc);
876 : }
877 : }
878 8555 : else if (peek_input () == '=')
879 : {
880 : // larger than or equal to
881 209 : skip_input ();
882 209 : current_column += 2;
883 209 : loc += 1;
884 :
885 209 : return Token::make (GREATER_OR_EQUAL, loc);
886 : }
887 : else
888 : {
889 : // larger than
890 8346 : current_column++;
891 8346 : return Token::make (RIGHT_ANGLE, loc);
892 : }
893 29342 : case ':':
894 29342 : if (peek_input () == ':')
895 : {
896 : // scope resolution ::
897 9911 : skip_input ();
898 9911 : current_column += 2;
899 9911 : loc += 1;
900 :
901 9911 : return Token::make (SCOPE_RESOLUTION, loc);
902 : }
903 : else
904 : {
905 : // single colon :
906 19431 : current_column++;
907 19431 : return Token::make (COLON, loc);
908 : }
909 16525 : case '!':
910 : // no special handling for macros in lexer?
911 16525 : if (peek_input () == '=')
912 : {
913 : // not equal boolean operator
914 1006 : skip_input ();
915 1006 : current_column += 2;
916 1006 : loc += 1;
917 :
918 1006 : return Token::make (NOT_EQUAL, loc);
919 : }
920 : else
921 : {
922 : // not equal unary operator
923 15519 : current_column++;
924 :
925 15519 : return Token::make (EXCLAM, loc);
926 : }
927 377 : case '?':
928 377 : current_column++;
929 377 : return Token::make (QUESTION_MARK, loc);
930 21005 : case '#':
931 21005 : current_column++;
932 21005 : return Token::make (HASH, loc);
933 23125 : case '[':
934 23125 : current_column++;
935 23125 : return Token::make (LEFT_SQUARE, loc);
936 23118 : case ']':
937 23118 : current_column++;
938 23118 : return Token::make (RIGHT_SQUARE, loc);
939 36649 : case '{':
940 36649 : current_column++;
941 36649 : return Token::make (LEFT_CURLY, loc);
942 36598 : case '}':
943 36598 : current_column++;
944 36598 : return Token::make (RIGHT_CURLY, loc);
945 19 : case '@':
946 19 : current_column++;
947 19 : return Token::make (PATTERN_BIND, loc);
948 3622 : case '$':
949 3622 : current_column++;
950 3622 : return Token::make (DOLLAR_SIGN, loc);
951 0 : case '~':
952 0 : current_column++;
953 0 : return Token::make (TILDE, loc);
954 0 : case '\\':
955 0 : current_column++;
956 0 : return Token::make (BACKSLASH, loc);
957 0 : case '`':
958 0 : current_column++;
959 0 : return Token::make (BACKTICK, loc);
960 490 : case '|':
961 490 : if (peek_input () == '=')
962 : {
963 : // bitwise or-assign?
964 28 : skip_input ();
965 28 : current_column += 2;
966 28 : loc += 1;
967 :
968 28 : return Token::make (PIPE_EQ, loc);
969 : }
970 462 : else if (peek_input () == '|')
971 : {
972 : // logical or
973 84 : skip_input ();
974 84 : current_column += 2;
975 84 : loc += 1;
976 :
977 84 : return Token::make (OR, loc);
978 : }
979 : else
980 : {
981 : // bitwise or
982 378 : current_column++;
983 :
984 378 : return Token::make (PIPE, loc);
985 : }
986 10180 : case '&':
987 10180 : if (peek_input () == '=')
988 : {
989 : // bitwise and-assign?
990 21 : skip_input ();
991 21 : current_column += 2;
992 21 : loc += 1;
993 :
994 21 : return Token::make (AMP_EQ, loc);
995 : }
996 10159 : else if (peek_input () == '&')
997 : {
998 : // logical and
999 309 : skip_input ();
1000 309 : current_column += 2;
1001 309 : loc += 1;
1002 :
1003 309 : return Token::make (LOGICAL_AND, loc);
1004 : }
1005 : else
1006 : {
1007 : // bitwise and/reference
1008 9850 : current_column++;
1009 :
1010 9850 : return Token::make (AMP, loc);
1011 : }
1012 6883 : case '.':
1013 6883 : if (peek_input () == '.')
1014 : {
1015 1195 : if (peek_input (1) == '.')
1016 : {
1017 : // ellipsis
1018 860 : skip_input (1);
1019 860 : current_column += 3;
1020 860 : loc += 2;
1021 :
1022 860 : return Token::make (ELLIPSIS, loc);
1023 : }
1024 335 : else if (peek_input (1) == '=')
1025 : {
1026 : // ..=
1027 40 : skip_input (1);
1028 40 : current_column += 3;
1029 40 : loc += 2;
1030 :
1031 40 : return Token::make (DOT_DOT_EQ, loc);
1032 : }
1033 : else
1034 : {
1035 : // ..
1036 295 : skip_input ();
1037 295 : current_column += 2;
1038 295 : loc += 1;
1039 :
1040 295 : return Token::make (DOT_DOT, loc);
1041 : }
1042 : }
1043 : else /*if (!ISDIGIT (peek_input ()))*/
1044 : {
1045 : // single dot .
1046 : // Only if followed by a non-number - otherwise is float
1047 : // nope, float cannot start with '.'.
1048 5688 : current_column++;
1049 5688 : return Token::make (DOT, loc);
1050 : }
1051 1129496 : }
1052 : // TODO: special handling of _ in the lexer? instead of being identifier
1053 :
1054 : // byte character, byte string and raw byte string literals
1055 325972 : if (current_char == 'b')
1056 : {
1057 10777 : if (peek_input () == '\'')
1058 78 : return parse_byte_char (loc);
1059 10699 : else if (peek_input () == '"')
1060 64 : return parse_byte_string (loc);
1061 10635 : else if (peek_input () == 'r'
1062 10635 : && (peek_input (1) == '#' || peek_input (1) == '"'))
1063 32 : return parse_raw_byte_string (loc);
1064 : }
1065 :
1066 : // C-style strings
1067 315195 : else if (current_char == 'c' && peek_input () == '"')
1068 15 : return parse_c_string (loc);
1069 :
1070 : // raw identifiers and raw strings
1071 325783 : if (current_char == 'r')
1072 : {
1073 4067 : Codepoint peek = peek_input ();
1074 4067 : Codepoint peek1 = peek_input (1);
1075 :
1076 : // TODO (tamaron) parse Unicode ident
1077 4067 : if (peek == '#' && is_identifier_start (peek1.value))
1078 : {
1079 81 : TokenPtr raw_ident_ptr = parse_raw_identifier (loc);
1080 81 : if (raw_ident_ptr != nullptr)
1081 80 : return raw_ident_ptr;
1082 : else
1083 1 : continue; /* input got parsed, it just wasn't valid. An error
1084 : was produced. */
1085 81 : }
1086 : else
1087 : {
1088 3986 : TokenPtr maybe_raw_string_ptr = maybe_parse_raw_string (loc);
1089 3986 : if (maybe_raw_string_ptr != nullptr)
1090 25 : return maybe_raw_string_ptr;
1091 3986 : }
1092 : }
1093 :
1094 : // find identifiers and keywords.
1095 325677 : if (is_identifier_start (current_char.value))
1096 294181 : return parse_identifier_or_keyword (loc);
1097 :
1098 : // int and float literals
1099 31496 : if (ISDIGIT (current_char.value))
1100 : { // _ not allowed as first char
1101 17764 : if (current_char == '0'
1102 17764 : && is_non_decimal_int_literal_separator (peek_input ().value))
1103 : {
1104 : // handle binary, octal, hex literals
1105 337 : TokenPtr non_dec_int_lit_ptr
1106 337 : = parse_non_decimal_int_literals (loc);
1107 337 : if (non_dec_int_lit_ptr != nullptr)
1108 337 : return non_dec_int_lit_ptr;
1109 337 : }
1110 : else
1111 : {
1112 : // handle decimals (integer or float)
1113 17427 : TokenPtr decimal_or_float_ptr = parse_decimal_int_or_float (loc);
1114 17427 : if (decimal_or_float_ptr != nullptr)
1115 17427 : return decimal_or_float_ptr;
1116 17427 : }
1117 : }
1118 :
1119 : // string literals
1120 13732 : if (current_char == '"')
1121 12893 : return parse_string (loc);
1122 :
1123 : // char literals and lifetime names
1124 839 : if (current_char == '\'')
1125 : {
1126 839 : TokenPtr char_or_lifetime_ptr = parse_char_or_lifetime (loc);
1127 839 : if (char_or_lifetime_ptr != nullptr)
1128 839 : return char_or_lifetime_ptr;
1129 839 : }
1130 :
1131 : // DEBUG: check for specific character problems:
1132 0 : if (current_char == '0')
1133 0 : rust_debug ("'0' uncaught before unexpected character");
1134 0 : else if (current_char == ']')
1135 0 : rust_debug ("']' uncaught before unexpected character");
1136 : else if (current_char == 0x5d)
1137 : rust_debug ("whatever 0x5d is (not '0' or ']') uncaught before "
1138 : "unexpected character");
1139 :
1140 : // didn't match anything so error
1141 0 : rust_error_at (loc, "unexpected character %<%x%>", current_char.value);
1142 0 : current_column++;
1143 : }
1144 : }
1145 :
1146 : // Parses in a suffix
1147 : std::pair<std::string, int>
1148 17761 : Lexer::parse_in_suffix ()
1149 : {
1150 17761 : std::string suffix;
1151 :
1152 17761 : int additional_length_offset = 0;
1153 :
1154 : // get suffix
1155 39097 : while (ISALPHA (current_char.value) || ISDIGIT (current_char.value)
1156 41215 : || current_char == '_')
1157 : {
1158 5692 : additional_length_offset++;
1159 :
1160 5692 : suffix += current_char;
1161 5692 : skip_input ();
1162 5692 : current_char = peek_input ();
1163 : }
1164 :
1165 17761 : return std::make_pair (std::move (suffix), additional_length_offset);
1166 17761 : }
1167 :
1168 : // Parses in the exponent part (if any) of a float literal.
1169 : std::pair<std::string, int>
1170 365 : Lexer::parse_in_exponent_part ()
1171 : {
1172 365 : int additional_length_offset = 0;
1173 365 : std::string str;
1174 365 : if (current_char == 'E' || current_char == 'e')
1175 : {
1176 : // add exponent to string as strtod works with it
1177 8 : str += current_char;
1178 8 : skip_input ();
1179 8 : current_char = peek_input ();
1180 :
1181 8 : additional_length_offset++;
1182 :
1183 : // special - and + handling
1184 8 : if (current_char == '-' || current_char == '+')
1185 : {
1186 8 : str += current_char;
1187 :
1188 8 : skip_input ();
1189 8 : current_char = peek_input ();
1190 :
1191 8 : additional_length_offset++;
1192 : }
1193 :
1194 : // parse another decimal number for exponent
1195 8 : auto str_length = parse_in_decimal ();
1196 8 : str += std::get<0> (str_length);
1197 8 : additional_length_offset += std::get<1> (str_length);
1198 8 : }
1199 730 : return std::make_pair (str, additional_length_offset);
1200 365 : }
1201 :
1202 : // Parses a decimal integer.
1203 : std::tuple<std::string, int, bool>
1204 17800 : Lexer::parse_in_decimal ()
1205 : {
1206 : /* A pure decimal contains only digits. */
1207 17800 : bool pure_decimal = true;
1208 17800 : int additional_length_offset = 0;
1209 17800 : std::string str;
1210 25373 : while (ISDIGIT (current_char.value) || current_char.value == '_')
1211 : {
1212 7573 : if (current_char == '_')
1213 : {
1214 21 : pure_decimal = false;
1215 : }
1216 7573 : additional_length_offset++;
1217 :
1218 7573 : str += current_char;
1219 7573 : skip_input ();
1220 7573 : current_char = peek_input ();
1221 : }
1222 35600 : return std::make_tuple (str, additional_length_offset, pure_decimal);
1223 17800 : }
1224 :
1225 : /* Parses escapes (and string continues) in "byte" strings and characters. Does
1226 : * not support unicode. */
1227 : std::tuple<char, int, bool>
1228 62 : Lexer::parse_escape (char opening_char)
1229 : {
1230 62 : int additional_length_offset = 0;
1231 62 : char output_char = 0;
1232 :
1233 : // skip to actual letter
1234 62 : skip_input ();
1235 62 : current_char = peek_input ();
1236 62 : additional_length_offset++;
1237 :
1238 62 : switch (current_char.value)
1239 : {
1240 17 : case 'x':
1241 17 : {
1242 17 : auto hex_escape_pair = parse_partial_hex_escape ();
1243 17 : long hexLong = hex_escape_pair.first;
1244 17 : additional_length_offset += hex_escape_pair.second;
1245 :
1246 17 : if (hexLong > 255 || hexLong < 0)
1247 0 : rust_error_at (
1248 : get_current_location (),
1249 : "byte \\x escape %<\\x%x%> out of range - allows up to %<\\xFF%>",
1250 : static_cast<unsigned int> (hexLong));
1251 : /* TODO: restore capital for escape output - gcc pretty-printer doesn't
1252 : * support %X directly */
1253 17 : char hexChar = static_cast<char> (hexLong);
1254 :
1255 17 : output_char = hexChar;
1256 : }
1257 17 : break;
1258 : case 'n':
1259 : output_char = '\n';
1260 : break;
1261 0 : case 'r':
1262 0 : output_char = '\r';
1263 0 : break;
1264 1 : case 't':
1265 1 : output_char = '\t';
1266 1 : break;
1267 8 : case '\\':
1268 8 : output_char = '\\';
1269 8 : break;
1270 10 : case '0':
1271 10 : output_char = '\0';
1272 10 : break;
1273 15 : case '\'':
1274 15 : output_char = '\'';
1275 15 : break;
1276 1 : case '"':
1277 1 : output_char = '"';
1278 1 : break;
1279 2 : case 'u':
1280 3 : rust_error_at (get_current_location (),
1281 : "cannot have a unicode escape \\u in a byte %s",
1282 : opening_char == '\'' ? "character" : "string");
1283 : // Try to parse it anyway, just to skip it
1284 2 : parse_partial_unicode_escape ();
1285 2 : return std::make_tuple (output_char, additional_length_offset, false);
1286 0 : case '\r':
1287 0 : case '\n':
1288 : // string continue
1289 0 : return std::make_tuple (0, parse_partial_string_continue (), true);
1290 1 : default:
1291 1 : rust_error_at (get_current_location (),
1292 : "unknown escape sequence %<\\%s%>",
1293 1 : current_char.as_string ().c_str ());
1294 : // returns false if no parsing could be done
1295 : // return false;
1296 1 : return std::make_tuple (output_char, additional_length_offset, false);
1297 59 : break;
1298 : }
1299 : // all non-special cases (string continue) should skip their used char
1300 59 : skip_input ();
1301 59 : current_char = peek_input ();
1302 59 : additional_length_offset++;
1303 :
1304 : // returns true if parsing was successful
1305 : // return true;
1306 59 : return std::make_tuple (output_char, additional_length_offset, false);
1307 : }
1308 :
1309 : /* Parses an escape (or string continue) in a string or character. Supports
1310 : * unicode escapes. */
1311 : std::tuple<Codepoint, int, bool>
1312 2830 : Lexer::parse_utf8_escape ()
1313 : {
1314 2830 : Codepoint output_char;
1315 2830 : int additional_length_offset = 0;
1316 :
1317 : // skip to actual letter
1318 2830 : skip_input ();
1319 2830 : current_char = peek_input ();
1320 2830 : additional_length_offset++;
1321 :
1322 2830 : switch (current_char.value)
1323 : {
1324 17 : case 'x':
1325 17 : {
1326 17 : auto hex_escape_pair = parse_partial_hex_escape ();
1327 17 : long hexLong = hex_escape_pair.first;
1328 17 : additional_length_offset += hex_escape_pair.second;
1329 :
1330 17 : if (hexLong > 127 || hexLong < 0)
1331 4 : rust_error_at (
1332 : get_current_location (),
1333 : "ascii \\x escape %<\\x%x%> out of range - allows up to %<\\x7F%>",
1334 : static_cast<unsigned int> (hexLong));
1335 : /* TODO: restore capital for escape output - gcc pretty-printer doesn't
1336 : * support %X directly */
1337 17 : char hexChar = static_cast<char> (hexLong);
1338 :
1339 17 : output_char = hexChar;
1340 : }
1341 17 : break;
1342 : case 'n':
1343 : output_char = '\n';
1344 : break;
1345 0 : case 'r':
1346 0 : output_char = '\r';
1347 0 : break;
1348 2 : case 't':
1349 2 : output_char = '\t';
1350 2 : break;
1351 1 : case '\\':
1352 1 : output_char = '\\';
1353 1 : break;
1354 1418 : case '0':
1355 1418 : output_char = '\0';
1356 1418 : break;
1357 1 : case '\'':
1358 1 : output_char = '\'';
1359 1 : break;
1360 1 : case '"':
1361 1 : output_char = '"';
1362 1 : break;
1363 46 : case 'u':
1364 46 : {
1365 46 : auto unicode_escape_pair = parse_partial_unicode_escape ();
1366 46 : output_char = unicode_escape_pair.first;
1367 46 : additional_length_offset += unicode_escape_pair.second;
1368 :
1369 46 : return std::make_tuple (output_char, additional_length_offset, false);
1370 : }
1371 28 : break;
1372 28 : case '\r':
1373 28 : case '\n':
1374 : // string continue
1375 28 : return std::make_tuple (0, parse_partial_string_continue (), true);
1376 1 : default:
1377 1 : rust_error_at (get_current_location (),
1378 : "unknown escape sequence %<\\%s%>",
1379 1 : current_char.as_string ().c_str ());
1380 : // returns false if no parsing could be done
1381 : // return false;
1382 1 : return std::make_tuple (output_char, additional_length_offset, false);
1383 2755 : break;
1384 : }
1385 : /* all non-special cases (unicode, string continue) should skip their used
1386 : * char */
1387 2755 : skip_input ();
1388 2755 : current_char = peek_input ();
1389 2755 : additional_length_offset++;
1390 :
1391 : // returns true if parsing was successful
1392 : // return true;
1393 2755 : return std::make_tuple (output_char, additional_length_offset, false);
1394 : }
1395 :
1396 : // Parses the body of a string continue that has been found in an escape.
1397 : int
1398 28 : Lexer::parse_partial_string_continue ()
1399 : {
1400 28 : int additional_length_offset = 1;
1401 :
1402 : // string continue
1403 : // TODO use utf-8 codepoint to skip whitespaces
1404 364 : while (is_whitespace (current_char.value))
1405 : {
1406 336 : if (current_char == '\n')
1407 : {
1408 28 : current_line++;
1409 28 : current_column = 1;
1410 : // tell line_table that new line starts
1411 28 : start_line (current_line, max_column_hint);
1412 :
1413 : // reset "length"
1414 28 : additional_length_offset = 1;
1415 :
1416 : // get next char
1417 28 : skip_input ();
1418 28 : current_char = peek_input ();
1419 :
1420 28 : continue;
1421 : }
1422 :
1423 308 : skip_input ();
1424 308 : current_char = peek_input ();
1425 308 : additional_length_offset++;
1426 : }
1427 :
1428 28 : return additional_length_offset;
1429 : }
1430 :
1431 : /* Parses the body of a '\x' escape. Note that it does not check that the number
1432 : * is valid and smaller than 255. */
1433 : std::pair<long, int>
1434 34 : Lexer::parse_partial_hex_escape ()
1435 : {
1436 : // hex char string (null-terminated)
1437 34 : char hexNum[3] = {0, 0, 0};
1438 :
1439 : // first hex char
1440 34 : current_char = peek_input (1);
1441 34 : int additional_length_offset = 1;
1442 :
1443 34 : if (!is_x_digit (current_char.value))
1444 : {
1445 4 : rust_error_at (get_current_location (),
1446 : "invalid character %<\\x%s%> in \\x sequence",
1447 4 : current_char.as_string ().c_str ());
1448 4 : return std::make_pair (0, 0);
1449 : }
1450 30 : hexNum[0] = current_char.value;
1451 :
1452 : // second hex char
1453 30 : skip_input ();
1454 30 : current_char = peek_input (1);
1455 30 : additional_length_offset++;
1456 :
1457 30 : if (!is_x_digit (current_char.value))
1458 : {
1459 2 : rust_error_at (get_current_location (),
1460 2 : "invalid character %<\\x%c%s%> in \\x sequence", hexNum[0],
1461 2 : current_char.as_string ().c_str ());
1462 2 : return std::make_pair (0, 1);
1463 : }
1464 28 : skip_input ();
1465 28 : hexNum[1] = current_char.value;
1466 :
1467 28 : long hexLong = std::strtol (hexNum, nullptr, 16);
1468 :
1469 28 : return std::make_pair (hexLong, additional_length_offset);
1470 : }
1471 :
1472 : // Parses the body of a unicode escape.
1473 : std::pair<Codepoint, int>
1474 48 : Lexer::parse_partial_unicode_escape ()
1475 : {
1476 48 : skip_input ();
1477 48 : current_char = peek_input ();
1478 48 : int additional_length_offset = 0;
1479 :
1480 48 : if (current_char != '{')
1481 : {
1482 2 : rust_error_at (get_current_location (),
1483 : "unicode escape should start with %<{%>");
1484 : /* Skip what should probaby have been between brackets. */
1485 10 : while (is_x_digit (current_char.value) || current_char == '_')
1486 : {
1487 6 : skip_input ();
1488 6 : current_char = peek_input ();
1489 6 : additional_length_offset++;
1490 : }
1491 2 : return std::make_pair (Codepoint (0), additional_length_offset);
1492 : }
1493 :
1494 46 : skip_input ();
1495 46 : current_char = peek_input ();
1496 46 : additional_length_offset++;
1497 :
1498 46 : if (current_char == '_')
1499 : {
1500 2 : rust_error_at (get_current_location (),
1501 : "unicode escape cannot start with %<_%>");
1502 2 : skip_input ();
1503 2 : current_char = peek_input ();
1504 2 : additional_length_offset++;
1505 : // fallthrough and try to parse the rest anyway
1506 : }
1507 :
1508 : // parse unicode escape - 1-6 hex digits
1509 46 : std::string num_str;
1510 46 : num_str.reserve (6);
1511 :
1512 : // loop through to add entire hex number to string
1513 304 : while (is_x_digit (current_char.value) || current_char.value == '_')
1514 : {
1515 212 : if (current_char == '_')
1516 : {
1517 : // don't add _ to number
1518 24 : skip_input ();
1519 24 : current_char = peek_input ();
1520 :
1521 24 : additional_length_offset++;
1522 :
1523 24 : continue;
1524 : }
1525 :
1526 188 : additional_length_offset++;
1527 :
1528 : // add raw hex numbers
1529 188 : num_str += current_char;
1530 :
1531 188 : skip_input ();
1532 188 : current_char = peek_input ();
1533 : }
1534 :
1535 46 : if (current_char == '}')
1536 : {
1537 44 : skip_input ();
1538 44 : current_char = peek_input ();
1539 44 : additional_length_offset++;
1540 : }
1541 : else
1542 : {
1543 : // actually an error, but allow propagation anyway Assume that
1544 : // wrong bracketm whitespace or single/double quotes are wrong
1545 : // termination, otherwise it is a wrong character, then skip to the actual
1546 : // terminator.
1547 : // TODO use utf-8 codepoint to skip whitespaces
1548 2 : if (current_char == '{' || is_whitespace (current_char.value)
1549 4 : || current_char == '\'' || current_char == '"')
1550 : {
1551 0 : rust_error_at (get_current_location (),
1552 : "expected terminating %<}%> in unicode escape");
1553 0 : return std::make_pair (Codepoint (0), additional_length_offset);
1554 : }
1555 : else
1556 : {
1557 2 : rust_error_at (get_current_location (),
1558 : "invalid character %qs in unicode escape",
1559 2 : current_char.as_string ().c_str ());
1560 : // TODO use utf-8 codepoint to skip whitespaces
1561 8 : while (current_char != '}' && current_char != '{'
1562 6 : && !is_whitespace (current_char.value) && current_char != '\''
1563 14 : && current_char != '"')
1564 : {
1565 6 : skip_input ();
1566 6 : current_char = peek_input ();
1567 6 : additional_length_offset++;
1568 : }
1569 : // Consume the actual closing bracket if found
1570 2 : if (current_char == '}')
1571 : {
1572 2 : skip_input ();
1573 2 : current_char = peek_input ();
1574 2 : additional_length_offset++;
1575 : }
1576 2 : return std::make_pair (Codepoint (0), additional_length_offset);
1577 : }
1578 : }
1579 :
1580 : // ensure 1-6 hex characters
1581 44 : if (num_str.length () > 6 || num_str.length () < 1)
1582 : {
1583 4 : rust_error_at (get_current_location (),
1584 : "unicode escape should be between 1 and 6 hex "
1585 : "characters; it is %lu",
1586 4 : (unsigned long) num_str.length ());
1587 : // return false;
1588 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1589 : }
1590 :
1591 40 : unsigned long hex_num = std::strtoul (num_str.c_str (), nullptr, 16);
1592 :
1593 40 : if (hex_num > 0xd7ff && hex_num < 0xe000)
1594 : {
1595 4 : rust_error_at (
1596 : get_current_location (),
1597 : "unicode escape cannot be a surrogate value (D800 to DFFF)");
1598 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1599 : }
1600 :
1601 36 : if (hex_num > 0x10ffff)
1602 : {
1603 4 : rust_error_at (get_current_location (),
1604 : "unicode escape cannot be larger than 10FFFF");
1605 4 : return std::make_pair (Codepoint (0), additional_length_offset);
1606 : }
1607 :
1608 : // return true;
1609 32 : return std::make_pair (Codepoint (static_cast<uint32_t> (hex_num)),
1610 : additional_length_offset);
1611 46 : }
1612 :
1613 : // Parses a byte character.
1614 : TokenPtr
1615 78 : Lexer::parse_byte_char (location_t loc)
1616 : {
1617 78 : skip_input ();
1618 78 : current_column++;
1619 : // make current char the next character
1620 78 : current_char = peek_input ();
1621 :
1622 78 : int length = 1;
1623 :
1624 : // char to save
1625 78 : Codepoint byte_char = 0;
1626 :
1627 : // detect escapes
1628 78 : if (current_char == '\\')
1629 : {
1630 30 : auto escape_length_pair = parse_escape ('\'');
1631 30 : byte_char = std::get<0> (escape_length_pair);
1632 30 : length += std::get<1> (escape_length_pair);
1633 :
1634 30 : current_char = peek_input ();
1635 :
1636 30 : if (current_char != '\'')
1637 : {
1638 0 : rust_error_at (get_current_location (), "unclosed %<byte char%>");
1639 : }
1640 :
1641 30 : skip_input ();
1642 30 : current_char = peek_input ();
1643 30 : length++; // go to next char
1644 : }
1645 48 : else if (current_char != '\'')
1646 : {
1647 : // otherwise, get character from direct input character
1648 48 : byte_char = current_char;
1649 :
1650 48 : if (!byte_char.is_ascii ())
1651 : {
1652 2 : rust_error_at (get_current_location (),
1653 : "non-ASCII character in %<byte char%>");
1654 : }
1655 :
1656 48 : skip_input ();
1657 48 : current_char = peek_input ();
1658 48 : length++;
1659 :
1660 48 : if (current_char != '\'')
1661 : {
1662 0 : rust_error_at (get_current_location (), "unclosed %<byte char%>");
1663 : }
1664 :
1665 48 : skip_input ();
1666 48 : current_char = peek_input ();
1667 48 : length++; // go to next char
1668 : }
1669 : else
1670 : {
1671 0 : rust_error_at (get_current_location (),
1672 : "no character inside %<%> for %<byte char%>");
1673 : }
1674 :
1675 78 : current_column += length;
1676 :
1677 78 : loc += length - 1;
1678 78 : return Token::make_byte_char (loc, byte_char.value);
1679 : }
1680 :
1681 : // Parses a byte string.
1682 : TokenPtr
1683 64 : Lexer::parse_byte_string (location_t loc)
1684 : {
1685 : // byte string
1686 :
1687 : // skip quote character
1688 64 : skip_input ();
1689 64 : current_column++;
1690 :
1691 64 : std::string str;
1692 64 : str.reserve (16); // some sensible default
1693 :
1694 64 : current_char = peek_input ();
1695 :
1696 64 : const location_t string_begin_locus = get_current_location ();
1697 :
1698 438 : while (current_char != '"' && !current_char.is_eof ())
1699 : {
1700 310 : if (current_char == '\\')
1701 : {
1702 31 : int length = 1;
1703 31 : auto escape_length_pair = parse_escape ('"');
1704 31 : char output_char = std::get<0> (escape_length_pair);
1705 :
1706 31 : if (output_char == 0 && std::get<2> (escape_length_pair))
1707 0 : length = std::get<1> (escape_length_pair) - 1;
1708 : else
1709 31 : length += std::get<1> (escape_length_pair);
1710 :
1711 31 : if (output_char != 0 || !std::get<2> (escape_length_pair))
1712 31 : str += output_char;
1713 :
1714 31 : current_column += length;
1715 :
1716 31 : continue;
1717 31 : }
1718 :
1719 279 : current_column++;
1720 279 : if (current_char.value == '\n')
1721 : {
1722 23 : current_line++;
1723 23 : current_column = 1;
1724 : // tell line_table that new line starts
1725 23 : start_line (current_line, max_column_hint);
1726 : }
1727 :
1728 279 : str += current_char;
1729 279 : skip_input ();
1730 279 : current_char = peek_input ();
1731 : }
1732 :
1733 64 : if (current_char == '"')
1734 : {
1735 57 : current_column++;
1736 :
1737 57 : skip_input ();
1738 57 : current_char = peek_input ();
1739 : }
1740 7 : else if (current_char.is_eof ())
1741 : {
1742 7 : rust_error_at (string_begin_locus, "unended byte string literal");
1743 7 : return Token::make (END_OF_FILE, get_current_location ());
1744 : }
1745 : else
1746 : {
1747 : rust_unreachable ();
1748 : }
1749 :
1750 57 : str.shrink_to_fit ();
1751 57 : loc += str.size () - 1;
1752 :
1753 57 : return Token::make_byte_string (loc, std::move (str));
1754 64 : }
1755 :
1756 : // Parses a C-style string.
1757 : TokenPtr
1758 15 : Lexer::parse_c_string (location_t loc)
1759 : {
1760 15 : skip_input ();
1761 15 : current_column++;
1762 :
1763 : // Mostly same code copied from parse_string...
1764 :
1765 15 : std::string str;
1766 15 : str.reserve (16); // some sensible default
1767 :
1768 15 : current_char = peek_input ();
1769 :
1770 15 : const location_t string_begin_locus = get_current_location ();
1771 :
1772 106 : while (current_char.value != '"' && !current_char.is_eof ())
1773 : {
1774 76 : if (current_char.value == '\\')
1775 : {
1776 1 : int length = 1;
1777 :
1778 1 : auto escape_pair = parse_escape ('"');
1779 1 : current_char = std::get<0> (escape_pair);
1780 :
1781 1 : if (current_char == Codepoint (0) && std::get<2> (escape_pair))
1782 0 : length = std::get<1> (escape_pair) - 1;
1783 : else
1784 1 : length += std::get<1> (escape_pair);
1785 :
1786 1 : if (current_char != Codepoint (0) || !std::get<2> (escape_pair))
1787 2 : str += current_char.as_string ();
1788 :
1789 1 : current_column += length;
1790 :
1791 : // FIXME: parse_escape does not update current_char correctly.
1792 1 : current_char = peek_input ();
1793 1 : continue;
1794 1 : }
1795 :
1796 75 : current_column++;
1797 75 : if (current_char.value == '\n')
1798 : {
1799 0 : current_line++;
1800 0 : current_column = 1;
1801 : // tell line_table that new line starts
1802 0 : start_line (current_line, max_column_hint);
1803 : }
1804 :
1805 75 : str += current_char;
1806 75 : skip_input ();
1807 75 : current_char = peek_input ();
1808 : }
1809 :
1810 15 : if (current_char.value == '"')
1811 : {
1812 15 : current_column++;
1813 :
1814 15 : skip_input ();
1815 15 : current_char = peek_input ();
1816 : }
1817 0 : else if (current_char.is_eof ())
1818 : {
1819 0 : rust_error_at (string_begin_locus, "unended C string literal");
1820 0 : return Token::make (END_OF_FILE, get_current_location ());
1821 : }
1822 : else
1823 : {
1824 : rust_unreachable ();
1825 : }
1826 :
1827 15 : str.shrink_to_fit ();
1828 :
1829 15 : return Token::make_c_string (loc, std::move (str));
1830 15 : }
1831 :
1832 : // Parses a raw byte string.
1833 : TokenPtr
1834 32 : Lexer::parse_raw_byte_string (location_t loc)
1835 : {
1836 : // raw byte string literals
1837 32 : std::string str;
1838 32 : str.reserve (16); // some sensible default
1839 :
1840 32 : int length = 1;
1841 32 : int hash_count = 0;
1842 :
1843 32 : const location_t string_begin_locus = get_current_location ();
1844 :
1845 : // get hash count at beginnning
1846 32 : skip_input ();
1847 32 : current_char = peek_input ();
1848 32 : length++;
1849 32 : current_column++;
1850 54 : while (current_char == '#')
1851 : {
1852 22 : hash_count++;
1853 22 : length++;
1854 22 : current_column++;
1855 :
1856 22 : skip_input ();
1857 22 : current_char = peek_input ();
1858 : }
1859 :
1860 32 : if (current_char != '"')
1861 : {
1862 0 : rust_error_at (get_current_location (),
1863 : "raw byte string has no opening %<\"%>");
1864 : }
1865 :
1866 32 : skip_input ();
1867 32 : current_char = peek_input ();
1868 32 : length++;
1869 32 : current_column++;
1870 :
1871 330 : while (true)
1872 : {
1873 181 : if (current_char == '"')
1874 : {
1875 51 : bool enough_hashes = true;
1876 :
1877 51 : for (int i = 0; i < hash_count; i++)
1878 : {
1879 26 : if (peek_input (i + 1) != '#')
1880 : {
1881 : enough_hashes = false;
1882 : break;
1883 : }
1884 : }
1885 :
1886 35 : if (enough_hashes)
1887 : {
1888 : // skip enough input and peek enough input
1889 25 : skip_input (hash_count);
1890 25 : current_char = peek_input ();
1891 25 : length += hash_count + 1;
1892 25 : current_column += hash_count + 1;
1893 25 : break;
1894 : }
1895 : }
1896 146 : else if (current_char.is_eof ())
1897 : {
1898 7 : rust_error_at (string_begin_locus, "unended raw byte string literal");
1899 7 : return Token::make (END_OF_FILE, get_current_location ());
1900 : }
1901 139 : else if (current_char.value > 127)
1902 : {
1903 1 : rust_error_at (get_current_location (),
1904 : "character %qs in raw byte string out of range",
1905 1 : current_char.as_string ().c_str ());
1906 1 : current_char = 0;
1907 : }
1908 :
1909 149 : length++;
1910 149 : current_column++;
1911 149 : if (current_char == '\n')
1912 : {
1913 22 : current_line++;
1914 22 : current_column = 1;
1915 22 : start_line (current_line, max_column_hint);
1916 : }
1917 :
1918 149 : str += current_char;
1919 149 : skip_input ();
1920 149 : current_char = peek_input ();
1921 149 : }
1922 :
1923 25 : loc += length - 1;
1924 :
1925 25 : str.shrink_to_fit ();
1926 :
1927 25 : return Token::make_byte_string (loc, std::move (str));
1928 32 : }
1929 :
1930 : // Parses a raw identifier.
1931 : TokenPtr
1932 81 : Lexer::parse_raw_identifier (location_t loc)
1933 : {
1934 : // raw identifier
1935 81 : std::string str;
1936 81 : str.reserve (16); // default
1937 :
1938 81 : skip_input ();
1939 81 : current_char = peek_input ();
1940 :
1941 81 : current_column += 2;
1942 :
1943 81 : bool first_is_underscore = current_char == '_';
1944 :
1945 81 : int length = 0;
1946 81 : current_char = peek_input ();
1947 : // loop through entire name
1948 475 : while (is_identifier_continue (current_char.value))
1949 : {
1950 313 : length++;
1951 :
1952 313 : str += current_char;
1953 313 : skip_input ();
1954 313 : current_char = peek_input ();
1955 : }
1956 :
1957 81 : current_column += length;
1958 :
1959 81 : rust_debug ("raw ident: %s", str.c_str ());
1960 :
1961 : // if just a single underscore, not an identifier
1962 81 : if (first_is_underscore && length == 1)
1963 1 : rust_error_at (get_current_location (),
1964 : "%<_%> is not a valid raw identifier");
1965 :
1966 81 : using namespace Rust::Values;
1967 81 : std::set<std::string> invalid{
1968 81 : Keywords::CRATE, Keywords::EXTERN_KW, Keywords::SELF,
1969 81 : Keywords::SUPER, Keywords::SELF_ALIAS,
1970 486 : };
1971 :
1972 81 : if (invalid.find (str) != invalid.end ())
1973 : {
1974 1 : rust_error_at (get_current_location (),
1975 : "%qs is a forbidden raw identifier", str.c_str ());
1976 :
1977 1 : return nullptr;
1978 : }
1979 : else
1980 : {
1981 80 : str.shrink_to_fit ();
1982 80 : loc += length - 1;
1983 :
1984 80 : return Token::make_identifier (loc, std::move (str));
1985 : }
1986 81 : }
1987 :
1988 : // skip broken string input (unterminated strings)
1989 : void
1990 0 : Lexer::skip_broken_string_input (Codepoint current_char)
1991 : {
1992 0 : while (current_char != '"' && !current_char.is_eof ())
1993 : {
1994 0 : if (current_char == '\n')
1995 : {
1996 0 : current_line++;
1997 0 : current_column = 1;
1998 : }
1999 : else
2000 : {
2001 0 : current_column++;
2002 : }
2003 0 : skip_input ();
2004 0 : current_char = peek_input ();
2005 : }
2006 0 : if (current_char == '"')
2007 : {
2008 0 : current_column++;
2009 :
2010 0 : skip_input ();
2011 0 : current_char = peek_input ();
2012 : }
2013 0 : rust_debug ("skipped to %d:%d due to bad quotes", current_line,
2014 : current_column);
2015 0 : }
2016 :
2017 : // Parses a string.
2018 : TokenPtr
2019 12893 : Lexer::parse_string (location_t loc)
2020 : {
2021 12893 : std::string str;
2022 12893 : str.reserve (16); // some sensible default
2023 :
2024 12893 : current_char = peek_input ();
2025 :
2026 12893 : const location_t string_begin_locus = get_current_location ();
2027 :
2028 : // FIXME: This fails if the input ends. How do we check for EOF?
2029 105608 : while (current_char.value != '"' && !current_char.is_eof ())
2030 : {
2031 79822 : if (current_char.value == '\\')
2032 : {
2033 2807 : int length = 1;
2034 :
2035 : // parse escape
2036 2807 : auto utf8_escape_pair = parse_utf8_escape ();
2037 2807 : current_char = std::get<0> (utf8_escape_pair);
2038 :
2039 2807 : if (current_char == Codepoint (0) && std::get<2> (utf8_escape_pair))
2040 28 : length = std::get<1> (utf8_escape_pair) - 1;
2041 : else
2042 2779 : length += std::get<1> (utf8_escape_pair);
2043 :
2044 2807 : if (current_char != Codepoint (0) || !std::get<2> (utf8_escape_pair))
2045 5558 : str += current_char.as_string ();
2046 :
2047 2807 : current_column += length;
2048 :
2049 : // FIXME: should remove this but can't.
2050 : // `parse_utf8_escape` does not update `current_char` correctly.
2051 2807 : current_char = peek_input ();
2052 2807 : continue;
2053 2807 : }
2054 :
2055 77015 : current_column++;
2056 77015 : if (current_char.value == '\n')
2057 : {
2058 67 : current_line++;
2059 67 : current_column = 1;
2060 : // tell line_table that new line starts
2061 67 : start_line (current_line, max_column_hint);
2062 : }
2063 :
2064 77015 : str += current_char;
2065 77015 : skip_input ();
2066 77015 : current_char = peek_input ();
2067 : }
2068 :
2069 12893 : if (current_char.value == '"')
2070 : {
2071 12879 : current_column++;
2072 :
2073 12879 : skip_input ();
2074 12879 : current_char = peek_input ();
2075 : }
2076 14 : else if (current_char.is_eof ())
2077 : {
2078 14 : rust_error_at (string_begin_locus, "unended string literal");
2079 14 : return Token::make (END_OF_FILE, get_current_location ());
2080 : }
2081 : else
2082 : {
2083 : rust_unreachable ();
2084 : }
2085 :
2086 12879 : str.shrink_to_fit ();
2087 :
2088 12879 : return Token::make_string (loc, std::move (str));
2089 12893 : }
2090 :
2091 : // Parses an identifier or keyword.
2092 : TokenPtr
2093 294181 : Lexer::parse_identifier_or_keyword (location_t loc)
2094 : {
2095 294181 : std::string str;
2096 294181 : str.reserve (16); // default
2097 588362 : str += current_char.as_string ();
2098 :
2099 294181 : bool first_is_underscore = current_char == '_';
2100 :
2101 294181 : int length = 1;
2102 294181 : current_char = peek_input ();
2103 :
2104 : // loop through entire name
2105 1493942 : while (is_identifier_continue (current_char.value))
2106 : {
2107 905580 : auto s = current_char.as_string ();
2108 905580 : length++;
2109 :
2110 1811160 : str += current_char.as_string ();
2111 905580 : skip_input ();
2112 905580 : current_char = peek_input ();
2113 905580 : }
2114 :
2115 294181 : current_column += length;
2116 :
2117 : // if just a single underscore, not an identifier
2118 294181 : if (first_is_underscore && length == 1)
2119 1346 : return Token::make (UNDERSCORE, loc);
2120 :
2121 292835 : str.shrink_to_fit ();
2122 :
2123 292835 : loc += length - 1;
2124 :
2125 292835 : TokenId keyword = classify_keyword (str);
2126 292835 : if (keyword == IDENTIFIER)
2127 194585 : return Token::make_identifier (loc, std::move (str));
2128 : else
2129 98250 : return Token::make (keyword, loc);
2130 294181 : }
2131 :
2132 : // Possibly returns a raw string token if it exists - otherwise returns null.
2133 : TokenPtr
2134 3986 : Lexer::maybe_parse_raw_string (location_t loc)
2135 : {
2136 3986 : int peek_index = 0;
2137 3995 : while (peek_input (peek_index) == '#')
2138 9 : peek_index++;
2139 :
2140 3986 : if (peek_input (peek_index) == '"')
2141 25 : return parse_raw_string (loc, peek_index);
2142 : else
2143 3961 : return nullptr;
2144 : }
2145 :
2146 : // Returns a raw string token.
2147 : TokenPtr
2148 25 : Lexer::parse_raw_string (location_t loc, int initial_hash_count)
2149 : {
2150 : // raw string literals
2151 25 : std::string str;
2152 25 : str.reserve (16); // some sensible default
2153 :
2154 25 : int length = 1 + initial_hash_count;
2155 25 : current_column += length;
2156 :
2157 25 : const location_t string_begin_locus = get_current_location ();
2158 :
2159 25 : if (initial_hash_count > 0)
2160 7 : skip_input (initial_hash_count - 1);
2161 :
2162 25 : current_char = peek_input ();
2163 :
2164 25 : if (current_char != '"')
2165 0 : rust_error_at (get_current_location (), "raw string has no opening %<\"%>");
2166 :
2167 25 : length++;
2168 25 : current_column++;
2169 25 : skip_input ();
2170 25 : current_char = peek_input ();
2171 :
2172 181 : while (true)
2173 : {
2174 103 : if (current_char.value == '"')
2175 : {
2176 38 : bool enough_hashes = true;
2177 :
2178 38 : for (int i = 0; i < initial_hash_count; i++)
2179 : {
2180 13 : if (peek_input (i + 1) != '#')
2181 : {
2182 : enough_hashes = false;
2183 : break;
2184 : }
2185 : }
2186 :
2187 28 : if (enough_hashes)
2188 : {
2189 : // skip enough input and peek enough input
2190 25 : skip_input (initial_hash_count);
2191 25 : current_char = peek_input ();
2192 25 : length += initial_hash_count + 1;
2193 25 : current_column += initial_hash_count + 1;
2194 25 : break;
2195 : }
2196 : }
2197 75 : else if (current_char.is_eof ())
2198 : {
2199 0 : rust_error_at (string_begin_locus, "unended raw string literal");
2200 0 : return Token::make (END_OF_FILE, get_current_location ());
2201 : }
2202 :
2203 78 : length++;
2204 78 : current_column++;
2205 78 : if (current_char == '\n')
2206 : {
2207 1 : current_line++;
2208 1 : current_column = 1;
2209 1 : start_line (current_line, max_column_hint);
2210 : }
2211 :
2212 156 : str += current_char.as_string ();
2213 78 : skip_input ();
2214 78 : current_char = peek_input ();
2215 78 : }
2216 :
2217 25 : loc += length - 1;
2218 :
2219 25 : str.shrink_to_fit ();
2220 :
2221 25 : return Token::make_raw_string (loc, std::move (str));
2222 25 : }
2223 :
2224 : template <typename IsDigitFunc>
2225 : TokenPtr
2226 337 : Lexer::parse_non_decimal_int_literal (location_t loc, IsDigitFunc is_digit_func,
2227 : IntegerLiteralBase base)
2228 : {
2229 337 : std::string raw_str = "0";
2230 337 : raw_str += current_char; // x, o, b
2231 337 : skip_input ();
2232 :
2233 337 : int length = 2;
2234 337 : bool has_valid_digit = false;
2235 :
2236 337 : current_char = peek_input ();
2237 :
2238 : // loop through to add entire number to string
2239 2391 : while (true)
2240 : {
2241 2728 : if (is_digit_func (current_char.value))
2242 : {
2243 : has_valid_digit = true;
2244 : }
2245 393 : else if (current_char != '_')
2246 : {
2247 : break;
2248 : }
2249 2391 : length++;
2250 :
2251 2391 : raw_str += current_char;
2252 2391 : skip_input ();
2253 2391 : current_char = peek_input ();
2254 : }
2255 :
2256 337 : int suffix_start = raw_str.length ();
2257 :
2258 : // parse in suffix if it exists
2259 337 : auto suffix_pair = parse_in_suffix ();
2260 337 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2261 337 : raw_str += suffix_pair.first;
2262 337 : length += suffix_pair.second;
2263 :
2264 337 : current_column += length;
2265 :
2266 337 : if (!has_valid_digit)
2267 : {
2268 5 : rust_error_at (loc, ErrorCode::E0768, "no valid digits found for number");
2269 : }
2270 :
2271 337 : loc += length - 1;
2272 :
2273 337 : return Token::make_int (loc, std::move (raw_str), suffix_start, base,
2274 337 : type_hint);
2275 337 : }
2276 :
2277 : // Parses a hex, binary or octal int literal.
2278 : TokenPtr
2279 337 : Lexer::parse_non_decimal_int_literals (location_t loc)
2280 : {
2281 337 : current_char = peek_input ();
2282 :
2283 337 : if (current_char == 'x')
2284 : {
2285 : // hex (integer only)
2286 298 : return parse_non_decimal_int_literal (loc, is_x_digit,
2287 298 : IntegerLiteralBase::Hex);
2288 : }
2289 39 : else if (current_char == 'o')
2290 : {
2291 : // octal (integer only)
2292 19 : return parse_non_decimal_int_literal (loc, is_octal_digit,
2293 19 : IntegerLiteralBase::Octal);
2294 : }
2295 20 : else if (current_char == 'b')
2296 : {
2297 : // binary (integer only)
2298 20 : return parse_non_decimal_int_literal (loc, is_bin_digit,
2299 20 : IntegerLiteralBase::Binary);
2300 : }
2301 : else
2302 : {
2303 0 : return nullptr;
2304 : }
2305 : }
2306 :
2307 : // Parses a decimal-based int literal or float literal.
2308 : TokenPtr
2309 17427 : Lexer::parse_decimal_int_or_float (location_t loc)
2310 : {
2311 17427 : std::string str;
2312 17427 : str.reserve (16); // some sensible default
2313 17427 : str += current_char;
2314 :
2315 17427 : int length = 1;
2316 17427 : bool first_zero = current_char == '0';
2317 :
2318 17427 : current_char = peek_input ();
2319 :
2320 : // parse initial decimal integer (or first integer part of float) literal
2321 17427 : auto initial_decimal = parse_in_decimal ();
2322 17427 : str += std::get<0> (initial_decimal);
2323 17427 : length += std::get<1> (initial_decimal);
2324 :
2325 : // detect float literal
2326 : //
2327 : // Note:
2328 : //
2329 : // We should not use is_float_digit () for this verification but instead
2330 : // directly ISDIGIT because rust does not support non digit values right after
2331 : // a dot.
2332 : // The following value is not legal in rust:
2333 : // let a = 3.e1;
2334 : // A `0` should be put between the dot and the exponent to be valid
2335 : // (eg. 3.0e1).
2336 17427 : if (current_char == '.' && ISDIGIT (peek_input (1).value))
2337 : {
2338 : // float with a '.', parse another decimal into it
2339 :
2340 : // add . to str
2341 365 : str += current_char;
2342 365 : skip_input ();
2343 365 : current_char = peek_input ();
2344 365 : length++;
2345 :
2346 : // parse another decimal number for float
2347 365 : auto second_decimal = parse_in_decimal ();
2348 365 : str += std::get<0> (second_decimal);
2349 365 : length += std::get<1> (second_decimal);
2350 :
2351 : // parse in exponent part if it exists
2352 365 : auto exponent_pair = parse_in_exponent_part ();
2353 365 : str += exponent_pair.first;
2354 365 : length += exponent_pair.second;
2355 :
2356 365 : int suffix_start = str.length ();
2357 :
2358 : // parse in type suffix if it exists
2359 365 : auto suffix_pair = parse_in_suffix ();
2360 365 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2361 365 : str += suffix_pair.first;
2362 365 : length += suffix_pair.second;
2363 :
2364 365 : current_column += length;
2365 :
2366 365 : loc += length - 1;
2367 :
2368 365 : str.shrink_to_fit ();
2369 365 : return Token::make_float (loc, std::move (str), suffix_start, type_hint);
2370 365 : }
2371 17062 : else if (current_char == '.'
2372 17062 : && check_valid_float_dot_end (peek_input (1).value))
2373 : {
2374 : // float that is just an integer with a terminating '.' character
2375 :
2376 : // add . to str
2377 3 : str += current_char;
2378 3 : skip_input ();
2379 3 : current_char = peek_input ();
2380 3 : length++;
2381 :
2382 : // type hint not allowed
2383 :
2384 3 : current_column += length;
2385 :
2386 3 : loc += length - 1;
2387 :
2388 3 : str.shrink_to_fit ();
2389 3 : auto suffix_start = str.length ();
2390 3 : return Token::make_float (loc, std::move (str), suffix_start,
2391 3 : CORETYPE_UNKNOWN);
2392 : }
2393 17059 : else if (current_char == 'E' || current_char == 'e')
2394 : {
2395 : // exponent float with no '.' character
2396 :
2397 : // parse exponent part
2398 0 : auto exponent_pair = parse_in_exponent_part ();
2399 0 : str += exponent_pair.first;
2400 0 : length += exponent_pair.second;
2401 :
2402 0 : int suffix_start = str.length ();
2403 :
2404 : // parse in type suffix if it exists
2405 0 : auto suffix_pair = parse_in_suffix ();
2406 0 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2407 0 : str += suffix_pair.first;
2408 0 : length += suffix_pair.second;
2409 :
2410 0 : current_column += length;
2411 :
2412 0 : loc += length - 1;
2413 :
2414 0 : str.shrink_to_fit ();
2415 0 : return Token::make_float (loc, std::move (str), suffix_start, type_hint);
2416 0 : }
2417 : else
2418 : {
2419 : // is an integer
2420 :
2421 17059 : int suffix_start = str.length ();
2422 :
2423 : // parse in type suffix if it exists
2424 17059 : auto suffix_pair = parse_in_suffix ();
2425 17059 : str += suffix_pair.first;
2426 :
2427 17059 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
2428 :
2429 : /* A "real" pure decimal doesn't have a suffix and no zero prefix. */
2430 17059 : bool pure_decimal = std::get<2> (initial_decimal);
2431 17044 : if (pure_decimal && (!first_zero || suffix_start == 1)
2432 34101 : && suffix_pair.first.empty ())
2433 : type_hint = CORETYPE_PURE_DECIMAL;
2434 :
2435 17059 : length += suffix_pair.second;
2436 :
2437 17059 : current_column += length;
2438 :
2439 17059 : loc += length - 1;
2440 :
2441 17059 : str.shrink_to_fit ();
2442 17059 : return Token::make_int (loc, std::move (str), suffix_start,
2443 17059 : IntegerLiteralBase::Decimal, type_hint);
2444 17059 : }
2445 17427 : }
2446 :
2447 : TokenPtr
2448 839 : Lexer::parse_char_or_lifetime (location_t loc)
2449 : {
2450 839 : int length = 1;
2451 :
2452 839 : current_char = peek_input ();
2453 839 : if (current_char.is_eof ())
2454 0 : return nullptr;
2455 :
2456 : // parse escaped char literal
2457 839 : if (current_char.value == '\\')
2458 : {
2459 : // parse escape
2460 23 : auto utf8_escape_pair = parse_utf8_escape ();
2461 23 : Codepoint escaped_char = std::get<0> (utf8_escape_pair);
2462 23 : length += std::get<1> (utf8_escape_pair);
2463 :
2464 23 : if (peek_input ().value != '\'')
2465 : {
2466 0 : rust_error_at (get_current_location (), "unended character literal");
2467 : }
2468 : else
2469 : {
2470 23 : skip_input ();
2471 23 : current_char = peek_input ();
2472 23 : length++;
2473 : }
2474 :
2475 23 : current_column += length;
2476 :
2477 23 : loc += length - 1;
2478 :
2479 23 : return Token::make_char (loc, escaped_char);
2480 : }
2481 : else
2482 : {
2483 816 : skip_input ();
2484 :
2485 816 : if (peek_input ().value == '\'')
2486 : {
2487 : // parse non-escaped char literal
2488 203 : Codepoint non_escaped_char = current_char;
2489 :
2490 : // skip the ' character
2491 203 : skip_input ();
2492 203 : current_char = peek_input ();
2493 :
2494 : // TODO fix due to different widths of utf-8 chars?
2495 203 : current_column += 3;
2496 :
2497 203 : loc += 2;
2498 :
2499 203 : return Token::make_char (loc, non_escaped_char);
2500 : }
2501 613 : else if (is_identifier_start (current_char.value))
2502 : {
2503 : // parse lifetime name
2504 613 : std::string str;
2505 1226 : str += current_char.as_string ();
2506 613 : length++;
2507 :
2508 613 : current_char = peek_input ();
2509 1978 : while (is_identifier_continue (current_char.value))
2510 : {
2511 1504 : str += current_char.as_string ();
2512 752 : skip_input ();
2513 752 : current_char = peek_input ();
2514 752 : length++;
2515 : }
2516 :
2517 613 : current_column += length;
2518 :
2519 613 : loc += length - 1;
2520 :
2521 : // TODO some keywords cannot be used for a lifetime label #2306
2522 : // https://doc.rust-lang.org/reference/tokens.html
2523 :
2524 613 : str.shrink_to_fit ();
2525 613 : return Token::make_lifetime (loc, std::move (str));
2526 613 : }
2527 : else
2528 : {
2529 0 : rust_error_at (
2530 : get_current_location (),
2531 : "expected %' after character constant in character literal");
2532 0 : return nullptr;
2533 : }
2534 : }
2535 : }
2536 :
2537 : void
2538 102 : Lexer::split_current_token (TokenId new_left, TokenId new_right)
2539 : {
2540 : /* TODO: assert that this TokenId is a "simple token" like punctuation and not
2541 : * like "IDENTIFIER"? */
2542 102 : location_t current_loc = peek_token ()->get_locus ();
2543 102 : TokenPtr new_left_tok = Token::make (new_left, current_loc);
2544 102 : TokenPtr new_right_tok = Token::make (new_right, current_loc + 1);
2545 :
2546 102 : token_queue.replace_current_value (std::move (new_left_tok));
2547 102 : token_queue.insert (1, std::move (new_right_tok));
2548 102 : }
2549 :
2550 : void
2551 2 : Lexer::split_current_token (std::vector<TokenPtr> new_tokens)
2552 : {
2553 2 : rust_assert (new_tokens.size () > 0);
2554 4 : token_queue.replace_current_value (new_tokens[0]);
2555 :
2556 5 : for (size_t i = 1; i < new_tokens.size (); i++)
2557 : {
2558 6 : token_queue.insert (i, new_tokens[i]);
2559 : }
2560 2 : }
2561 :
2562 : void
2563 177397 : Lexer::start_line (int current_line, int current_column)
2564 : {
2565 177397 : if (line_map)
2566 177397 : linemap_line_start (line_table, current_line, current_column);
2567 177397 : }
2568 :
2569 : } // namespace Rust
2570 :
2571 : #if CHECKING_P
2572 :
2573 : namespace selftest {
2574 :
2575 : // Checks if `src` has the same contents as the given characters
2576 : static void
2577 6 : assert_source_content (Rust::InputSource &src,
2578 : const std::vector<uint32_t> &expected)
2579 : {
2580 6 : Rust::Codepoint src_char = src.next ();
2581 41 : for (auto expected_char : expected)
2582 : {
2583 : // Make sure that `src` is not shorter than `expected`
2584 35 : ASSERT_FALSE (src_char.is_eof ());
2585 : // Checks skipped character is expeceted one.
2586 35 : ASSERT_EQ (src_char.value, expected_char);
2587 35 : src_char = src.next ();
2588 : }
2589 : // Checks if `src` and `chars` has the same length.
2590 6 : ASSERT_TRUE (src_char.is_eof ());
2591 6 : }
2592 :
2593 : static void
2594 4 : test_buffer_input_source (std::string str,
2595 : const std::vector<uint32_t> &expected)
2596 : {
2597 4 : Rust::BufferInputSource source (str, 0);
2598 4 : assert_source_content (source, expected);
2599 4 : }
2600 :
2601 : static void
2602 2 : test_file_input_source (std::string str, const std::vector<uint32_t> &expected)
2603 : {
2604 2 : FILE *tmpf = tmpfile ();
2605 : // Moves to the first character
2606 2 : fputs (str.c_str (), tmpf);
2607 2 : std::rewind (tmpf);
2608 2 : Rust::FileInputSource source (tmpf);
2609 2 : assert_source_content (source, expected);
2610 2 : }
2611 :
2612 : void
2613 1 : rust_input_source_test ()
2614 : {
2615 : // ASCII
2616 1 : std::string src = (const char *) u8"_abcde\tXYZ\v\f";
2617 1 : std::vector<uint32_t> expected = {u'_', u'a', u'b', u'c', u'd', u'e',
2618 1 : u'\t', u'X', u'Y', u'Z', u'\v', u'\f'};
2619 2 : test_buffer_input_source (src, expected);
2620 :
2621 : // BOM
2622 1 : src = (const char *) u8"\xef\xbb\xbfOK";
2623 1 : expected = {u'O', u'K'};
2624 2 : test_buffer_input_source (src, expected);
2625 :
2626 : // Russian
2627 1 : src = (const char *) u8"приве́т";
2628 1 : expected = {u'п',
2629 : u'р',
2630 : u'и',
2631 : u'в',
2632 : 0x0435 /* CYRILLIC SMALL LETTER IE е */,
2633 : 0x301 /* COMBINING ACUTE ACCENT ́ */,
2634 1 : u'т'};
2635 2 : test_buffer_input_source (src, expected);
2636 :
2637 1 : src = (const char *) u8"❤️🦀";
2638 1 : expected = {0x2764 /* HEAVY BLACK HEART */,
2639 1 : 0xfe0f /* VARIATION SELECTOR-16 */, U'🦀'};
2640 2 : test_buffer_input_source (src, expected);
2641 :
2642 1 : src = (const char *) u8"こんにちは";
2643 1 : expected = {u'こ', u'ん', u'に', u'ち', u'は'};
2644 2 : test_file_input_source (src, expected);
2645 :
2646 1 : src = (const char *) u8"👮♂👩⚕";
2647 1 : expected
2648 : = {0x1f46e /* POLICE OFFICER */, 0x200d /* ZERO WIDTH JOINER */,
2649 : 0x2642 /* MALE SIGN */, 0x1f469 /* WOMAN */,
2650 1 : 0x200d /* ZERO WIDTH JOINER */, 0x2695 /* STAFF OF AESCULAPIUS */};
2651 2 : test_file_input_source (src, expected);
2652 1 : }
2653 :
2654 : } // namespace selftest
2655 :
2656 : #endif // CHECKING_P
|
