LCOV - code coverage report
Current view: top level - gcc/rust/lex - rust-lex.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 93.9 % 1364 1281
Test Date: 2026-11-14 16:39:50 Functions: 94.4 % 54 51
Legend: Lines:     hit not hit

            Line data    Source code
       1              : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
       2              : 
       3              : // This file is part of GCC.
       4              : 
       5              : // GCC is free software; you can redistribute it and/or modify it under
       6              : // the terms of the GNU General Public License as published by the Free
       7              : // Software Foundation; either version 3, or (at your option) any later
       8              : // version.
       9              : 
      10              : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      11              : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
      12              : // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      13              : // for more details.
      14              : 
      15              : // You should have received a copy of the GNU General Public License
      16              : // along with GCC; see the file COPYING3.  If not see
      17              : // <http://www.gnu.org/licenses/>.
      18              : 
      19              : #include "rust-codepoint.h"
      20              : #include "rust-system.h"
      21              : #include "rust-lex.h"
      22              : #include "rust-diagnostics.h"
      23              : #include "rust-linemap.h"
      24              : #include "rust-edition.h"
      25              : #include "safe-ctype.h"
      26              : #include "cpplib.h"
      27              : #include "rust-keyword-values.h"
      28              : 
      29              : namespace Rust {
      30              : // TODO: move to separate compilation unit?
      31              : // overload += for uint32_t to allow 32-bit encoded utf-8 to be added
      32              : std::string &
      33     48394467 : operator+= (std::string &str, Codepoint char32)
      34              : {
      35     48394467 :   if (char32.value < 0x80)
      36              :     {
      37     48344071 :       str += static_cast<char> (char32.value);
      38              :     }
      39        50396 :   else if (char32.value < (0x1F + 1) << (1 * 6))
      40              :     {
      41        17907 :       str += static_cast<char> (0xC0 | ((char32.value >> 6) & 0x1F));
      42        17907 :       str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
      43              :     }
      44        32489 :   else if (char32.value < (0x0F + 1) << (2 * 6))
      45              :     {
      46        24804 :       str += static_cast<char> (0xE0 | ((char32.value >> 12) & 0x0F));
      47        24804 :       str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
      48        24804 :       str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
      49              :     }
      50         7685 :   else if (char32.value < (0x07 + 1) << (3 * 6))
      51              :     {
      52         7681 :       str += static_cast<char> (0xF0 | ((char32.value >> 18) & 0x07));
      53         7681 :       str += static_cast<char> (0x80 | ((char32.value >> 12) & 0x3F));
      54         7681 :       str += static_cast<char> (0x80 | ((char32.value >> 6) & 0x3F));
      55         7681 :       str += static_cast<char> (0x80 | ((char32.value >> 0) & 0x3F));
      56              :     }
      57              :   else
      58              :     {
      59            4 :       rust_debug ("Invalid unicode codepoint found: '%u' ", char32.value);
      60              :     }
      61     48394467 :   return str;
      62              : }
      63              : 
      64              : std::string
      65     31670036 : Codepoint::as_string ()
      66              : {
      67     31670036 :   std::string str;
      68              : 
      69              :   // str += Codepoint (value);
      70     31670036 :   str += *this;
      71              : 
      72     31670036 :   return str;
      73              : }
      74              : 
      75              : /* Includes all allowable float digits EXCEPT _ and . as that needs lookahead
      76              :  * for handling. */
      77              : bool
      78            0 : is_float_digit (uint32_t number)
      79              : {
      80            0 :   return ISDIGIT (number) || number == 'E' || number == 'e';
      81              : }
      82              : 
      83              : /* Basically ISXDIGIT from safe-ctype but may change if Rust's encoding or
      84              :  * whatever is different */
      85              : bool
      86       483224 : is_x_digit (uint32_t number)
      87              : {
      88       483224 :   return ISXDIGIT (number);
      89              : }
      90              : 
      91              : bool
      92           87 : is_octal_digit (uint32_t number)
      93              : {
      94           87 :   return number >= '0' && number <= '7';
      95              : }
      96              : 
      97              : bool
      98       153740 : is_bin_digit (uint32_t number)
      99              : {
     100       153740 :   return number == '0' || number == '1';
     101              : }
     102              : 
     103              : bool
     104        84900 : check_valid_float_dot_end (uint32_t character)
     105              : {
     106        84900 :   return character != '.' && character != '_' && !ISALPHA (character);
     107              : }
     108              : 
     109              : bool
     110        33046 : is_whitespace (uint32_t character)
     111              : {
     112              :   // https://doc.rust-lang.org/reference/whitespace.html
     113        33046 :   switch (character)
     114              :     {
     115              :     case '\t':
     116              :     case '\n':
     117              :     case '\v':
     118              :     case '\f':
     119              :     case '\r':
     120              :     case ' ':
     121              :     case 0x0085: // next line
     122              :     case 0x200e: // left-to-right mark
     123              :     case 0x200f: // right-to-left mark
     124              :     case 0x2028: // line separator
     125              :     case 0x2029: // paragraph separator
     126              :       return true;
     127         5100 :     default:
     128         5100 :       return false;
     129              :     }
     130              : }
     131              : 
     132              : bool
     133       106787 : is_non_decimal_int_literal_separator (uint32_t character)
     134              : {
     135       106787 :   return character == 'x' || character == 'o' || character == 'b';
     136              : }
     137              : 
     138              : bool
     139      2850210 : is_identifier_start (uint32_t codepoint)
     140              : {
     141      2850210 :   return (cpp_check_xid_property (codepoint) & CPP_XID_START)
     142      2850210 :          || codepoint == '_';
     143              : }
     144              : 
     145              : bool
     146     11437634 : is_identifier_continue (uint32_t codepoint)
     147              : {
     148     11437634 :   return cpp_check_xid_property (codepoint) & CPP_XID_CONTINUE;
     149              : }
     150              : 
     151          164 : Lexer::Lexer (const std::string &input, Linemap *linemap)
     152          164 :   : input (RAIIFile::create_error ()), current_line (1), current_column (1),
     153          164 :     line_map (linemap), dump_lex_out ({}),
     154          164 :     raw_input_source (new BufferInputSource (input, 0)),
     155          164 :     input_queue{*raw_input_source}, token_queue (TokenSource (this))
     156          164 : {}
     157              : 
     158         7108 : Lexer::Lexer (const char *filename, RAIIFile file_input, Linemap *linemap,
     159              :               tl::optional<std::ofstream &> dump_lex_opt)
     160         7108 :   : input (std::move (file_input)), current_line (1), current_column (1),
     161         7108 :     line_map (linemap), dump_lex_out (dump_lex_opt),
     162        14216 :     raw_input_source (new FileInputSource (input.get_raw ())),
     163        14216 :     input_queue{*raw_input_source}, token_queue (TokenSource (this))
     164              : {
     165              :   // inform line_table that file is being entered and is in line 1
     166              :   // TODO: filename needs to live ~forever, is there a better way than xstrdup?
     167              :   //                               ^ (as long as the linemap does)
     168         7108 :   if (linemap)
     169         7108 :     line_map->start_file (xstrdup (filename), current_line);
     170         7108 : }
     171              : 
     172         7249 : Lexer::~Lexer ()
     173              : {
     174              :   /* ok apparently stop (which is equivalent of original code in destructor) is
     175              :    * meant to be called after all files have finished parsing, for cleanup. On
     176              :    * the other hand, actual code that it calls to leave a certain line map is
     177              :    * mentioned in GCC docs as being useful for "just leaving an included header"
     178              :    * and stuff like that, so this line mapping functionality may need fixing.
     179              :    * FIXME: find out whether this occurs. */
     180              : 
     181              :   // line_map->stop();
     182         7249 : }
     183              : 
     184              : bool
     185         5410 : Lexer::input_source_is_valid_utf8 ()
     186              : {
     187         5410 :   return raw_input_source->is_valid ();
     188              : }
     189              : 
     190              : location_t
     191     16137398 : Lexer::get_current_location ()
     192              : {
     193     16137398 :   if (line_map)
     194     16137169 :     return linemap_position_for_column (line_table, current_column);
     195              :   else
     196              :     // If we have no linemap, we're lexing something without proper locations
     197              :     return UNDEF_LOCATION;
     198              : }
     199              : 
     200              : Codepoint
     201     54802437 : Lexer::peek_input (int n)
     202              : {
     203     54802437 :   return input_queue.peek (n);
     204              : }
     205              : 
     206              : Codepoint
     207     51693125 : Lexer::peek_input ()
     208              : {
     209     51693125 :   return peek_input (0);
     210              : }
     211              : 
     212              : void
     213     45789773 : Lexer::skip_input (int n)
     214              : {
     215     45789773 :   input_queue.skip (n);
     216     45789773 : }
     217              : 
     218              : void
     219     45411005 : Lexer::skip_input ()
     220              : {
     221     45411005 :   skip_input (0);
     222     45411005 : }
     223              : 
     224              : void
     225      7028607 : Lexer::skip_token (int n)
     226              : {
     227              :   // dump tokens if dump-lex option is enabled
     228      7028607 :   if (dump_lex_out.has_value ())
     229           55 :     dump_and_skip (n);
     230              :   else
     231      7028552 :     token_queue.skip (n);
     232      7028607 : }
     233              : 
     234              : void
     235           55 : Lexer::dump_and_skip (int n)
     236              : {
     237           55 :   std::ofstream &out = dump_lex_out.value ();
     238           55 :   bool found_eof = false;
     239           55 :   const_TokenPtr tok;
     240          110 :   for (int i = 0; i < n + 1; i++)
     241              :     {
     242           55 :       if (!found_eof)
     243              :         {
     244           55 :           tok = peek_token ();
     245           55 :           found_eof |= tok->get_id () == Rust::END_OF_FILE;
     246              : 
     247           55 :           location_t loc = tok->get_locus ();
     248              : 
     249           55 :           out << "<id=";
     250           55 :           out << tok->token_id_to_str ();
     251           55 :           out << (tok->should_have_str ()
     252          167 :                     ? (std::string (", text=") + tok->get_str ()
     253          167 :                        + std::string (", typehint=")
     254           93 :                        + std::string (tok->get_type_hint_str ()))
     255           55 :                     : "")
     256          110 :               << " ";
     257          110 :           out << Linemap::location_to_string (loc) << '\n';
     258              :         }
     259              : 
     260           55 :       token_queue.skip (0);
     261              :     }
     262           55 : }
     263              : 
     264              : void
     265            0 : Lexer::replace_current_token (TokenPtr replacement)
     266              : {
     267            0 :   token_queue.replace_current_value (replacement);
     268              : 
     269            0 :   rust_debug ("called 'replace_current_token' - this is deprecated");
     270            0 : }
     271              : 
     272              : /* Determines whether the string passed in is a keyword or not. If it is, it
     273              :  * returns the keyword name.  */
     274              : TokenId
     275      2138174 : Lexer::classify_keyword (const std::string &str)
     276              : {
     277      2138174 :   auto &keywords = Rust::Values::Keywords::keywords_tokens;
     278      2138174 :   auto keyword = keywords.find (str);
     279              : 
     280      2138174 :   if (keyword == keywords.end ())
     281              :     return IDENTIFIER;
     282              : 
     283       522171 :   auto id = keyword->second;
     284              : 
     285              :   // We now have the expected token ID of the reserved keyword. However, some
     286              :   // keywords are reserved starting in certain editions. For example, `try` is
     287              :   // only a reserved keyword in editions >=2018. The language might gain new
     288              :   // reserved keywords in the future.
     289              :   //
     290              :   // https://doc.rust-lang.org/reference/keywords.html#reserved-keywords
     291              : 
     292              :   // `try` is not a reserved keyword before 2018
     293       522171 :   if (get_rust_edition () == Edition::E2015 && id == TRY)
     294            1 :     return IDENTIFIER;
     295              : 
     296              :   return id;
     297              : }
     298              : 
     299              : TokenPtr
     300      7052698 : Lexer::build_token ()
     301              : {
     302              :   // loop to go through multiple characters to build a single token
     303     15993388 :   while (true)
     304              :     {
     305     15993388 :       location_t loc = get_current_location ();
     306              : 
     307     15993388 :       current_char = peek_input ();
     308     15993388 :       skip_input ();
     309              : 
     310              :       // detect shebang
     311              :       // Must be the first thing on the first line, starting with #!
     312              :       // But since an attribute can also start with an #! we don't count it as a
     313              :       // shebang line when after any whitespace or comments there is a [. If it
     314              :       // is a shebang line we simple drop the line. Otherwise we don't consume
     315              :       // any characters and fall through to the real tokenizer.
     316        46100 :       if (current_line == 1 && current_column == 1 && current_char == '#'
     317     16039488 :           && peek_input () == '!')
     318              :         {
     319              :           int n = 1;
     320         3844 :           while (true)
     321              :             {
     322         3830 :               Codepoint next_char = peek_input (n);
     323         3830 :               if (is_whitespace (next_char.value))
     324            7 :                 n++;
     325         3823 :               else if ((next_char == '/' && peek_input (n + 1) == '/'
     326            7 :                         && peek_input (n + 2) != '!'
     327            7 :                         && peek_input (n + 2) != '/')
     328         3844 :                        || (next_char == '/' && peek_input (n + 1) == '/'
     329            0 :                            && peek_input (n + 2) == '/'
     330            7 :                            && peek_input (n + 3) == '/'))
     331              :                 {
     332              :                   // two // or four ////
     333              :                   // A single line comment
     334              :                   // (but not an inner or outer doc comment)
     335            7 :                   n += 2;
     336            7 :                   next_char = peek_input (n);
     337          119 :                   while (next_char != '\n' && !next_char.is_eof ())
     338              :                     {
     339          112 :                       n++;
     340          112 :                       next_char = peek_input (n);
     341              :                     }
     342            7 :                   if (next_char == '\n')
     343            7 :                     n++;
     344              :                 }
     345         3816 :               else if (next_char == '/' && peek_input (n + 1) == '*'
     346            0 :                        && peek_input (n + 2) == '*'
     347         3816 :                        && peek_input (n + 3) == '/')
     348              :                 {
     349              :                   /**/
     350            0 :                   n += 4;
     351              :                 }
     352         3816 :               else if (next_char == '/' && peek_input (n + 1) == '*'
     353            0 :                        && peek_input (n + 2) == '*' && peek_input (n + 3) == '*'
     354         3816 :                        && peek_input (n + 4) == '/')
     355              :                 {
     356              :                   /***/
     357            0 :                   n += 5;
     358              :                 }
     359         3816 :               else if ((next_char == '/' && peek_input (n + 1) == '*'
     360            0 :                         && peek_input (n + 2) != '*'
     361            0 :                         && peek_input (n + 2) != '!')
     362         3837 :                        || (next_char == '/' && peek_input (n + 1) == '*'
     363            0 :                            && peek_input (n + 2) == '*'
     364            0 :                            && peek_input (n + 3) == '*'))
     365              :                 {
     366              :                   // one /* or three /***
     367              :                   // Start of a block comment
     368              :                   // (but not an inner or outer doc comment)
     369            0 :                   n += 2;
     370            0 :                   int level = 1;
     371            0 :                   while (level > 0)
     372              :                     {
     373            0 :                       if (peek_input (n).is_eof ())
     374              :                         break;
     375            0 :                       else if (peek_input (n) == '/'
     376            0 :                                && peek_input (n + 1) == '*')
     377              :                         {
     378            0 :                           n += 2;
     379            0 :                           level += 1;
     380              :                         }
     381            0 :                       else if (peek_input (n) == '*'
     382            0 :                                && peek_input (n + 1) == '/')
     383              :                         {
     384            0 :                           n += 2;
     385            0 :                           level -= 1;
     386              :                         }
     387              :                       else
     388            0 :                         n++;
     389              :                     }
     390              :                 }
     391         3816 :               else if (next_char != '[')
     392              :                 {
     393              :                   // definitely shebang, ignore the first line
     394          518 :                   while (current_char != '\n' && !current_char.is_eof ())
     395              :                     {
     396          490 :                       current_char = peek_input ();
     397          490 :                       skip_input ();
     398              :                     }
     399              : 
     400              :                   // newline
     401           28 :                   current_line++;
     402           28 :                   current_column = 1;
     403              :                   // tell line_table that new line starts
     404           28 :                   start_line (current_line, max_column_hint);
     405           28 :                   break;
     406              :                 }
     407              :               else
     408              :                 break; /* Definitely not a shebang line. */
     409           14 :             }
     410              :         }
     411              : 
     412              :       // return end of file token if end of file
     413     15993388 :       if (current_char.is_eof ())
     414         7564 :         return Token::make (END_OF_FILE, loc);
     415              : 
     416              :       // if not end of file, start tokenising
     417     15985824 :       switch (current_char.value)
     418              :         {
     419              :         /* ignore whitespace characters for tokens but continue updating
     420              :          * location */
     421       973526 :         case '\n':   // newline
     422       973526 :         case 0x0085: // next line
     423       973526 :         case 0x2028: // line separator
     424       973526 :         case 0x2029: // paragraph separator
     425       973526 :           current_line++;
     426       973526 :           current_column = 1;
     427              :           // tell line_table that new line starts
     428       973526 :           start_line (current_line, max_column_hint);
     429       973526 :           continue;
     430          252 :         case '\r': // cr
     431              :           // Ignore, we expect a newline (lf) soon.
     432          252 :           continue;
     433      7917870 :         case ' ': // space
     434      7917870 :           current_column++;
     435      7917870 :           continue;
     436          114 :         case '\t': // horizontal tab
     437              :           // width of a tab is not well-defined, assume 8 spaces
     438          114 :           current_column += 8;
     439          114 :           continue;
     440           28 :         case '\v':   // vertical tab
     441           28 :         case 0x000c: // form feed
     442           28 :         case 0x200e: // left-to-right mark
     443           28 :         case 0x200f: // right-to-left mark
     444              :           // Ignored.
     445           28 :           continue;
     446              : 
     447              :         // punctuation - actual tokens
     448       271622 :         case '=':
     449       271622 :           if (peek_input () == '>')
     450              :             {
     451              :               // match arm arrow
     452        31696 :               skip_input ();
     453        31696 :               current_column += 2;
     454        31696 :               loc += 1;
     455              : 
     456        31696 :               return Token::make (MATCH_ARROW, loc);
     457              :             }
     458       239926 :           else if (peek_input () == '=')
     459              :             {
     460              :               // equality operator
     461         3383 :               skip_input ();
     462         3383 :               current_column += 2;
     463         3383 :               loc += 1;
     464              : 
     465         3383 :               return Token::make (EQUAL_EQUAL, loc);
     466              :             }
     467              :           else
     468              :             {
     469              :               // assignment operator
     470       236543 :               current_column++;
     471       236543 :               return Token::make (EQUAL, loc);
     472              :             }
     473       495234 :         case '(':
     474       495234 :           current_column++;
     475       495234 :           return Token::make (LEFT_PAREN, loc);
     476        91719 :         case '-':
     477        91719 :           if (peek_input () == '>')
     478              :             {
     479              :               // return type specifier
     480        61251 :               skip_input ();
     481        61251 :               current_column += 2;
     482        61251 :               loc += 1;
     483              : 
     484        61251 :               return Token::make (RETURN_TYPE, loc);
     485              :             }
     486        30468 :           else if (peek_input () == '=')
     487              :             {
     488              :               // minus-assign
     489          869 :               skip_input ();
     490          869 :               current_column += 2;
     491          869 :               loc += 1;
     492              : 
     493          869 :               return Token::make (MINUS_EQ, loc);
     494              :             }
     495              :           else
     496              :             {
     497              :               // minus
     498        29599 :               current_column++;
     499        29599 :               return Token::make (MINUS, loc);
     500              :             }
     501         9536 :         case '+':
     502         9536 :           if (peek_input () == '=')
     503              :             {
     504              :               // add-assign
     505         1343 :               skip_input ();
     506         1343 :               current_column += 2;
     507         1343 :               loc += 1;
     508              : 
     509         1343 :               return Token::make (PLUS_EQ, loc);
     510              :             }
     511              :           else
     512              :             {
     513              :               // add
     514         8193 :               current_column++;
     515         8193 :               return Token::make (PLUS, loc);
     516              :             }
     517       495215 :         case ')':
     518       495215 :           current_column++;
     519       495215 :           return Token::make (RIGHT_PAREN, loc);
     520       182374 :         case ';':
     521       182374 :           current_column++;
     522       182374 :           return Token::make (SEMICOLON, loc);
     523        33068 :         case '*':
     524        33068 :           if (peek_input () == '=')
     525              :             {
     526              :               // multiplication-assign
     527           63 :               skip_input ();
     528           63 :               current_column += 2;
     529           63 :               loc += 1;
     530              : 
     531           63 :               return Token::make (ASTERISK_EQ, loc);
     532              :             }
     533              :           else
     534              :             {
     535              :               // multiplication
     536        33005 :               current_column++;
     537        33005 :               return Token::make (ASTERISK, loc);
     538              :             }
     539       752268 :         case ',':
     540       752268 :           current_column++;
     541       752268 :           return Token::make (COMMA, loc);
     542       425088 :         case '/':
     543       425088 :           if (peek_input () == '=')
     544              :             {
     545              :               // division-assign
     546          215 :               skip_input ();
     547          215 :               current_column += 2;
     548          215 :               loc += 1;
     549              : 
     550          215 :               return Token::make (DIV_EQ, loc);
     551              :             }
     552       424873 :           else if ((peek_input () == '/' && peek_input (1) != '!'
     553       400647 :                     && peek_input (1) != '/')
     554       800290 :                    || (peek_input () == '/' && peek_input (1) == '/'
     555       400647 :                        && peek_input (2) == '/'))
     556              :             {
     557              :               // two // or four ////
     558              :               // single line comment
     559              :               // (but not an inner or outer doc comment)
     560        47325 :               skip_input ();
     561        47325 :               current_column += 2;
     562        47325 :               current_char = peek_input ();
     563              : 
     564              :               // basically ignore until line finishes
     565      2476246 :               while (current_char != '\n' && !current_char.is_eof ())
     566              :                 {
     567      2381596 :                   skip_input ();
     568      2381596 :                   current_column++; // not used
     569      2381596 :                   current_char = peek_input ();
     570              :                 }
     571        47325 :               continue;
     572              :             }
     573       377548 :           else if (peek_input () == '/'
     574       377548 :                    && (peek_input (1) == '!' || peek_input (1) == '/'))
     575              :             {
     576              :               /* single line doc comment, inner or outer.  */
     577       374816 :               bool is_inner = peek_input (1) == '!';
     578       374816 :               skip_input (1);
     579       374816 :               current_column += 3;
     580              : 
     581       374816 :               std::string str;
     582       374816 :               str.reserve (32);
     583       374816 :               current_char = peek_input ();
     584     14304334 :               while (current_char != '\n')
     585              :                 {
     586     13554751 :                   skip_input ();
     587     13554751 :                   if (current_char == '\r')
     588              :                     {
     589           51 :                       Codepoint next_char = peek_input ();
     590           51 :                       if (next_char == '\n')
     591              :                         {
     592           49 :                           current_char = '\n';
     593           49 :                           break;
     594              :                         }
     595            2 :                       rust_error_at (
     596              :                         loc, "Isolated CR %<\\r%> not allowed in doc comment");
     597            2 :                       current_char = next_char;
     598            2 :                       continue;
     599            2 :                     }
     600     13554700 :                   if (current_char.is_eof ())
     601              :                     {
     602            0 :                       rust_error_at (
     603              :                         loc, ErrorCode::E0758,
     604              :                         "unexpected EOF while looking for end of comment");
     605            0 :                       break;
     606              :                     }
     607     13554700 :                   str += current_char;
     608     13554700 :                   current_char = peek_input ();
     609              :                 }
     610       374816 :               skip_input ();
     611       374816 :               current_line++;
     612       374816 :               current_column = 1;
     613              :               // tell line_table that new line starts
     614       374816 :               start_line (current_line, max_column_hint);
     615              : 
     616       374816 :               str.shrink_to_fit ();
     617              : 
     618       374816 :               loc += str.size () - 1;
     619       374816 :               if (is_inner)
     620        21494 :                 return Token::make_inner_doc_comment (loc, std::move (str));
     621              :               else
     622       353322 :                 return Token::make_outer_doc_comment (loc, std::move (str));
     623       374816 :             }
     624         2732 :           else if (peek_input () == '*' && peek_input (1) == '*'
     625         2817 :                    && peek_input (2) == '/')
     626              :             {
     627              :               /**/
     628           14 :               skip_input (2);
     629           14 :               current_column += 4;
     630           14 :               continue;
     631              :             }
     632         2718 :           else if (peek_input () == '*' && peek_input (1) == '*'
     633         2789 :                    && peek_input (2) == '*' && peek_input (3) == '/')
     634              :             {
     635              :               /***/
     636           14 :               skip_input (3);
     637           14 :               current_column += 5;
     638           14 :               continue;
     639              :             }
     640         2704 :           else if ((peek_input () == '*' && peek_input (1) != '!'
     641         1589 :                     && peek_input (1) != '*')
     642         2842 :                    || (peek_input () == '*' && peek_input (1) == '*'
     643         1589 :                        && peek_input (2) == '*'))
     644              :             {
     645              :               // one /* or three /***
     646              :               // block comment
     647              :               // (but not an inner or outer doc comment)
     648         1546 :               skip_input ();
     649         1546 :               current_column += 2;
     650              : 
     651         1546 :               int level = 1;
     652       144338 :               while (level > 0)
     653              :                 {
     654       142793 :                   current_char = peek_input ();
     655              : 
     656       142793 :                   if (current_char.is_eof ())
     657              :                     {
     658            1 :                       rust_error_at (
     659              :                         loc, ErrorCode::E0758,
     660              :                         "unexpected EOF while looking for end of comment");
     661            1 :                       break;
     662              :                     }
     663              : 
     664              :                   // if /* found
     665       142792 :                   if (current_char == '/' && peek_input (1) == '*')
     666              :                     {
     667              :                       // skip /* characters
     668           49 :                       skip_input (1);
     669              : 
     670           49 :                       current_column += 2;
     671              : 
     672           49 :                       level += 1;
     673           49 :                       continue;
     674              :                     }
     675              : 
     676              :                   // ignore until */ is found
     677       142743 :                   if (current_char == '*' && peek_input (1) == '/')
     678              :                     {
     679              :                       // skip */ characters
     680         1594 :                       skip_input (1);
     681              : 
     682         1594 :                       current_column += 2;
     683              : 
     684         1594 :                       level -= 1;
     685         1594 :                       continue;
     686              :                     }
     687              : 
     688       141149 :                   if (current_char == '\n')
     689              :                     {
     690         2981 :                       skip_input ();
     691         2981 :                       current_line++;
     692         2981 :                       current_column = 1;
     693              :                       // tell line_table that new line starts
     694         2981 :                       start_line (current_line, max_column_hint);
     695         2981 :                       continue;
     696              :                     }
     697              : 
     698       138168 :                   skip_input ();
     699       138168 :                   current_column++;
     700              :                 }
     701              : 
     702              :               // refresh new token
     703         1546 :               continue;
     704         1546 :             }
     705         1158 :           else if (peek_input () == '*'
     706         1158 :                    && (peek_input (1) == '!' || peek_input (1) == '*'))
     707              :             {
     708              :               // block doc comment, inner /*! or outer /**
     709          124 :               bool is_inner = peek_input (1) == '!';
     710          124 :               skip_input (1);
     711          124 :               current_column += 3;
     712              : 
     713          124 :               std::string str;
     714          124 :               str.reserve (96);
     715              : 
     716          124 :               int level = 1;
     717          124 :               while (level > 0)
     718              :                 {
     719        45757 :                   current_char = peek_input ();
     720              : 
     721        45757 :                   if (current_char.is_eof ())
     722              :                     {
     723            0 :                       rust_error_at (
     724              :                         loc, ErrorCode::E0758,
     725              :                         "unexpected EOF while looking for end of comment");
     726            0 :                       break;
     727              :                     }
     728              : 
     729              :                   // if /* found
     730        45757 :                   if (current_char == '/' && peek_input (1) == '*')
     731              :                     {
     732              :                       // skip /* characters
     733           84 :                       skip_input (1);
     734           84 :                       current_column += 2;
     735              : 
     736           84 :                       level += 1;
     737           84 :                       str += "/*";
     738           84 :                       continue;
     739              :                     }
     740              : 
     741              :                   // ignore until */ is found
     742        45673 :                   if (current_char == '*' && peek_input (1) == '/')
     743              :                     {
     744              :                       // skip */ characters
     745          208 :                       skip_input (1);
     746          208 :                       current_column += 2;
     747              : 
     748          208 :                       level -= 1;
     749          208 :                       if (level > 0)
     750           84 :                         str += "*/";
     751          208 :                       continue;
     752              :                     }
     753              : 
     754        45465 :                   if (current_char == '\r' && peek_input (1) != '\n')
     755            2 :                     rust_error_at (
     756              :                       loc, "Isolated CR %<\\r%> not allowed in doc comment");
     757              : 
     758        45465 :                   if (current_char == '\n')
     759              :                     {
     760          904 :                       skip_input ();
     761          904 :                       current_line++;
     762          904 :                       current_column = 1;
     763              :                       // tell line_table that new line starts
     764          904 :                       start_line (current_line, max_column_hint);
     765          904 :                       str += '\n';
     766          904 :                       continue;
     767              :                     }
     768              : 
     769        44561 :                   str += current_char;
     770        44561 :                   skip_input ();
     771        44561 :                   current_column++;
     772              :                 }
     773              : 
     774          124 :               str.shrink_to_fit ();
     775              : 
     776          124 :               loc += str.size () - 1;
     777          124 :               if (is_inner)
     778           81 :                 return Token::make_inner_doc_comment (loc, std::move (str));
     779              :               else
     780           43 :                 return Token::make_outer_doc_comment (loc, std::move (str));
     781          124 :             }
     782              :           else
     783              :             {
     784              :               // division
     785         1034 :               current_column++;
     786         1034 :               return Token::make (DIV, loc);
     787              :             }
     788          768 :         case '%':
     789          768 :           if (peek_input () == '=')
     790              :             {
     791              :               // modulo-assign
     792           23 :               skip_input ();
     793           23 :               current_column += 2;
     794           23 :               loc += 1;
     795              : 
     796           23 :               return Token::make (PERCENT_EQ, loc);
     797              :             }
     798              :           else
     799              :             {
     800              :               // modulo
     801          745 :               current_column++;
     802          745 :               return Token::make (PERCENT, loc);
     803              :             }
     804          487 :         case '^':
     805          487 :           if (peek_input () == '=')
     806              :             {
     807              :               // xor-assign?
     808          236 :               skip_input ();
     809          236 :               current_column += 2;
     810          236 :               loc += 1;
     811              : 
     812          236 :               return Token::make (CARET_EQ, loc);
     813              :             }
     814              :           else
     815              :             {
     816              :               // xor?
     817          251 :               current_column++;
     818          251 :               return Token::make (CARET, loc);
     819              :             }
     820        71101 :         case '<':
     821        71101 :           if (peek_input () == '<')
     822              :             {
     823         9383 :               if (peek_input (1) == '=')
     824              :                 {
     825              :                   // left-shift assign
     826           71 :                   skip_input (1);
     827           71 :                   current_column += 3;
     828           71 :                   loc += 2;
     829              : 
     830           71 :                   return Token::make (LEFT_SHIFT_EQ, loc);
     831              :                 }
     832              :               else
     833              :                 {
     834              :                   // left-shift
     835         9312 :                   skip_input ();
     836         9312 :                   current_column += 2;
     837         9312 :                   loc += 1;
     838              : 
     839         9312 :                   return Token::make (LEFT_SHIFT, loc);
     840              :                 }
     841              :             }
     842        61718 :           else if (peek_input () == '=')
     843              :             {
     844              :               // smaller than or equal to
     845         1321 :               skip_input ();
     846         1321 :               current_column += 2;
     847         1321 :               loc += 1;
     848              : 
     849         1321 :               return Token::make (LESS_OR_EQUAL, loc);
     850              :             }
     851              :           else
     852              :             {
     853              :               // smaller than
     854        60397 :               current_column++;
     855        60397 :               return Token::make (LEFT_ANGLE, loc);
     856              :             }
     857        61441 :           break;
     858        61441 :         case '>':
     859        61441 :           if (peek_input () == '>')
     860              :             {
     861         3918 :               if (peek_input (1) == '=')
     862              :                 {
     863              :                   // right-shift-assign
     864           41 :                   skip_input (1);
     865           41 :                   current_column += 3;
     866           41 :                   loc += 2;
     867              : 
     868           41 :                   return Token::make (RIGHT_SHIFT_EQ, loc);
     869              :                 }
     870              :               else
     871              :                 {
     872              :                   // right-shift
     873         3877 :                   skip_input ();
     874         3877 :                   current_column += 2;
     875         3877 :                   loc += 1;
     876              : 
     877         3877 :                   return Token::make (RIGHT_SHIFT, loc);
     878              :                 }
     879              :             }
     880        57523 :           else if (peek_input () == '=')
     881              :             {
     882              :               // larger than or equal to
     883         1595 :               skip_input ();
     884         1595 :               current_column += 2;
     885         1595 :               loc += 1;
     886              : 
     887         1595 :               return Token::make (GREATER_OR_EQUAL, loc);
     888              :             }
     889              :           else
     890              :             {
     891              :               // larger than
     892        55928 :               current_column++;
     893        55928 :               return Token::make (RIGHT_ANGLE, loc);
     894              :             }
     895       211626 :         case ':':
     896       211626 :           if (peek_input () == ':')
     897              :             {
     898              :               // scope resolution ::
     899        69191 :               skip_input ();
     900        69191 :               current_column += 2;
     901        69191 :               loc += 1;
     902              : 
     903        69191 :               return Token::make (SCOPE_RESOLUTION, loc);
     904              :             }
     905              :           else
     906              :             {
     907              :               // single colon :
     908       142435 :               current_column++;
     909       142435 :               return Token::make (COLON, loc);
     910              :             }
     911        66933 :         case '!':
     912              :           // no special handling for macros in lexer?
     913        66933 :           if (peek_input () == '=')
     914              :             {
     915              :               // not equal boolean operator
     916         3176 :               skip_input ();
     917         3176 :               current_column += 2;
     918         3176 :               loc += 1;
     919              : 
     920         3176 :               return Token::make (NOT_EQUAL, loc);
     921              :             }
     922              :           else
     923              :             {
     924              :               // not equal unary operator
     925        63757 :               current_column++;
     926              : 
     927        63757 :               return Token::make (EXCLAM, loc);
     928              :             }
     929         4596 :         case '?':
     930         4596 :           current_column++;
     931         4596 :           return Token::make (QUESTION_MARK, loc);
     932       154837 :         case '#':
     933       154837 :           current_column++;
     934       154837 :           return Token::make (HASH, loc);
     935       196911 :         case '[':
     936       196911 :           current_column++;
     937       196911 :           return Token::make (LEFT_SQUARE, loc);
     938       196904 :         case ']':
     939       196904 :           current_column++;
     940       196904 :           return Token::make (RIGHT_SQUARE, loc);
     941       158729 :         case '{':
     942       158729 :           current_column++;
     943       158729 :           return Token::make (LEFT_CURLY, loc);
     944       158676 :         case '}':
     945       158676 :           current_column++;
     946       158676 :           return Token::make (RIGHT_CURLY, loc);
     947          203 :         case '@':
     948          203 :           current_column++;
     949          203 :           return Token::make (PATTERN_BIND, loc);
     950        69503 :         case '$':
     951        69503 :           current_column++;
     952        69503 :           return Token::make (DOLLAR_SIGN, loc);
     953            0 :         case '~':
     954            0 :           current_column++;
     955            0 :           return Token::make (TILDE, loc);
     956            0 :         case '\\':
     957            0 :           current_column++;
     958            0 :           return Token::make (BACKSLASH, loc);
     959            0 :         case '`':
     960            0 :           current_column++;
     961            0 :           return Token::make (BACKTICK, loc);
     962         9986 :         case '|':
     963         9986 :           if (peek_input () == '=')
     964              :             {
     965              :               // bitwise or-assign?
     966          140 :               skip_input ();
     967          140 :               current_column += 2;
     968          140 :               loc += 1;
     969              : 
     970          140 :               return Token::make (PIPE_EQ, loc);
     971              :             }
     972         9846 :           else if (peek_input () == '|')
     973              :             {
     974              :               // logical or
     975         1289 :               skip_input ();
     976         1289 :               current_column += 2;
     977         1289 :               loc += 1;
     978              : 
     979         1289 :               return Token::make (OR, loc);
     980              :             }
     981              :           else
     982              :             {
     983              :               // bitwise or
     984         8557 :               current_column++;
     985              : 
     986         8557 :               return Token::make (PIPE, loc);
     987              :             }
     988        57285 :         case '&':
     989        57285 :           if (peek_input () == '=')
     990              :             {
     991              :               // bitwise and-assign?
     992           29 :               skip_input ();
     993           29 :               current_column += 2;
     994           29 :               loc += 1;
     995              : 
     996           29 :               return Token::make (AMP_EQ, loc);
     997              :             }
     998        57256 :           else if (peek_input () == '&')
     999              :             {
    1000              :               // logical and
    1001         1947 :               skip_input ();
    1002         1947 :               current_column += 2;
    1003         1947 :               loc += 1;
    1004              : 
    1005         1947 :               return Token::make (LOGICAL_AND, loc);
    1006              :             }
    1007              :           else
    1008              :             {
    1009              :               // bitwise and/reference
    1010        55309 :               current_column++;
    1011              : 
    1012        55309 :               return Token::make (AMP, loc);
    1013              :             }
    1014        84386 :         case '.':
    1015        84386 :           if (peek_input () == '.')
    1016              :             {
    1017         5801 :               if (peek_input (1) == '.')
    1018              :                 {
    1019              :                   // ellipsis
    1020          921 :                   skip_input (1);
    1021          921 :                   current_column += 3;
    1022          921 :                   loc += 2;
    1023              : 
    1024          921 :                   return Token::make (ELLIPSIS, loc);
    1025              :                 }
    1026         4880 :               else if (peek_input (1) == '=')
    1027              :                 {
    1028              :                   // ..=
    1029          679 :                   skip_input (1);
    1030          679 :                   current_column += 3;
    1031          679 :                   loc += 2;
    1032              : 
    1033          679 :                   return Token::make (DOT_DOT_EQ, loc);
    1034              :                 }
    1035              :               else
    1036              :                 {
    1037              :                   // ..
    1038         4201 :                   skip_input ();
    1039         4201 :                   current_column += 2;
    1040         4201 :                   loc += 1;
    1041              : 
    1042         4201 :                   return Token::make (DOT_DOT, loc);
    1043              :                 }
    1044              :             }
    1045              :           else /*if (!ISDIGIT (peek_input ()))*/
    1046              :             {
    1047              :               // single dot .
    1048              :               // Only if followed by a non-number - otherwise is float
    1049              :               // nope, float cannot start with '.'.
    1050        78585 :               current_column++;
    1051        78585 :               return Token::make (DOT, loc);
    1052              :             }
    1053      8891790 :         }
    1054              :       // TODO: special handling of _ in the lexer? instead of being identifier
    1055              : 
    1056              :       // byte character, byte string and raw byte string literals
    1057      2833538 :       if (current_char == 'b')
    1058              :         {
    1059        71145 :           if (peek_input () == '\'')
    1060         1086 :             return parse_byte_char (loc);
    1061        70059 :           else if (peek_input () == '"')
    1062          520 :             return parse_byte_string (loc);
    1063        69539 :           else if (peek_input () == 'r'
    1064        69539 :                    && (peek_input (1) == '#' || peek_input (1) == '"'))
    1065           32 :             return parse_raw_byte_string (loc);
    1066              :         }
    1067              : 
    1068              :       // C-style strings
    1069      2762393 :       else if (current_char == 'c' && peek_input () == '"')
    1070           15 :         return parse_c_string (loc);
    1071              : 
    1072              :       // raw identifiers and raw strings
    1073      2831885 :       if (current_char == 'r')
    1074              :         {
    1075        72588 :           Codepoint peek = peek_input ();
    1076        72588 :           Codepoint peek1 = peek_input (1);
    1077              : 
    1078              :           // TODO (tamaron) parse Unicode ident
    1079        72588 :           if (peek == '#' && is_identifier_start (peek1.value))
    1080              :             {
    1081          116 :               TokenPtr raw_ident_ptr = parse_raw_identifier (loc);
    1082          116 :               if (raw_ident_ptr != nullptr)
    1083          115 :                 return raw_ident_ptr;
    1084              :               else
    1085            1 :                 continue; /* input got parsed, it just wasn't valid. An error
    1086              :                              was produced. */
    1087          116 :             }
    1088              :           else
    1089              :             {
    1090        72472 :               TokenPtr maybe_raw_string_ptr = maybe_parse_raw_string (loc);
    1091        72472 :               if (maybe_raw_string_ptr != nullptr)
    1092           73 :                 return maybe_raw_string_ptr;
    1093        72472 :             }
    1094              :         }
    1095              : 
    1096              :       // find identifiers and keywords.
    1097      2831696 :       if (is_identifier_start (current_char.value))
    1098      2145390 :         return parse_identifier_or_keyword (loc);
    1099              : 
    1100              :       // int and float literals
    1101       686306 :       if (ISDIGIT (current_char.value))
    1102              :         { //  _ not allowed as first char
    1103       431372 :           if (current_char == '0'
    1104       431372 :               && is_non_decimal_int_literal_separator (peek_input ().value))
    1105              :             {
    1106              :               // handle binary, octal, hex literals
    1107        35180 :               TokenPtr non_dec_int_lit_ptr
    1108        35180 :                 = parse_non_decimal_int_literals (loc);
    1109        35180 :               if (non_dec_int_lit_ptr != nullptr)
    1110        35180 :                 return non_dec_int_lit_ptr;
    1111        35180 :             }
    1112              :           else
    1113              :             {
    1114              :               // handle decimals (integer or float)
    1115       396192 :               TokenPtr decimal_or_float_ptr = parse_decimal_int_or_float (loc);
    1116       396192 :               if (decimal_or_float_ptr != nullptr)
    1117       396192 :                 return decimal_or_float_ptr;
    1118       396192 :             }
    1119              :         }
    1120              : 
    1121              :       // string literals
    1122       254934 :       if (current_char == '"')
    1123       143305 :         return parse_string (loc);
    1124              : 
    1125              :       // char literals and lifetime names
    1126       111629 :       if (current_char == '\'')
    1127              :         {
    1128       111629 :           TokenPtr char_or_lifetime_ptr = parse_char_or_lifetime (loc);
    1129       111629 :           if (char_or_lifetime_ptr != nullptr)
    1130       111629 :             return char_or_lifetime_ptr;
    1131       111629 :         }
    1132              : 
    1133              :       // DEBUG: check for specific character problems:
    1134            0 :       if (current_char == '0')
    1135            0 :         rust_debug ("'0' uncaught before unexpected character");
    1136            0 :       else if (current_char == ']')
    1137            0 :         rust_debug ("']' uncaught before unexpected character");
    1138              :       else if (current_char == 0x5d)
    1139              :         rust_debug ("whatever 0x5d is (not '0' or ']') uncaught before "
    1140              :                     "unexpected character");
    1141              : 
    1142              :       // didn't match anything so error
    1143            0 :       rust_error_at (loc, "unexpected character %<%x%>", current_char.value);
    1144            0 :       current_column++;
    1145              :     }
    1146              : }
    1147              : 
    1148              : // Parses in a suffix
    1149              : std::pair<std::string, int>
    1150       348913 : Lexer::parse_in_suffix ()
    1151              : {
    1152       348913 :   std::string suffix;
    1153              : 
    1154       348913 :   int additional_length_offset = 0;
    1155              : 
    1156              :   // get suffix
    1157       710911 :   while (ISALPHA (current_char.value) || ISDIGIT (current_char.value)
    1158       718780 :          || current_char == '_')
    1159              :     {
    1160        20953 :       additional_length_offset++;
    1161              : 
    1162        20953 :       suffix += current_char;
    1163        20953 :       skip_input ();
    1164        20953 :       current_char = peek_input ();
    1165              :     }
    1166              : 
    1167       348913 :   return std::make_pair (std::move (suffix), additional_length_offset);
    1168       348913 : }
    1169              : 
    1170              : // Parses in the exponent part (if any) of a float literal.
    1171              : std::pair<std::string, int>
    1172        25821 : Lexer::parse_in_exponent_part ()
    1173              : {
    1174        25821 :   int additional_length_offset = 0;
    1175        25821 :   std::string str;
    1176        25821 :   if (current_char == 'E' || current_char == 'e')
    1177              :     {
    1178              :       // add exponent to string as strtod works with it
    1179          672 :       str += current_char;
    1180          672 :       skip_input ();
    1181          672 :       current_char = peek_input ();
    1182              : 
    1183          672 :       additional_length_offset++;
    1184              : 
    1185              :       // special - and + handling
    1186          672 :       if (current_char == '-' || current_char == '+')
    1187              :         {
    1188          152 :           str += current_char;
    1189              : 
    1190          152 :           skip_input ();
    1191          152 :           current_char = peek_input ();
    1192              : 
    1193          152 :           additional_length_offset++;
    1194              :         }
    1195              : 
    1196              :       // parse another decimal number for exponent
    1197          672 :       auto str_length = parse_in_decimal ();
    1198          672 :       str += std::get<0> (str_length);
    1199          672 :       additional_length_offset += std::get<1> (str_length);
    1200          672 :     }
    1201        51642 :   return std::make_pair (str, additional_length_offset);
    1202        25821 : }
    1203              : 
    1204              : // Parses a decimal integer.
    1205              : std::tuple<std::string, int, bool>
    1206       422229 : Lexer::parse_in_decimal ()
    1207              : {
    1208              :   /* A pure decimal contains only digits.  */
    1209       422229 :   bool pure_decimal = true;
    1210       422229 :   int additional_length_offset = 0;
    1211       422229 :   std::string str;
    1212       660799 :   while (ISDIGIT (current_char.value) || current_char.value == '_')
    1213              :     {
    1214       238570 :       if (current_char == '_')
    1215              :         {
    1216         1630 :           pure_decimal = false;
    1217              :         }
    1218       238570 :       additional_length_offset++;
    1219              : 
    1220       238570 :       str += current_char;
    1221       238570 :       skip_input ();
    1222       238570 :       current_char = peek_input ();
    1223              :     }
    1224       844458 :   return std::make_tuple (str, additional_length_offset, pure_decimal);
    1225       422229 : }
    1226              : 
    1227              : /* Parses escapes (and string continues) in "byte" strings and characters. Does
    1228              :  * not support unicode. */
    1229              : std::tuple<char, int, bool>
    1230          286 : Lexer::parse_escape (char opening_char)
    1231              : {
    1232          286 :   int additional_length_offset = 0;
    1233          286 :   char output_char = 0;
    1234              : 
    1235              :   // skip to actual letter
    1236          286 :   skip_input ();
    1237          286 :   current_char = peek_input ();
    1238          286 :   additional_length_offset++;
    1239              : 
    1240          286 :   switch (current_char.value)
    1241              :     {
    1242           57 :     case 'x':
    1243           57 :       {
    1244           57 :         auto hex_escape_pair = parse_partial_hex_escape ();
    1245           57 :         long hexLong = hex_escape_pair.first;
    1246           57 :         additional_length_offset += hex_escape_pair.second;
    1247              : 
    1248           57 :         if (hexLong > 255 || hexLong < 0)
    1249            0 :           rust_error_at (
    1250              :             get_current_location (),
    1251              :             "byte \\x escape %<\\x%x%> out of range - allows up to %<\\xFF%>",
    1252              :             static_cast<unsigned int> (hexLong));
    1253              :         /* TODO: restore capital for escape output - gcc pretty-printer doesn't
    1254              :          * support %X directly */
    1255           57 :         char hexChar = static_cast<char> (hexLong);
    1256              : 
    1257           57 :         output_char = hexChar;
    1258              :       }
    1259           57 :       break;
    1260              :     case 'n':
    1261              :       output_char = '\n';
    1262              :       break;
    1263           24 :     case 'r':
    1264           24 :       output_char = '\r';
    1265           24 :       break;
    1266           17 :     case 't':
    1267           17 :       output_char = '\t';
    1268           17 :       break;
    1269           80 :     case '\\':
    1270           80 :       output_char = '\\';
    1271           80 :       break;
    1272           18 :     case '0':
    1273           18 :       output_char = '\0';
    1274           18 :       break;
    1275           31 :     case '\'':
    1276           31 :       output_char = '\'';
    1277           31 :       break;
    1278            1 :     case '"':
    1279            1 :       output_char = '"';
    1280            1 :       break;
    1281            2 :     case 'u':
    1282            3 :       rust_error_at (get_current_location (),
    1283              :                      "cannot have a unicode escape \\u in a byte %s",
    1284              :                      opening_char == '\'' ? "character" : "string");
    1285              :       // Try to parse it anyway, just to skip it
    1286            2 :       parse_partial_unicode_escape ();
    1287            2 :       return std::make_tuple (output_char, additional_length_offset, false);
    1288           32 :     case '\r':
    1289           32 :     case '\n':
    1290              :       // string continue
    1291           32 :       return std::make_tuple (0, parse_partial_string_continue (), true);
    1292            1 :     default:
    1293            1 :       rust_error_at (get_current_location (),
    1294              :                      "unknown escape sequence %<\\%s%>",
    1295            1 :                      current_char.as_string ().c_str ());
    1296              :       // returns false if no parsing could be done
    1297              :       // return false;
    1298            1 :       return std::make_tuple (output_char, additional_length_offset, false);
    1299          251 :       break;
    1300              :     }
    1301              :   // all non-special cases (string continue) should skip their used char
    1302          251 :   skip_input ();
    1303          251 :   current_char = peek_input ();
    1304          251 :   additional_length_offset++;
    1305              : 
    1306              :   // returns true if parsing was successful
    1307              :   // return true;
    1308          251 :   return std::make_tuple (output_char, additional_length_offset, false);
    1309              : }
    1310              : 
    1311              : /* Parses an escape (or string continue) in a string or character. Supports
    1312              :  * unicode escapes. */
    1313              : std::tuple<Codepoint, int, bool>
    1314        96324 : Lexer::parse_utf8_escape ()
    1315              : {
    1316        96324 :   Codepoint output_char;
    1317        96324 :   int additional_length_offset = 0;
    1318              : 
    1319              :   // skip to actual letter
    1320        96324 :   skip_input ();
    1321        96324 :   current_char = peek_input ();
    1322        96324 :   additional_length_offset++;
    1323              : 
    1324        96324 :   switch (current_char.value)
    1325              :     {
    1326          121 :     case 'x':
    1327          121 :       {
    1328          121 :         auto hex_escape_pair = parse_partial_hex_escape ();
    1329          121 :         long hexLong = hex_escape_pair.first;
    1330          121 :         additional_length_offset += hex_escape_pair.second;
    1331              : 
    1332          121 :         if (hexLong > 127 || hexLong < 0)
    1333            4 :           rust_error_at (
    1334              :             get_current_location (),
    1335              :             "ascii \\x escape %<\\x%x%> out of range - allows up to %<\\x7F%>",
    1336              :             static_cast<unsigned int> (hexLong));
    1337              :         /* TODO: restore capital for escape output - gcc pretty-printer doesn't
    1338              :          * support %X directly */
    1339          121 :         char hexChar = static_cast<char> (hexLong);
    1340              : 
    1341          121 :         output_char = hexChar;
    1342              :       }
    1343          121 :       break;
    1344              :     case 'n':
    1345              :       output_char = '\n';
    1346              :       break;
    1347           24 :     case 'r':
    1348           24 :       output_char = '\r';
    1349           24 :       break;
    1350           90 :     case 't':
    1351           90 :       output_char = '\t';
    1352           90 :       break;
    1353           57 :     case '\\':
    1354           57 :       output_char = '\\';
    1355           57 :       break;
    1356         1572 :     case '0':
    1357         1572 :       output_char = '\0';
    1358         1572 :       break;
    1359           33 :     case '\'':
    1360           33 :       output_char = '\'';
    1361           33 :       break;
    1362          465 :     case '"':
    1363          465 :       output_char = '"';
    1364          465 :       break;
    1365        91096 :     case 'u':
    1366        91096 :       {
    1367        91096 :         auto unicode_escape_pair = parse_partial_unicode_escape ();
    1368        91096 :         output_char = unicode_escape_pair.first;
    1369        91096 :         additional_length_offset += unicode_escape_pair.second;
    1370              : 
    1371        91096 :         return std::make_tuple (output_char, additional_length_offset, false);
    1372              :       }
    1373         1237 :       break;
    1374         1237 :     case '\r':
    1375         1237 :     case '\n':
    1376              :       // string continue
    1377         1237 :       return std::make_tuple (0, parse_partial_string_continue (), true);
    1378            1 :     default:
    1379            1 :       rust_error_at (get_current_location (),
    1380              :                      "unknown escape sequence %<\\%s%>",
    1381            1 :                      current_char.as_string ().c_str ());
    1382              :       // returns false if no parsing could be done
    1383              :       // return false;
    1384            1 :       return std::make_tuple (output_char, additional_length_offset, false);
    1385         3990 :       break;
    1386              :     }
    1387              :   /* all non-special cases (unicode, string continue) should skip their used
    1388              :    * char */
    1389         3990 :   skip_input ();
    1390         3990 :   current_char = peek_input ();
    1391         3990 :   additional_length_offset++;
    1392              : 
    1393              :   // returns true if parsing was successful
    1394              :   // return true;
    1395         3990 :   return std::make_tuple (output_char, additional_length_offset, false);
    1396              : }
    1397              : 
    1398              : // Parses the body of a string continue that has been found in an escape.
    1399              : int
    1400         1269 : Lexer::parse_partial_string_continue ()
    1401              : {
    1402         1269 :   int additional_length_offset = 1;
    1403              : 
    1404              :   // string continue
    1405              :   // TODO use utf-8 codepoint to skip whitespaces
    1406        29208 :   while (is_whitespace (current_char.value))
    1407              :     {
    1408        27939 :       if (current_char == '\n')
    1409              :         {
    1410         1269 :           current_line++;
    1411         1269 :           current_column = 1;
    1412              :           // tell line_table that new line starts
    1413         1269 :           start_line (current_line, max_column_hint);
    1414              : 
    1415              :           // reset "length"
    1416         1269 :           additional_length_offset = 1;
    1417              : 
    1418              :           // get next char
    1419         1269 :           skip_input ();
    1420         1269 :           current_char = peek_input ();
    1421              : 
    1422         1269 :           continue;
    1423              :         }
    1424              : 
    1425        26670 :       skip_input ();
    1426        26670 :       current_char = peek_input ();
    1427        26670 :       additional_length_offset++;
    1428              :     }
    1429              : 
    1430         1269 :   return additional_length_offset;
    1431              : }
    1432              : 
    1433              : /* Parses the body of a '\x' escape. Note that it does not check that the number
    1434              :  * is valid and smaller than 255. */
    1435              : std::pair<long, int>
    1436          178 : Lexer::parse_partial_hex_escape ()
    1437              : {
    1438              :   // hex char string (null-terminated)
    1439          178 :   char hexNum[3] = {0, 0, 0};
    1440              : 
    1441              :   // first hex char
    1442          178 :   current_char = peek_input (1);
    1443          178 :   int additional_length_offset = 1;
    1444              : 
    1445          178 :   if (!is_x_digit (current_char.value))
    1446              :     {
    1447            4 :       rust_error_at (get_current_location (),
    1448              :                      "invalid character %<\\x%s%> in \\x sequence",
    1449            4 :                      current_char.as_string ().c_str ());
    1450            4 :       return std::make_pair (0, 0);
    1451              :     }
    1452          174 :   hexNum[0] = current_char.value;
    1453              : 
    1454              :   // second hex char
    1455          174 :   skip_input ();
    1456          174 :   current_char = peek_input (1);
    1457          174 :   additional_length_offset++;
    1458              : 
    1459          174 :   if (!is_x_digit (current_char.value))
    1460              :     {
    1461            2 :       rust_error_at (get_current_location (),
    1462            2 :                      "invalid character %<\\x%c%s%> in \\x sequence", hexNum[0],
    1463            2 :                      current_char.as_string ().c_str ());
    1464            2 :       return std::make_pair (0, 1);
    1465              :     }
    1466          172 :   skip_input ();
    1467          172 :   hexNum[1] = current_char.value;
    1468              : 
    1469          172 :   long hexLong = std::strtol (hexNum, nullptr, 16);
    1470              : 
    1471          172 :   return std::make_pair (hexLong, additional_length_offset);
    1472              : }
    1473              : 
    1474              : // Parses the body of a unicode escape.
    1475              : std::pair<Codepoint, int>
    1476        91098 : Lexer::parse_partial_unicode_escape ()
    1477              : {
    1478        91098 :   skip_input ();
    1479        91098 :   current_char = peek_input ();
    1480        91098 :   int additional_length_offset = 0;
    1481              : 
    1482        91098 :   if (current_char != '{')
    1483              :     {
    1484            2 :       rust_error_at (get_current_location (),
    1485              :                      "unicode escape should start with %<{%>");
    1486              :       /* Skip what should probaby have been between brackets.  */
    1487           10 :       while (is_x_digit (current_char.value) || current_char == '_')
    1488              :         {
    1489            6 :           skip_input ();
    1490            6 :           current_char = peek_input ();
    1491            6 :           additional_length_offset++;
    1492              :         }
    1493            2 :       return std::make_pair (Codepoint (0), additional_length_offset);
    1494              :     }
    1495              : 
    1496        91096 :   skip_input ();
    1497        91096 :   current_char = peek_input ();
    1498        91096 :   additional_length_offset++;
    1499              : 
    1500        91096 :   if (current_char == '_')
    1501              :     {
    1502            2 :       rust_error_at (get_current_location (),
    1503              :                      "unicode escape cannot start with %<_%>");
    1504            2 :       skip_input ();
    1505            2 :       current_char = peek_input ();
    1506            2 :       additional_length_offset++;
    1507              :       // fallthrough and try to parse the rest anyway
    1508              :     }
    1509              : 
    1510              :   // parse unicode escape - 1-6 hex digits
    1511        91096 :   std::string num_str;
    1512        91096 :   num_str.reserve (6);
    1513              : 
    1514              :   // loop through to add entire hex number to string
    1515       401652 :   while (is_x_digit (current_char.value) || current_char.value == '_')
    1516              :     {
    1517       219460 :       if (current_char == '_')
    1518              :         {
    1519              :           // don't add _ to number
    1520           24 :           skip_input ();
    1521           24 :           current_char = peek_input ();
    1522              : 
    1523           24 :           additional_length_offset++;
    1524              : 
    1525           24 :           continue;
    1526              :         }
    1527              : 
    1528       219436 :       additional_length_offset++;
    1529              : 
    1530              :       // add raw hex numbers
    1531       219436 :       num_str += current_char;
    1532              : 
    1533       219436 :       skip_input ();
    1534       219436 :       current_char = peek_input ();
    1535              :     }
    1536              : 
    1537        91096 :   if (current_char == '}')
    1538              :     {
    1539        91094 :       skip_input ();
    1540        91094 :       current_char = peek_input ();
    1541        91094 :       additional_length_offset++;
    1542              :     }
    1543              :   else
    1544              :     {
    1545              :       // actually an error, but allow propagation anyway Assume that
    1546              :       // wrong bracketm whitespace or single/double quotes are wrong
    1547              :       // termination, otherwise it is a wrong character, then skip to the actual
    1548              :       // terminator.
    1549              :       // TODO use utf-8 codepoint to skip whitespaces
    1550            2 :       if (current_char == '{' || is_whitespace (current_char.value)
    1551            4 :           || current_char == '\'' || current_char == '"')
    1552              :         {
    1553            0 :           rust_error_at (get_current_location (),
    1554              :                          "expected terminating %<}%> in unicode escape");
    1555            0 :           return std::make_pair (Codepoint (0), additional_length_offset);
    1556              :         }
    1557              :       else
    1558              :         {
    1559            2 :           rust_error_at (get_current_location (),
    1560              :                          "invalid character %qs in unicode escape",
    1561            2 :                          current_char.as_string ().c_str ());
    1562              :           // TODO use utf-8 codepoint to skip whitespaces
    1563            8 :           while (current_char != '}' && current_char != '{'
    1564            6 :                  && !is_whitespace (current_char.value) && current_char != '\''
    1565           14 :                  && current_char != '"')
    1566              :             {
    1567            6 :               skip_input ();
    1568            6 :               current_char = peek_input ();
    1569            6 :               additional_length_offset++;
    1570              :             }
    1571              :           // Consume the actual closing bracket if found
    1572            2 :           if (current_char == '}')
    1573              :             {
    1574            2 :               skip_input ();
    1575            2 :               current_char = peek_input ();
    1576            2 :               additional_length_offset++;
    1577              :             }
    1578            2 :           return std::make_pair (Codepoint (0), additional_length_offset);
    1579              :         }
    1580              :     }
    1581              : 
    1582              :   // ensure 1-6 hex characters
    1583        91094 :   if (num_str.length () > 6 || num_str.length () < 1)
    1584              :     {
    1585            4 :       rust_error_at (get_current_location (),
    1586              :                      "unicode escape should be between 1 and 6 hex "
    1587              :                      "characters; it is %lu",
    1588            4 :                      (unsigned long) num_str.length ());
    1589              :       // return false;
    1590            4 :       return std::make_pair (Codepoint (0), additional_length_offset);
    1591              :     }
    1592              : 
    1593        91090 :   unsigned long hex_num = std::strtoul (num_str.c_str (), nullptr, 16);
    1594              : 
    1595        91090 :   if (hex_num > 0xd7ff && hex_num < 0xe000)
    1596              :     {
    1597            4 :       rust_error_at (
    1598              :         get_current_location (),
    1599              :         "unicode escape cannot be a surrogate value (D800 to DFFF)");
    1600            4 :       return std::make_pair (Codepoint (0), additional_length_offset);
    1601              :     }
    1602              : 
    1603        91086 :   if (hex_num > 0x10ffff)
    1604              :     {
    1605            4 :       rust_error_at (get_current_location (),
    1606              :                      "unicode escape cannot be larger than 10FFFF");
    1607            4 :       return std::make_pair (Codepoint (0), additional_length_offset);
    1608              :     }
    1609              : 
    1610              :   // return true;
    1611        91082 :   return std::make_pair (Codepoint (static_cast<uint32_t> (hex_num)),
    1612              :                          additional_length_offset);
    1613        91096 : }
    1614              : 
    1615              : // Parses a byte character.
    1616              : TokenPtr
    1617         1086 : Lexer::parse_byte_char (location_t loc)
    1618              : {
    1619         1086 :   skip_input ();
    1620         1086 :   current_column++;
    1621              :   // make current char the next character
    1622         1086 :   current_char = peek_input ();
    1623              : 
    1624         1086 :   int length = 1;
    1625              : 
    1626              :   // char to save
    1627         1086 :   Codepoint byte_char = 0;
    1628              : 
    1629              :   // detect escapes
    1630         1086 :   if (current_char == '\\')
    1631              :     {
    1632          222 :       auto escape_length_pair = parse_escape ('\'');
    1633          222 :       byte_char = std::get<0> (escape_length_pair);
    1634          222 :       length += std::get<1> (escape_length_pair);
    1635              : 
    1636          222 :       current_char = peek_input ();
    1637              : 
    1638          222 :       if (current_char != '\'')
    1639              :         {
    1640            0 :           rust_error_at (get_current_location (), "unclosed %<byte char%>");
    1641              :         }
    1642              : 
    1643          222 :       skip_input ();
    1644          222 :       current_char = peek_input ();
    1645          222 :       length++; // go to next char
    1646              :     }
    1647          864 :   else if (current_char != '\'')
    1648              :     {
    1649              :       // otherwise, get character from direct input character
    1650          864 :       byte_char = current_char;
    1651              : 
    1652          864 :       if (!byte_char.is_ascii ())
    1653              :         {
    1654            2 :           rust_error_at (get_current_location (),
    1655              :                          "non-ASCII character in %<byte char%>");
    1656              :         }
    1657              : 
    1658          864 :       skip_input ();
    1659          864 :       current_char = peek_input ();
    1660          864 :       length++;
    1661              : 
    1662          864 :       if (current_char != '\'')
    1663              :         {
    1664            0 :           rust_error_at (get_current_location (), "unclosed %<byte char%>");
    1665              :         }
    1666              : 
    1667          864 :       skip_input ();
    1668          864 :       current_char = peek_input ();
    1669          864 :       length++; // go to next char
    1670              :     }
    1671              :   else
    1672              :     {
    1673            0 :       rust_error_at (get_current_location (),
    1674              :                      "no character inside %<%> for %<byte char%>");
    1675              :     }
    1676              : 
    1677         1086 :   current_column += length;
    1678              : 
    1679         1086 :   loc += length - 1;
    1680         1086 :   return Token::make_byte_char (loc, byte_char.value);
    1681              : }
    1682              : 
    1683              : // Parses a byte string.
    1684              : TokenPtr
    1685          520 : Lexer::parse_byte_string (location_t loc)
    1686              : {
    1687              :   // byte string
    1688              : 
    1689              :   // skip quote character
    1690          520 :   skip_input ();
    1691          520 :   current_column++;
    1692              : 
    1693          520 :   std::string str;
    1694          520 :   str.reserve (16); // some sensible default
    1695              : 
    1696          520 :   current_char = peek_input ();
    1697              : 
    1698          520 :   const location_t string_begin_locus = get_current_location ();
    1699              : 
    1700         5086 :   while (current_char != '"' && !current_char.is_eof ())
    1701              :     {
    1702         4046 :       if (current_char == '\\')
    1703              :         {
    1704           63 :           int length = 1;
    1705           63 :           auto escape_length_pair = parse_escape ('"');
    1706           63 :           char output_char = std::get<0> (escape_length_pair);
    1707              : 
    1708           63 :           if (output_char == 0 && std::get<2> (escape_length_pair))
    1709           32 :             length = std::get<1> (escape_length_pair) - 1;
    1710              :           else
    1711           31 :             length += std::get<1> (escape_length_pair);
    1712              : 
    1713           63 :           if (output_char != 0 || !std::get<2> (escape_length_pair))
    1714           31 :             str += output_char;
    1715              : 
    1716           63 :           current_column += length;
    1717              : 
    1718           63 :           continue;
    1719           63 :         }
    1720              : 
    1721         3983 :       current_column++;
    1722         3983 :       if (current_char.value == '\n')
    1723              :         {
    1724           23 :           current_line++;
    1725           23 :           current_column = 1;
    1726              :           // tell line_table that new line starts
    1727           23 :           start_line (current_line, max_column_hint);
    1728              :         }
    1729              : 
    1730         3983 :       str += current_char;
    1731         3983 :       skip_input ();
    1732         3983 :       current_char = peek_input ();
    1733              :     }
    1734              : 
    1735          520 :   if (current_char == '"')
    1736              :     {
    1737          513 :       current_column++;
    1738              : 
    1739          513 :       skip_input ();
    1740          513 :       current_char = peek_input ();
    1741              :     }
    1742            7 :   else if (current_char.is_eof ())
    1743              :     {
    1744            7 :       rust_error_at (string_begin_locus, "unended byte string literal");
    1745            7 :       return Token::make (END_OF_FILE, get_current_location ());
    1746              :     }
    1747              :   else
    1748              :     {
    1749              :       rust_unreachable ();
    1750              :     }
    1751              : 
    1752          513 :   str.shrink_to_fit ();
    1753          513 :   loc += str.size () - 1;
    1754              : 
    1755          513 :   return Token::make_byte_string (loc, std::move (str));
    1756          520 : }
    1757              : 
    1758              : // Parses a C-style string.
    1759              : TokenPtr
    1760           15 : Lexer::parse_c_string (location_t loc)
    1761              : {
    1762           15 :   skip_input ();
    1763           15 :   current_column++;
    1764              : 
    1765              :   // Mostly same code copied from parse_string...
    1766              : 
    1767           15 :   std::string str;
    1768           15 :   str.reserve (16); // some sensible default
    1769              : 
    1770           15 :   current_char = peek_input ();
    1771              : 
    1772           15 :   const location_t string_begin_locus = get_current_location ();
    1773              : 
    1774          106 :   while (current_char.value != '"' && !current_char.is_eof ())
    1775              :     {
    1776           76 :       if (current_char.value == '\\')
    1777              :         {
    1778            1 :           int length = 1;
    1779              : 
    1780            1 :           auto escape_pair = parse_escape ('"');
    1781            1 :           current_char = std::get<0> (escape_pair);
    1782              : 
    1783            1 :           if (current_char == Codepoint (0) && std::get<2> (escape_pair))
    1784            0 :             length = std::get<1> (escape_pair) - 1;
    1785              :           else
    1786            1 :             length += std::get<1> (escape_pair);
    1787              : 
    1788            1 :           if (current_char != Codepoint (0) || !std::get<2> (escape_pair))
    1789            2 :             str += current_char.as_string ();
    1790              : 
    1791            1 :           current_column += length;
    1792              : 
    1793              :           // FIXME: parse_escape does not update current_char correctly.
    1794            1 :           current_char = peek_input ();
    1795            1 :           continue;
    1796            1 :         }
    1797              : 
    1798           75 :       current_column++;
    1799           75 :       if (current_char.value == '\n')
    1800              :         {
    1801            0 :           current_line++;
    1802            0 :           current_column = 1;
    1803              :           // tell line_table that new line starts
    1804            0 :           start_line (current_line, max_column_hint);
    1805              :         }
    1806              : 
    1807           75 :       str += current_char;
    1808           75 :       skip_input ();
    1809           75 :       current_char = peek_input ();
    1810              :     }
    1811              : 
    1812           15 :   if (current_char.value == '"')
    1813              :     {
    1814           15 :       current_column++;
    1815              : 
    1816           15 :       skip_input ();
    1817           15 :       current_char = peek_input ();
    1818              :     }
    1819            0 :   else if (current_char.is_eof ())
    1820              :     {
    1821            0 :       rust_error_at (string_begin_locus, "unended C string literal");
    1822            0 :       return Token::make (END_OF_FILE, get_current_location ());
    1823              :     }
    1824              :   else
    1825              :     {
    1826              :       rust_unreachable ();
    1827              :     }
    1828              : 
    1829           15 :   str.shrink_to_fit ();
    1830              : 
    1831           15 :   return Token::make_c_string (loc, std::move (str));
    1832           15 : }
    1833              : 
    1834              : // Parses a raw byte string.
    1835              : TokenPtr
    1836           32 : Lexer::parse_raw_byte_string (location_t loc)
    1837              : {
    1838              :   // raw byte string literals
    1839           32 :   std::string str;
    1840           32 :   str.reserve (16); // some sensible default
    1841              : 
    1842           32 :   int length = 1;
    1843           32 :   int hash_count = 0;
    1844              : 
    1845           32 :   const location_t string_begin_locus = get_current_location ();
    1846              : 
    1847              :   // get hash count at beginnning
    1848           32 :   skip_input ();
    1849           32 :   current_char = peek_input ();
    1850           32 :   length++;
    1851           32 :   current_column++;
    1852           54 :   while (current_char == '#')
    1853              :     {
    1854           22 :       hash_count++;
    1855           22 :       length++;
    1856           22 :       current_column++;
    1857              : 
    1858           22 :       skip_input ();
    1859           22 :       current_char = peek_input ();
    1860              :     }
    1861              : 
    1862           32 :   if (current_char != '"')
    1863              :     {
    1864            0 :       rust_error_at (get_current_location (),
    1865              :                      "raw byte string has no opening %<\"%>");
    1866              :     }
    1867              : 
    1868           32 :   skip_input ();
    1869           32 :   current_char = peek_input ();
    1870           32 :   length++;
    1871           32 :   current_column++;
    1872              : 
    1873          330 :   while (true)
    1874              :     {
    1875          181 :       if (current_char == '"')
    1876              :         {
    1877           51 :           bool enough_hashes = true;
    1878              : 
    1879           51 :           for (int i = 0; i < hash_count; i++)
    1880              :             {
    1881           26 :               if (peek_input (i + 1) != '#')
    1882              :                 {
    1883              :                   enough_hashes = false;
    1884              :                   break;
    1885              :                 }
    1886              :             }
    1887              : 
    1888           35 :           if (enough_hashes)
    1889              :             {
    1890              :               // skip enough input and peek enough input
    1891           25 :               skip_input (hash_count);
    1892           25 :               current_char = peek_input ();
    1893           25 :               length += hash_count + 1;
    1894           25 :               current_column += hash_count + 1;
    1895           25 :               break;
    1896              :             }
    1897              :         }
    1898          146 :       else if (current_char.is_eof ())
    1899              :         {
    1900            7 :           rust_error_at (string_begin_locus, "unended raw byte string literal");
    1901            7 :           return Token::make (END_OF_FILE, get_current_location ());
    1902              :         }
    1903          139 :       else if (current_char.value > 127)
    1904              :         {
    1905            1 :           rust_error_at (get_current_location (),
    1906              :                          "character %qs in raw byte string out of range",
    1907            1 :                          current_char.as_string ().c_str ());
    1908            1 :           current_char = 0;
    1909              :         }
    1910              : 
    1911          149 :       length++;
    1912          149 :       current_column++;
    1913          149 :       if (current_char == '\n')
    1914              :         {
    1915           22 :           current_line++;
    1916           22 :           current_column = 1;
    1917           22 :           start_line (current_line, max_column_hint);
    1918              :         }
    1919              : 
    1920          149 :       str += current_char;
    1921          149 :       skip_input ();
    1922          149 :       current_char = peek_input ();
    1923          149 :     }
    1924              : 
    1925           25 :   loc += length - 1;
    1926              : 
    1927           25 :   str.shrink_to_fit ();
    1928              : 
    1929           25 :   return Token::make_byte_string (loc, std::move (str));
    1930           32 : }
    1931              : 
    1932              : // Parses a raw identifier.
    1933              : TokenPtr
    1934          116 : Lexer::parse_raw_identifier (location_t loc)
    1935              : {
    1936              :   // raw identifier
    1937          116 :   std::string str;
    1938          116 :   str.reserve (16); // default
    1939              : 
    1940          116 :   skip_input ();
    1941          116 :   current_char = peek_input ();
    1942              : 
    1943          116 :   current_column += 2;
    1944              : 
    1945          116 :   bool first_is_underscore = current_char == '_';
    1946              : 
    1947          116 :   int length = 0;
    1948          116 :   current_char = peek_input ();
    1949              :   // loop through entire name
    1950          650 :   while (is_identifier_continue (current_char.value))
    1951              :     {
    1952          418 :       length++;
    1953              : 
    1954          418 :       str += current_char;
    1955          418 :       skip_input ();
    1956          418 :       current_char = peek_input ();
    1957              :     }
    1958              : 
    1959          116 :   current_column += length;
    1960              : 
    1961          116 :   rust_debug ("raw ident: %s", str.c_str ());
    1962              : 
    1963              :   // if just a single underscore, not an identifier
    1964          116 :   if (first_is_underscore && length == 1)
    1965            1 :     rust_error_at (get_current_location (),
    1966              :                    "%<_%> is not a valid raw identifier");
    1967              : 
    1968          116 :   using namespace Rust::Values;
    1969          116 :   std::set<std::string> invalid{
    1970          116 :     Keywords::CRATE, Keywords::EXTERN_KW,  Keywords::SELF,
    1971          116 :     Keywords::SUPER, Keywords::SELF_ALIAS,
    1972          696 :   };
    1973              : 
    1974          116 :   if (invalid.find (str) != invalid.end ())
    1975              :     {
    1976            1 :       rust_error_at (get_current_location (),
    1977              :                      "%qs is a forbidden raw identifier", str.c_str ());
    1978              : 
    1979            1 :       return nullptr;
    1980              :     }
    1981              :   else
    1982              :     {
    1983          115 :       str.shrink_to_fit ();
    1984          115 :       loc += length - 1;
    1985              : 
    1986          115 :       return Token::make_identifier (loc, std::move (str));
    1987              :     }
    1988          116 : }
    1989              : 
    1990              : // skip broken string input (unterminated strings)
    1991              : void
    1992            0 : Lexer::skip_broken_string_input (Codepoint current_char)
    1993              : {
    1994            0 :   while (current_char != '"' && !current_char.is_eof ())
    1995              :     {
    1996            0 :       if (current_char == '\n')
    1997              :         {
    1998            0 :           current_line++;
    1999            0 :           current_column = 1;
    2000              :         }
    2001              :       else
    2002              :         {
    2003            0 :           current_column++;
    2004              :         }
    2005            0 :       skip_input ();
    2006            0 :       current_char = peek_input ();
    2007              :     }
    2008            0 :   if (current_char == '"')
    2009              :     {
    2010            0 :       current_column++;
    2011              : 
    2012            0 :       skip_input ();
    2013            0 :       current_char = peek_input ();
    2014              :     }
    2015            0 :   rust_debug ("skipped to %d:%d due to bad quotes", current_line,
    2016              :               current_column);
    2017            0 : }
    2018              : 
    2019              : // Parses a string.
    2020              : TokenPtr
    2021       143305 : Lexer::parse_string (location_t loc)
    2022              : {
    2023       143305 :   std::string str;
    2024       143305 :   str.reserve (16); // some sensible default
    2025              : 
    2026       143305 :   current_char = peek_input ();
    2027              : 
    2028       143305 :   const location_t string_begin_locus = get_current_location ();
    2029              : 
    2030              :   // FIXME: This fails if the input ends. How do we check for EOF?
    2031      2102144 :   while (current_char.value != '"' && !current_char.is_eof ())
    2032              :     {
    2033      1815534 :       if (current_char.value == '\\')
    2034              :         {
    2035         4923 :           int length = 1;
    2036              : 
    2037              :           // parse escape
    2038         4923 :           auto utf8_escape_pair = parse_utf8_escape ();
    2039         4923 :           current_char = std::get<0> (utf8_escape_pair);
    2040              : 
    2041         4923 :           if (current_char == Codepoint (0) && std::get<2> (utf8_escape_pair))
    2042         1237 :             length = std::get<1> (utf8_escape_pair) - 1;
    2043              :           else
    2044         3686 :             length += std::get<1> (utf8_escape_pair);
    2045              : 
    2046         4923 :           if (current_char != Codepoint (0) || !std::get<2> (utf8_escape_pair))
    2047         7372 :             str += current_char.as_string ();
    2048              : 
    2049         4923 :           current_column += length;
    2050              : 
    2051              :           // FIXME: should remove this but can't.
    2052              :           // `parse_utf8_escape` does not update `current_char` correctly.
    2053         4923 :           current_char = peek_input ();
    2054         4923 :           continue;
    2055         4923 :         }
    2056              : 
    2057      1810611 :       current_column++;
    2058      1810611 :       if (current_char.value == '\n')
    2059              :         {
    2060        21731 :           current_line++;
    2061        21731 :           current_column = 1;
    2062              :           // tell line_table that new line starts
    2063        21731 :           start_line (current_line, max_column_hint);
    2064              :         }
    2065              : 
    2066      1810611 :       str += current_char;
    2067      1810611 :       skip_input ();
    2068      1810611 :       current_char = peek_input ();
    2069              :     }
    2070              : 
    2071       143305 :   if (current_char.value == '"')
    2072              :     {
    2073       143291 :       current_column++;
    2074              : 
    2075       143291 :       skip_input ();
    2076       143291 :       current_char = peek_input ();
    2077              :     }
    2078           14 :   else if (current_char.is_eof ())
    2079              :     {
    2080           14 :       rust_error_at (string_begin_locus, "unended string literal");
    2081           14 :       return Token::make (END_OF_FILE, get_current_location ());
    2082              :     }
    2083              :   else
    2084              :     {
    2085              :       rust_unreachable ();
    2086              :     }
    2087              : 
    2088       143291 :   str.shrink_to_fit ();
    2089              : 
    2090       143291 :   return Token::make_string (loc, std::move (str));
    2091       143305 : }
    2092              : 
    2093              : // Parses an identifier or keyword.
    2094              : TokenPtr
    2095      2145390 : Lexer::parse_identifier_or_keyword (location_t loc)
    2096              : {
    2097      2145390 :   std::string str;
    2098      2145390 :   str.reserve (16); // default
    2099      4290780 :   str += current_char.as_string ();
    2100              : 
    2101      2145390 :   bool first_is_underscore = current_char == '_';
    2102              : 
    2103      2145390 :   int length = 1;
    2104      2145390 :   current_char = peek_input ();
    2105              : 
    2106              :   // loop through entire name
    2107     13561031 :   while (is_identifier_continue (current_char.value))
    2108              :     {
    2109      9270251 :       auto s = current_char.as_string ();
    2110      9270251 :       length++;
    2111              : 
    2112     18540502 :       str += current_char.as_string ();
    2113      9270251 :       skip_input ();
    2114      9270251 :       current_char = peek_input ();
    2115      9270251 :     }
    2116              : 
    2117      2145390 :   current_column += length;
    2118              : 
    2119              :   // if just a single underscore, not an identifier
    2120      2145390 :   if (first_is_underscore && length == 1)
    2121         7216 :     return Token::make (UNDERSCORE, loc);
    2122              : 
    2123      2138174 :   str.shrink_to_fit ();
    2124              : 
    2125      2138174 :   loc += length - 1;
    2126              : 
    2127      2138174 :   TokenId keyword = classify_keyword (str);
    2128      2138174 :   if (keyword == IDENTIFIER)
    2129      1616004 :     return Token::make_identifier (loc, std::move (str));
    2130              :   else
    2131       522170 :     return Token::make (keyword, loc);
    2132      2145390 : }
    2133              : 
    2134              : // Possibly returns a raw string token if it exists - otherwise returns null.
    2135              : TokenPtr
    2136        72472 : Lexer::maybe_parse_raw_string (location_t loc)
    2137              : {
    2138        72472 :   int peek_index = 0;
    2139        72529 :   while (peek_input (peek_index) == '#')
    2140           57 :     peek_index++;
    2141              : 
    2142        72472 :   if (peek_input (peek_index) == '"')
    2143           73 :     return parse_raw_string (loc, peek_index);
    2144              :   else
    2145        72399 :     return nullptr;
    2146              : }
    2147              : 
    2148              : // Returns a raw string token.
    2149              : TokenPtr
    2150           73 : Lexer::parse_raw_string (location_t loc, int initial_hash_count)
    2151              : {
    2152              :   // raw string literals
    2153           73 :   std::string str;
    2154           73 :   str.reserve (16); // some sensible default
    2155              : 
    2156           73 :   int length = 1 + initial_hash_count;
    2157           73 :   current_column += length;
    2158              : 
    2159           73 :   const location_t string_begin_locus = get_current_location ();
    2160              : 
    2161           73 :   if (initial_hash_count > 0)
    2162           55 :     skip_input (initial_hash_count - 1);
    2163              : 
    2164           73 :   current_char = peek_input ();
    2165              : 
    2166           73 :   if (current_char != '"')
    2167            0 :     rust_error_at (get_current_location (), "raw string has no opening %<\"%>");
    2168              : 
    2169           73 :   length++;
    2170           73 :   current_column++;
    2171           73 :   skip_input ();
    2172           73 :   current_char = peek_input ();
    2173              : 
    2174        18741 :   while (true)
    2175              :     {
    2176         9407 :       if (current_char.value == '"')
    2177              :         {
    2178          134 :           bool enough_hashes = true;
    2179              : 
    2180          134 :           for (int i = 0; i < initial_hash_count; i++)
    2181              :             {
    2182           61 :               if (peek_input (i + 1) != '#')
    2183              :                 {
    2184              :                   enough_hashes = false;
    2185              :                   break;
    2186              :                 }
    2187              :             }
    2188              : 
    2189           76 :           if (enough_hashes)
    2190              :             {
    2191              :               // skip enough input and peek enough input
    2192           73 :               skip_input (initial_hash_count);
    2193           73 :               current_char = peek_input ();
    2194           73 :               length += initial_hash_count + 1;
    2195           73 :               current_column += initial_hash_count + 1;
    2196           73 :               break;
    2197              :             }
    2198              :         }
    2199         9331 :       else if (current_char.is_eof ())
    2200              :         {
    2201            0 :           rust_error_at (string_begin_locus, "unended raw string literal");
    2202            0 :           return Token::make (END_OF_FILE, get_current_location ());
    2203              :         }
    2204              : 
    2205         9334 :       length++;
    2206         9334 :       current_column++;
    2207         9334 :       if (current_char == '\n')
    2208              :         {
    2209          241 :           current_line++;
    2210          241 :           current_column = 1;
    2211          241 :           start_line (current_line, max_column_hint);
    2212              :         }
    2213              : 
    2214        18668 :       str += current_char.as_string ();
    2215         9334 :       skip_input ();
    2216         9334 :       current_char = peek_input ();
    2217         9334 :     }
    2218              : 
    2219           73 :   loc += length - 1;
    2220              : 
    2221           73 :   str.shrink_to_fit ();
    2222              : 
    2223           73 :   return Token::make_raw_string (loc, std::move (str));
    2224           73 : }
    2225              : 
    2226              : template <typename IsDigitFunc>
    2227              : TokenPtr
    2228        35180 : Lexer::parse_non_decimal_int_literal (location_t loc, IsDigitFunc is_digit_func,
    2229              :                                       IntegerLiteralBase base)
    2230              : {
    2231        35180 :   std::string raw_str = "0";
    2232        35180 :   raw_str += current_char; // x, o, b
    2233        35180 :   skip_input ();
    2234              : 
    2235        35180 :   int length = 2;
    2236        35180 :   bool has_valid_digit = false;
    2237              : 
    2238        35180 :   current_char = peek_input ();
    2239              : 
    2240              :   // loop through to add entire number to string
    2241       290955 :   while (true)
    2242              :     {
    2243       326135 :       if (is_digit_func (current_char.value))
    2244              :         {
    2245              :           has_valid_digit = true;
    2246              :         }
    2247        41468 :       else if (current_char != '_')
    2248              :         {
    2249              :           break;
    2250              :         }
    2251       290955 :       length++;
    2252              : 
    2253       290955 :       raw_str += current_char;
    2254       290955 :       skip_input ();
    2255       290955 :       current_char = peek_input ();
    2256              :     }
    2257              : 
    2258        35180 :   int suffix_start = raw_str.length ();
    2259              : 
    2260              :   // parse in suffix if it exists
    2261        35180 :   auto suffix_pair = parse_in_suffix ();
    2262        35180 :   PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
    2263        35180 :   raw_str += suffix_pair.first;
    2264        35180 :   length += suffix_pair.second;
    2265              : 
    2266        35180 :   current_column += length;
    2267              : 
    2268        35180 :   if (!has_valid_digit)
    2269              :     {
    2270            5 :       rust_error_at (loc, ErrorCode::E0768, "no valid digits found for number");
    2271              :     }
    2272              : 
    2273        35180 :   loc += length - 1;
    2274              : 
    2275        35180 :   return Token::make_int (loc, std::move (raw_str), suffix_start, base,
    2276        35180 :                           type_hint);
    2277        35180 : }
    2278              : 
    2279              : // Parses a hex, binary or octal int literal.
    2280              : TokenPtr
    2281        35180 : Lexer::parse_non_decimal_int_literals (location_t loc)
    2282              : {
    2283        35180 :   current_char = peek_input ();
    2284              : 
    2285        35180 :   if (current_char == 'x')
    2286              :     {
    2287              :       // hex (integer only)
    2288        24593 :       return parse_non_decimal_int_literal (loc, is_x_digit,
    2289        24593 :                                             IntegerLiteralBase::Hex);
    2290              :     }
    2291        10587 :   else if (current_char == 'o')
    2292              :     {
    2293              :       // octal (integer only)
    2294           19 :       return parse_non_decimal_int_literal (loc, is_octal_digit,
    2295           19 :                                             IntegerLiteralBase::Octal);
    2296              :     }
    2297        10568 :   else if (current_char == 'b')
    2298              :     {
    2299              :       // binary (integer only)
    2300        10568 :       return parse_non_decimal_int_literal (loc, is_bin_digit,
    2301        10568 :                                             IntegerLiteralBase::Binary);
    2302              :     }
    2303              :   else
    2304              :     {
    2305            0 :       return nullptr;
    2306              :     }
    2307              : }
    2308              : 
    2309              : // Parses a decimal-based int literal or float literal.
    2310              : TokenPtr
    2311       396192 : Lexer::parse_decimal_int_or_float (location_t loc)
    2312              : {
    2313       396192 :   std::string str;
    2314       396192 :   str.reserve (16); // some sensible default
    2315       396192 :   str += current_char;
    2316              : 
    2317       396192 :   int length = 1;
    2318       396192 :   bool first_zero = current_char == '0';
    2319              : 
    2320       396192 :   current_char = peek_input ();
    2321              : 
    2322              :   // parse initial decimal integer (or first integer part of float) literal
    2323       396192 :   auto initial_decimal = parse_in_decimal ();
    2324       396192 :   str += std::get<0> (initial_decimal);
    2325       396192 :   length += std::get<1> (initial_decimal);
    2326              : 
    2327              :   // detect float literal
    2328              :   //
    2329              :   // Note:
    2330              :   //
    2331              :   // We should not use is_float_digit () for this verification but instead
    2332              :   // directly ISDIGIT because rust does not support non digit values right after
    2333              :   // a dot.
    2334              :   // The following value is not legal in rust:
    2335              :   // let a = 3.e1;
    2336              :   // A `0` should be put between the dot and the exponent to be valid
    2337              :   // (eg. 3.0e1).
    2338       396192 :   if (current_char == '.' && ISDIGIT (peek_input (1).value))
    2339              :     {
    2340              :       // float with a '.', parse another decimal into it
    2341              : 
    2342              :       // add . to str
    2343        25365 :       str += current_char;
    2344        25365 :       skip_input ();
    2345        25365 :       current_char = peek_input ();
    2346        25365 :       length++;
    2347              : 
    2348              :       // parse another decimal number for float
    2349        25365 :       auto second_decimal = parse_in_decimal ();
    2350        25365 :       str += std::get<0> (second_decimal);
    2351        25365 :       length += std::get<1> (second_decimal);
    2352              : 
    2353              :       // parse in exponent part if it exists
    2354        25365 :       auto exponent_pair = parse_in_exponent_part ();
    2355        25365 :       str += exponent_pair.first;
    2356        25365 :       length += exponent_pair.second;
    2357              : 
    2358        25365 :       int suffix_start = str.length ();
    2359              : 
    2360              :       // parse in type suffix if it exists
    2361        25365 :       auto suffix_pair = parse_in_suffix ();
    2362        25365 :       PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
    2363        25365 :       str += suffix_pair.first;
    2364        25365 :       length += suffix_pair.second;
    2365              : 
    2366        25365 :       current_column += length;
    2367              : 
    2368        25365 :       loc += length - 1;
    2369              : 
    2370        25365 :       str.shrink_to_fit ();
    2371        25365 :       return Token::make_float (loc, std::move (str), suffix_start, type_hint);
    2372        25365 :     }
    2373       370827 :   else if (current_char == '.'
    2374       370827 :            && check_valid_float_dot_end (peek_input (1).value))
    2375              :     {
    2376              :       // float that is just an integer with a terminating '.' character
    2377              : 
    2378              :       // add . to str
    2379        82459 :       str += current_char;
    2380        82459 :       skip_input ();
    2381        82459 :       current_char = peek_input ();
    2382        82459 :       length++;
    2383              : 
    2384              :       // type hint not allowed
    2385              : 
    2386        82459 :       current_column += length;
    2387              : 
    2388        82459 :       loc += length - 1;
    2389              : 
    2390        82459 :       str.shrink_to_fit ();
    2391        82459 :       auto suffix_start = str.length ();
    2392        82459 :       return Token::make_float (loc, std::move (str), suffix_start,
    2393        82459 :                                 CORETYPE_UNKNOWN);
    2394              :     }
    2395       288368 :   else if (current_char == 'E' || current_char == 'e')
    2396              :     {
    2397              :       // exponent float with no '.' character
    2398              : 
    2399              :       // parse exponent part
    2400          456 :       auto exponent_pair = parse_in_exponent_part ();
    2401          456 :       str += exponent_pair.first;
    2402          456 :       length += exponent_pair.second;
    2403              : 
    2404          456 :       int suffix_start = str.length ();
    2405              : 
    2406              :       // parse in type suffix if it exists
    2407          456 :       auto suffix_pair = parse_in_suffix ();
    2408          456 :       PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
    2409          456 :       str += suffix_pair.first;
    2410          456 :       length += suffix_pair.second;
    2411              : 
    2412          456 :       current_column += length;
    2413              : 
    2414          456 :       loc += length - 1;
    2415              : 
    2416          456 :       str.shrink_to_fit ();
    2417          456 :       return Token::make_float (loc, std::move (str), suffix_start, type_hint);
    2418          456 :     }
    2419              :   else
    2420              :     {
    2421              :       // is an integer
    2422              : 
    2423       287912 :       int suffix_start = str.length ();
    2424              : 
    2425              :       // parse in type suffix if it exists
    2426       287912 :       auto suffix_pair = parse_in_suffix ();
    2427       287912 :       str += suffix_pair.first;
    2428              : 
    2429       287912 :       PrimitiveCoreType type_hint = CORETYPE_UNKNOWN;
    2430              : 
    2431              :       /* A "real" pure decimal doesn't have a suffix and no zero prefix.  */
    2432       287912 :       bool pure_decimal = std::get<2> (initial_decimal);
    2433       287184 :       if (pure_decimal && (!first_zero || suffix_start == 1)
    2434       575094 :           && suffix_pair.first.empty ())
    2435              :         type_hint = CORETYPE_PURE_DECIMAL;
    2436              : 
    2437       287912 :       length += suffix_pair.second;
    2438              : 
    2439       287912 :       current_column += length;
    2440              : 
    2441       287912 :       loc += length - 1;
    2442              : 
    2443       287912 :       str.shrink_to_fit ();
    2444       287912 :       return Token::make_int (loc, std::move (str), suffix_start,
    2445       287912 :                               IntegerLiteralBase::Decimal, type_hint);
    2446       287912 :     }
    2447       396192 : }
    2448              : 
    2449              : TokenPtr
    2450       111629 : Lexer::parse_char_or_lifetime (location_t loc)
    2451              : {
    2452       111629 :   int length = 1;
    2453              : 
    2454       111629 :   current_char = peek_input ();
    2455       111629 :   if (current_char.is_eof ())
    2456            0 :     return nullptr;
    2457              : 
    2458              :   // parse escaped char literal
    2459       111629 :   if (current_char.value == '\\')
    2460              :     {
    2461              :       // parse escape
    2462        91401 :       auto utf8_escape_pair = parse_utf8_escape ();
    2463        91401 :       Codepoint escaped_char = std::get<0> (utf8_escape_pair);
    2464        91401 :       length += std::get<1> (utf8_escape_pair);
    2465              : 
    2466        91401 :       if (peek_input ().value != '\'')
    2467              :         {
    2468            0 :           rust_error_at (get_current_location (), "unended character literal");
    2469              :         }
    2470              :       else
    2471              :         {
    2472        91401 :           skip_input ();
    2473        91401 :           current_char = peek_input ();
    2474        91401 :           length++;
    2475              :         }
    2476              : 
    2477        91401 :       current_column += length;
    2478              : 
    2479        91401 :       loc += length - 1;
    2480              : 
    2481        91401 :       return Token::make_char (loc, escaped_char);
    2482              :     }
    2483              :   else
    2484              :     {
    2485        20228 :       skip_input ();
    2486              : 
    2487        20228 :       if (peek_input ().value == '\'')
    2488              :         {
    2489              :           // parse non-escaped char literal
    2490         1885 :           Codepoint non_escaped_char = current_char;
    2491              : 
    2492              :           // skip the ' character
    2493         1885 :           skip_input ();
    2494         1885 :           current_char = peek_input ();
    2495              : 
    2496              :           // TODO fix due to different widths of utf-8 chars?
    2497         1885 :           current_column += 3;
    2498              : 
    2499         1885 :           loc += 2;
    2500              : 
    2501         1885 :           return Token::make_char (loc, non_escaped_char);
    2502              :         }
    2503        18343 :       else if (is_identifier_start (current_char.value))
    2504              :         {
    2505              :           // parse lifetime name
    2506        18343 :           std::string str;
    2507        36686 :           str += current_char.as_string ();
    2508        18343 :           length++;
    2509              : 
    2510        18343 :           current_char = peek_input ();
    2511        39802 :           while (is_identifier_continue (current_char.value))
    2512              :             {
    2513         6232 :               str += current_char.as_string ();
    2514         3116 :               skip_input ();
    2515         3116 :               current_char = peek_input ();
    2516         3116 :               length++;
    2517              :             }
    2518              : 
    2519        18343 :           current_column += length;
    2520              : 
    2521        18343 :           loc += length - 1;
    2522              : 
    2523              :           // TODO some keywords cannot be used for a lifetime label #2306
    2524              :           // https://doc.rust-lang.org/reference/tokens.html
    2525              : 
    2526        18343 :           str.shrink_to_fit ();
    2527        18343 :           return Token::make_lifetime (loc, std::move (str));
    2528        18343 :         }
    2529              :       else
    2530              :         {
    2531            0 :           rust_error_at (
    2532              :             get_current_location (),
    2533              :             "expected %' after character constant in character literal");
    2534            0 :           return nullptr;
    2535              :         }
    2536              :     }
    2537              : }
    2538              : 
    2539              : void
    2540         2086 : Lexer::split_current_token (TokenId new_left, TokenId new_right)
    2541              : {
    2542              :   /* TODO: assert that this TokenId is a "simple token" like punctuation and not
    2543              :    * like "IDENTIFIER"? */
    2544         2086 :   location_t current_loc = peek_token ()->get_locus ();
    2545         2086 :   TokenPtr new_left_tok = Token::make (new_left, current_loc);
    2546         2086 :   TokenPtr new_right_tok = Token::make (new_right, current_loc + 1);
    2547              : 
    2548         2086 :   token_queue.replace_current_value (std::move (new_left_tok));
    2549         2086 :   token_queue.insert (1, std::move (new_right_tok));
    2550         2086 : }
    2551              : 
    2552              : void
    2553            2 : Lexer::split_current_token (std::vector<TokenPtr> new_tokens)
    2554              : {
    2555            2 :   rust_assert (new_tokens.size () > 0);
    2556            4 :   token_queue.replace_current_value (new_tokens[0]);
    2557              : 
    2558            5 :   for (size_t i = 1; i < new_tokens.size (); i++)
    2559              :     {
    2560            6 :       token_queue.insert (i, new_tokens[i]);
    2561              :     }
    2562            2 : }
    2563              : 
    2564              : void
    2565      1375541 : Lexer::start_line (int current_line, int current_column)
    2566              : {
    2567      1375541 :   if (line_map)
    2568      1375541 :     linemap_line_start (line_table, current_line, current_column);
    2569      1375541 : }
    2570              : 
    2571              : } // namespace Rust
    2572              : 
    2573              : #if CHECKING_P
    2574              : 
    2575              : namespace selftest {
    2576              : 
    2577              : // Checks if `src` has the same contents as the given characters
    2578              : static void
    2579            6 : assert_source_content (Rust::InputSource &src,
    2580              :                        const std::vector<uint32_t> &expected)
    2581              : {
    2582            6 :   Rust::Codepoint src_char = src.next ();
    2583           41 :   for (auto expected_char : expected)
    2584              :     {
    2585              :       // Make sure that `src` is not shorter than `expected`
    2586           35 :       ASSERT_FALSE (src_char.is_eof ());
    2587              :       // Checks skipped character is expeceted one.
    2588           35 :       ASSERT_EQ (src_char.value, expected_char);
    2589           35 :       src_char = src.next ();
    2590              :     }
    2591              :   // Checks if `src` and `chars` has the same length.
    2592            6 :   ASSERT_TRUE (src_char.is_eof ());
    2593            6 : }
    2594              : 
    2595              : static void
    2596            4 : test_buffer_input_source (std::string str,
    2597              :                           const std::vector<uint32_t> &expected)
    2598              : {
    2599            4 :   Rust::BufferInputSource source (str, 0);
    2600            4 :   assert_source_content (source, expected);
    2601            4 : }
    2602              : 
    2603              : static void
    2604            2 : test_file_input_source (std::string str, const std::vector<uint32_t> &expected)
    2605              : {
    2606            2 :   FILE *tmpf = tmpfile ();
    2607              :   // Moves to the first character
    2608            2 :   fputs (str.c_str (), tmpf);
    2609            2 :   std::rewind (tmpf);
    2610            2 :   Rust::FileInputSource source (tmpf);
    2611            2 :   assert_source_content (source, expected);
    2612            2 : }
    2613              : 
    2614              : void
    2615            1 : rust_input_source_test ()
    2616              : {
    2617              :   // ASCII
    2618            1 :   std::string src = (const char *) u8"_abcde\tXYZ\v\f";
    2619            1 :   std::vector<uint32_t> expected = {u'_',  u'a', u'b', u'c', u'd',  u'e',
    2620            1 :                                     u'\t', u'X', u'Y', u'Z', u'\v', u'\f'};
    2621            2 :   test_buffer_input_source (src, expected);
    2622              : 
    2623              :   // BOM
    2624            1 :   src = (const char *) u8"\xef\xbb\xbfOK";
    2625            1 :   expected = {u'O', u'K'};
    2626            2 :   test_buffer_input_source (src, expected);
    2627              : 
    2628              :   // Russian
    2629            1 :   src = (const char *) u8"приве́т";
    2630            1 :   expected = {u'п',
    2631              :               u'р',
    2632              :               u'и',
    2633              :               u'в',
    2634              :               0x0435 /* CYRILLIC SMALL LETTER IE е */,
    2635              :               0x301 /* COMBINING ACUTE ACCENT ́ */,
    2636            1 :               u'т'};
    2637            2 :   test_buffer_input_source (src, expected);
    2638              : 
    2639            1 :   src = (const char *) u8"❤️🦀";
    2640            1 :   expected = {0x2764 /* HEAVY BLACK HEART */,
    2641            1 :               0xfe0f /* VARIATION SELECTOR-16 */, U'🦀'};
    2642            2 :   test_buffer_input_source (src, expected);
    2643              : 
    2644            1 :   src = (const char *) u8"こんにちは";
    2645            1 :   expected = {u'こ', u'ん', u'に', u'ち', u'は'};
    2646            2 :   test_file_input_source (src, expected);
    2647              : 
    2648            1 :   src = (const char *) u8"👮‍♂👩‍⚕";
    2649            1 :   expected
    2650              :     = {0x1f46e /* POLICE OFFICER */,   0x200d /* ZERO WIDTH JOINER */,
    2651              :        0x2642 /* MALE SIGN */,         0x1f469 /* WOMAN */,
    2652            1 :        0x200d /* ZERO WIDTH JOINER */, 0x2695 /* STAFF OF AESCULAPIUS */};
    2653            2 :   test_file_input_source (src, expected);
    2654            1 : }
    2655              : 
    2656              : } // namespace selftest
    2657              : 
    2658              : #endif // CHECKING_P
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.