Line data Source code
1 : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
2 :
3 : // This file is part of GCC.
4 :
5 : // GCC is free software; you can redistribute it and/or modify it under
6 : // the terms of the GNU General Public License as published by the Free
7 : // Software Foundation; either version 3, or (at your option) any later
8 : // version.
9 :
10 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 : // for more details.
14 :
15 : // You should have received a copy of the GNU General Public License
16 : // along with GCC; see the file COPYING3. If not see
17 : // <http://www.gnu.org/licenses/>.
18 :
19 : #ifndef RUST_TOKEN_H
20 : #define RUST_TOKEN_H
21 :
22 : #include "rust-system.h"
23 : #include "rust-linemap.h"
24 : #include "rust-unicode.h"
25 : #include "rust-diagnostics.h"
26 :
27 : namespace Rust {
28 :
29 : // used by Rust::Token::make_identifier
30 : class Identifier;
31 :
32 : // "Primitive core types" in Rust - the different int and float types, as well
33 : // as some others
34 : enum PrimitiveCoreType
35 : {
36 : CORETYPE_UNKNOWN,
37 : // named primitives
38 : CORETYPE_BOOL,
39 : CORETYPE_CHAR,
40 : CORETYPE_STR,
41 : // okay technically int and uint are arch-dependent (pointer size)
42 : CORETYPE_INT,
43 : CORETYPE_UINT,
44 : // numbered number primitives
45 : CORETYPE_F32,
46 : CORETYPE_F64,
47 : CORETYPE_I8,
48 : CORETYPE_I16,
49 : CORETYPE_I32,
50 : CORETYPE_I64,
51 : CORETYPE_I128,
52 : CORETYPE_U8,
53 : CORETYPE_U16,
54 : CORETYPE_U32,
55 : CORETYPE_U64,
56 : CORETYPE_U128,
57 : // Pure decimals are used for tuple index.
58 : // Also means there is no type hint.
59 : CORETYPE_PURE_DECIMAL,
60 : // arch-dependent pointer sizes
61 : CORETYPE_ISIZE = CORETYPE_INT,
62 : CORETYPE_USIZE = CORETYPE_UINT
63 : };
64 :
65 : // RS_TOKEN(name, description)
66 : // RS_TOKEN_KEYWORD_{2015,2018}(name, identifier)
67 :
68 : // Keep RS_TOKEN_KEYWORD sorted
69 :
70 : /* note that abstract, async, become, box, do, final, macro, override, priv,
71 : * try, typeof, unsized, virtual, and yield are unused */
72 : #define RS_TOKEN_LIST \
73 : RS_TOKEN (FIRST_TOKEN, "<first-token-marker>") \
74 : RS_TOKEN (END_OF_FILE, "end of file") \
75 : RS_TOKEN (EXCLAM, "!") \
76 : RS_TOKEN (NOT_EQUAL, "!=") \
77 : RS_TOKEN (PERCENT, "%") \
78 : RS_TOKEN (PERCENT_EQ, "%=") \
79 : RS_TOKEN (AMP, "&") \
80 : RS_TOKEN (AMP_EQ, "&=") \
81 : RS_TOKEN (LOGICAL_AND, "&&") \
82 : RS_TOKEN (ASTERISK, "*") \
83 : RS_TOKEN (ASTERISK_EQ, "*=") \
84 : RS_TOKEN (PLUS, "+") \
85 : RS_TOKEN (PLUS_EQ, "+=") \
86 : RS_TOKEN (COMMA, ",") \
87 : RS_TOKEN (MINUS, "-") \
88 : RS_TOKEN (MINUS_EQ, "-=") \
89 : RS_TOKEN (RETURN_TYPE, "->") \
90 : RS_TOKEN (DOT, ".") \
91 : RS_TOKEN (DOT_DOT, "..") \
92 : RS_TOKEN (DOT_DOT_EQ, "..=") \
93 : RS_TOKEN (ELLIPSIS, "...") \
94 : RS_TOKEN (DIV, "/") \
95 : RS_TOKEN (DIV_EQ, "/=") \
96 : RS_TOKEN (COLON, ":") \
97 : RS_TOKEN (SEMICOLON, ";") \
98 : RS_TOKEN (LEFT_SHIFT, "<<") \
99 : RS_TOKEN (LEFT_SHIFT_EQ, "<<=") \
100 : RS_TOKEN (LEFT_ANGLE, "<") \
101 : RS_TOKEN (LESS_OR_EQUAL, "<=") \
102 : RS_TOKEN (EQUAL, "=") \
103 : RS_TOKEN (EQUAL_EQUAL, "==") \
104 : RS_TOKEN (MATCH_ARROW, "=>") \
105 : RS_TOKEN (RIGHT_ANGLE, ">") \
106 : RS_TOKEN (GREATER_OR_EQUAL, ">=") \
107 : RS_TOKEN (RIGHT_SHIFT, ">>") \
108 : RS_TOKEN (RIGHT_SHIFT_EQ, ">>=") \
109 : RS_TOKEN (PATTERN_BIND, "@") \
110 : RS_TOKEN (TILDE, "~") \
111 : RS_TOKEN (BACKSLASH, "\\") \
112 : RS_TOKEN (BACKTICK, "`") \
113 : RS_TOKEN (CARET, "^") \
114 : RS_TOKEN (CARET_EQ, "^=") \
115 : RS_TOKEN (PIPE, "|") \
116 : RS_TOKEN (PIPE_EQ, "|=") \
117 : RS_TOKEN (OR, "||") \
118 : RS_TOKEN (QUESTION_MARK, "?") \
119 : RS_TOKEN (HASH, "#") \
120 : /* from here on, dodgy and may not be correct. not operators and may be \
121 : * symbols */ \
122 : /* RS_TOKEN(SPACE, " ") probably too dodgy */ \
123 : /* RS_TOKEN(NEWLINE, "\n")*/ \
124 : RS_TOKEN (SCOPE_RESOLUTION, "::") /* dodgy */ \
125 : RS_TOKEN (SINGLE_QUOTE, "'") /* should i differentiate from lifetime? */ \
126 : RS_TOKEN (DOUBLE_QUOTE, "\"") \
127 : RS_TOKEN (IDENTIFIER, "identifier") \
128 : RS_TOKEN (INT_LITERAL, \
129 : "integer literal") /* do different int and float types need \
130 : different literal types? */ \
131 : RS_TOKEN (FLOAT_LITERAL, "float literal") \
132 : RS_TOKEN (STRING_LITERAL, "string literal") \
133 : RS_TOKEN (CHAR_LITERAL, "character literal") \
134 : RS_TOKEN (BYTE_STRING_LITERAL, "byte string literal") \
135 : RS_TOKEN (RAW_STRING_LITERAL, "raw string literal") \
136 : RS_TOKEN (BYTE_CHAR_LITERAL, "byte character literal") \
137 : RS_TOKEN (C_STRING_LITERAL, "C string literal") \
138 : RS_TOKEN (RAW_C_STRING_LITERAL, "raw C string literal") \
139 : RS_TOKEN (LIFETIME, "lifetime") /* TODO: improve token type */ \
140 : /* Have "interpolated" tokens (whatever that means)? identifer, path, type, \
141 : * pattern, */ \
142 : /* expression, statement, block, meta, item in mrustc (but not directly in \
143 : * lexer). */ \
144 : RS_TOKEN (LEFT_PAREN, "(") \
145 : RS_TOKEN (RIGHT_PAREN, ")") \
146 : RS_TOKEN (LEFT_CURLY, "{") \
147 : RS_TOKEN (RIGHT_CURLY, "}") \
148 : RS_TOKEN (LEFT_SQUARE, "[") \
149 : RS_TOKEN (RIGHT_SQUARE, "]") \
150 : /* Macros */ \
151 : RS_TOKEN (DOLLAR_SIGN, "$") \
152 : /* Doc Comments */ \
153 : RS_TOKEN (INNER_DOC_COMMENT, "#![doc]") \
154 : RS_TOKEN (OUTER_DOC_COMMENT, "#[doc]") \
155 : RS_TOKEN_KEYWORD_2015 (ABSTRACT, "abstract") /* unused */ \
156 : RS_TOKEN_KEYWORD_2015 (AS, "as") \
157 : RS_TOKEN_KEYWORD_2018 (ASYNC, "async") /* unused */ \
158 : RS_TOKEN_KEYWORD_2015 (AUTO, "auto") \
159 : RS_TOKEN_KEYWORD_2018 (AWAIT, "await") \
160 : RS_TOKEN_KEYWORD_2015 (BECOME, "become") /* unused */ \
161 : RS_TOKEN_KEYWORD_2015 (BOX, "box") /* unused */ \
162 : RS_TOKEN_KEYWORD_2015 (BREAK, "break") \
163 : RS_TOKEN_KEYWORD_2015 (CONST, "const") \
164 : RS_TOKEN_KEYWORD_2015 (CONTINUE, "continue") \
165 : RS_TOKEN_KEYWORD_2015 (CRATE, "crate") \
166 : RS_TOKEN_KEYWORD_2015 (DO, "do") /* unused */ \
167 : RS_TOKEN_KEYWORD_2018 (DYN, "dyn") \
168 : RS_TOKEN_KEYWORD_2015 (ELSE, "else") \
169 : RS_TOKEN_KEYWORD_2015 (ENUM_KW, "enum") \
170 : RS_TOKEN_KEYWORD_2015 (EXTERN_KW, "extern") \
171 : RS_TOKEN_KEYWORD_2015 (FALSE_LITERAL, "false") \
172 : RS_TOKEN_KEYWORD_2015 (FINAL_KW, "final") /* unused */ \
173 : RS_TOKEN_KEYWORD_2015 (FN_KW, "fn") \
174 : RS_TOKEN_KEYWORD_2015 (FOR, "for") \
175 : RS_TOKEN_KEYWORD_2015 (IF, "if") \
176 : RS_TOKEN_KEYWORD_2015 (IMPL, "impl") \
177 : RS_TOKEN_KEYWORD_2015 (IN, "in") \
178 : RS_TOKEN_KEYWORD_2015 (LET, "let") \
179 : RS_TOKEN_KEYWORD_2015 (LOOP, "loop") \
180 : RS_TOKEN_KEYWORD_2015 (MACRO, "macro") \
181 : RS_TOKEN_KEYWORD_2015 (MATCH_KW, "match") \
182 : RS_TOKEN_KEYWORD_2015 (MOD, "mod") \
183 : RS_TOKEN_KEYWORD_2015 (MOVE, "move") \
184 : RS_TOKEN_KEYWORD_2015 (MUT, "mut") \
185 : RS_TOKEN_KEYWORD_2015 (OVERRIDE_KW, "override") /* unused */ \
186 : RS_TOKEN_KEYWORD_2015 (PRIV, "priv") /* unused */ \
187 : RS_TOKEN_KEYWORD_2015 (PUB, "pub") \
188 : RS_TOKEN_KEYWORD_2015 (REF, "ref") \
189 : RS_TOKEN_KEYWORD_2015 (RETURN_KW, "return") \
190 : RS_TOKEN_KEYWORD_2015 ( \
191 : SELF_ALIAS, "Self") /* mrustc does not treat this as a reserved word*/ \
192 : RS_TOKEN_KEYWORD_2015 (SELF, "self") \
193 : RS_TOKEN_KEYWORD_2015 (STATIC_KW, "static") \
194 : RS_TOKEN_KEYWORD_2015 (STRUCT_KW, "struct") \
195 : RS_TOKEN_KEYWORD_2015 (SUPER, "super") \
196 : RS_TOKEN_KEYWORD_2015 (TRAIT, "trait") \
197 : RS_TOKEN_KEYWORD_2015 (TRUE_LITERAL, "true") \
198 : RS_TOKEN_KEYWORD_2015 (TRY, "try") /* unused */ \
199 : RS_TOKEN_KEYWORD_2015 (TYPE, "type") \
200 : RS_TOKEN_KEYWORD_2015 (TYPEOF, "typeof") /* unused */ \
201 : RS_TOKEN_KEYWORD_2015 (UNDERSCORE, "_") \
202 : RS_TOKEN_KEYWORD_2015 (UNSAFE, "unsafe") \
203 : RS_TOKEN_KEYWORD_2015 (UNSIZED, "unsized") /* unused */ \
204 : RS_TOKEN_KEYWORD_2015 (USE, "use") \
205 : RS_TOKEN_KEYWORD_2015 (VIRTUAL, "virtual") /* unused */ \
206 : RS_TOKEN_KEYWORD_2015 (WHERE, "where") \
207 : RS_TOKEN_KEYWORD_2015 (WHILE, "while") \
208 : RS_TOKEN_KEYWORD_2015 (YIELD, "yield") /* unused */ \
209 : RS_TOKEN (LAST_TOKEN, "<last-token-marker>")
210 :
211 : // Contains all token types. Crappy implementation via x-macros.
212 : enum TokenId
213 : {
214 : #define RS_TOKEN(name, _) name,
215 : #define RS_TOKEN_KEYWORD_2015(x, y) RS_TOKEN (x, y)
216 : #define RS_TOKEN_KEYWORD_2018 RS_TOKEN_KEYWORD_2015
217 : RS_TOKEN_LIST
218 : #undef RS_TOKEN_KEYWORD_2015
219 : #undef RS_TOKEN_KEYWORD_2018
220 : #undef RS_TOKEN
221 : };
222 :
223 : // Holds the base information for integer-literal tokens. For other tokens, its
224 : // value is 0.
225 : enum class IntegerLiteralBase : uint8_t
226 : {
227 : None = 0,
228 : Binary,
229 : Octal,
230 : Decimal,
231 : Hex,
232 : };
233 :
234 : // dodgy "TokenPtr" declaration with Token forward declaration
235 : class Token;
236 : // A smart pointer (shared_ptr) to Token.
237 : typedef std::shared_ptr<Token> TokenPtr;
238 : // A smart pointer (shared_ptr) to a constant Token.
239 : typedef std::shared_ptr<const Token> const_TokenPtr;
240 :
241 : // Hackily defined way to get token description for enum value using x-macros
242 : const char *get_token_description (TokenId id);
243 : /* Hackily defined way to get token description as a string for enum value using
244 : * x-macros */
245 : const char *token_id_to_str (TokenId id);
246 : /* checks if a token is a keyword */
247 : bool token_id_is_keyword (TokenId id);
248 : /* gets the string associated with a keyword */
249 : const std::string &token_id_keyword_string (TokenId id);
250 : // Get type hint description as a string.
251 : const char *get_type_hint_string (PrimitiveCoreType type);
252 :
253 : /* Normalize string if a token is a identifier */
254 : std::string nfc_normalize_token_string (location_t loc, TokenId id,
255 : const std::string &str);
256 :
257 : // Represents a single token. Create using factory static methods.
258 : class Token
259 : {
260 : private:
261 : // Token kind.
262 : TokenId token_id;
263 : // Token location.
264 : location_t locus;
265 : // Associated text (if any) of token.
266 : std::string str;
267 : // TODO: maybe remove issues and just store std::string as value?
268 : /* Type hint for token based on lexer data (e.g. type suffix). Does not exist
269 : * for most tokens. */
270 : PrimitiveCoreType type_hint;
271 :
272 : // Suffix start index if it exist. Otherwise has token str's length
273 : uint16_t suffix_start;
274 :
275 : // Base if it is an integer literal. Otherwise has LITERALBASE_NONE
276 : IntegerLiteralBase integer_literal_base;
277 :
278 : // Token constructor from token id and location. Has a null string.
279 594850 : Token (TokenId token_id, location_t location)
280 594850 : : token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN),
281 594850 : suffix_start (0), integer_literal_base (IntegerLiteralBase::None)
282 : {}
283 :
284 : // Token constructor from token id, location, and a string.
285 222387 : Token (TokenId token_id, location_t location, std::string paramStr)
286 222387 : : token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN),
287 222387 : integer_literal_base (IntegerLiteralBase::None)
288 : {
289 : // Normalize identifier tokens
290 222387 : str = nfc_normalize_token_string (location, token_id, std::move (paramStr));
291 222387 : suffix_start = str.length ();
292 222387 : }
293 :
294 : // Token constructor from token id, location, and a char.
295 1838 : Token (TokenId token_id, location_t location, char paramChar)
296 1838 : : token_id (token_id), locus (location), str (1, paramChar),
297 1838 : type_hint (CORETYPE_UNKNOWN), suffix_start (1),
298 1838 : integer_literal_base (IntegerLiteralBase::None)
299 : {
300 : // Do not need to normalize 1byte char
301 : }
302 :
303 : // Token constructor from token id, location, and a "codepoint".
304 275 : Token (TokenId token_id, location_t location, Codepoint paramCodepoint)
305 275 : : token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN),
306 275 : integer_literal_base (IntegerLiteralBase::None)
307 : {
308 : // Normalize identifier tokens
309 550 : str = nfc_normalize_token_string (location, token_id,
310 550 : paramCodepoint.as_string ());
311 275 : suffix_start = str.length ();
312 275 : }
313 :
314 : // Token constructor from token id, location, a string, and type hint.
315 16134 : Token (TokenId token_id, location_t location, std::string paramStr,
316 : PrimitiveCoreType parType)
317 16134 : : token_id (token_id), locus (location), type_hint (parType),
318 16134 : integer_literal_base (IntegerLiteralBase::None)
319 : {
320 : // Normalize identifier tokens
321 16134 : str = nfc_normalize_token_string (location, token_id, std::move (paramStr));
322 16134 : suffix_start = str.length ();
323 16134 : }
324 :
325 : // Token constructor from token id, location, a string, a suffix start index,
326 : // a integer base type and type hint.
327 19390 : Token (TokenId token_id, location_t location, std::string paramStr,
328 : PrimitiveCoreType parType, uint16_t suffix_start,
329 : IntegerLiteralBase base)
330 19390 : : token_id (token_id), locus (location), str (std::move (paramStr)),
331 19390 : type_hint (parType), suffix_start (suffix_start),
332 19390 : integer_literal_base (base)
333 : {
334 : // Do not need to normalize literal str
335 : }
336 :
337 : public:
338 : // No default constructor.
339 : Token () = delete;
340 : // Do not copy/assign tokens.
341 : Token (const Token &) = delete;
342 : Token &operator= (const Token &) = delete;
343 :
344 : // Allow moving tokens.
345 : Token (Token &&other) = default;
346 : Token &operator= (Token &&other) = default;
347 :
348 766434 : ~Token () = default;
349 :
350 : /* TODO: make_shared (which saves a heap allocation) does not work with the
351 : * private constructor */
352 :
353 : // Makes and returns a new TokenPtr (with null string).
354 594850 : static TokenPtr make (TokenId token_id, location_t locus)
355 : {
356 : // return std::make_shared<Token> (token_id, locus);
357 594850 : return TokenPtr (new Token (token_id, locus));
358 : }
359 :
360 : // Makes and returns a new TokenPtr of type IDENTIFIER.
361 212425 : static TokenPtr make_identifier (location_t locus, std::string str)
362 : {
363 : // return std::make_shared<Token> (IDENTIFIER, locus, str);
364 212425 : return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
365 : }
366 :
367 : static TokenPtr make_identifier (const Identifier &ident);
368 :
369 : // Makes and returns a new TokenPtr of type INT_LITERAL.
370 18998 : static TokenPtr make_int (location_t locus, std::string str,
371 : uint16_t suffix_start, IntegerLiteralBase base,
372 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
373 : {
374 : // return std::make_shared<Token> (INT_LITERAL, locus, str, type_hint,
375 : // suffix_start, base);
376 18998 : return TokenPtr (new Token (INT_LITERAL, locus, std::move (str), type_hint,
377 18998 : suffix_start, base));
378 : }
379 :
380 : // Makes and returns a new TokenPtr of type FLOAT_LITERAL.
381 : static TokenPtr
382 392 : make_float (location_t locus, std::string str, uint16_t suffix_start,
383 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN,
384 : IntegerLiteralBase base = IntegerLiteralBase::None)
385 : {
386 : // return std::make_shared<Token> (FLOAT_LITERAL, locus, str, type_hint);
387 392 : return TokenPtr (new Token (FLOAT_LITERAL, locus, std::move (str),
388 392 : type_hint, suffix_start, base));
389 : }
390 :
391 : // Makes and returns a new TokenPtr of type STRING_LITERAL.
392 16134 : static TokenPtr make_string (location_t locus, std::string str)
393 : {
394 : // return std::make_shared<Token> (STRING_LITERAL, locus, str,
395 : // CORETYPE_STR);
396 16134 : return TokenPtr (
397 16134 : new Token (STRING_LITERAL, locus, std::move (str), CORETYPE_STR));
398 : }
399 :
400 : // Makes and returns a new TokenPtr of type CHAR_LITERAL.
401 275 : static TokenPtr make_char (location_t locus, Codepoint char_lit)
402 : {
403 : // return std::make_shared<Token> (CHAR_LITERAL, locus, char_lit);
404 275 : return TokenPtr (new Token (CHAR_LITERAL, locus, char_lit));
405 : }
406 :
407 : // Makes and returns a new TokenPtr of type BYTE_CHAR_LITERAL.
408 1838 : static TokenPtr make_byte_char (location_t locus, char byte_char)
409 : {
410 : // return std::make_shared<Token> (BYTE_CHAR_LITERAL, locus, byte_char);
411 1838 : return TokenPtr (new Token (BYTE_CHAR_LITERAL, locus, byte_char));
412 : }
413 :
414 : // Makes and returns a new TokenPtr of type BYTE_STRING_LITERAL (fix).
415 96 : static TokenPtr make_byte_string (location_t locus, std::string str)
416 : {
417 : // return std::make_shared<Token> (BYTE_STRING_LITERAL, locus, str);
418 96 : return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, std::move (str)));
419 : }
420 :
421 : // Makes and returns a new TokenPtr of type RAW_STRING_LITERAL.
422 25 : static TokenPtr make_raw_string (location_t locus, std::string str)
423 : {
424 25 : return TokenPtr (new Token (RAW_STRING_LITERAL, locus, std::move (str)));
425 : }
426 :
427 22 : static TokenPtr make_c_string (location_t locus, std::string str)
428 : {
429 22 : return TokenPtr (new Token (C_STRING_LITERAL, locus, std::move (str)));
430 : }
431 :
432 : // Makes and returns a new TokenPtr of type INNER_DOC_COMMENT.
433 173 : static TokenPtr make_inner_doc_comment (location_t locus, std::string str)
434 : {
435 173 : return TokenPtr (new Token (INNER_DOC_COMMENT, locus, std::move (str)));
436 : }
437 :
438 : // Makes and returns a new TokenPtr of type OUTER_DOC_COMMENT.
439 7928 : static TokenPtr make_outer_doc_comment (location_t locus, std::string str)
440 : {
441 7928 : return TokenPtr (new Token (OUTER_DOC_COMMENT, locus, std::move (str)));
442 : }
443 :
444 : // Makes and returns a new TokenPtr of type LIFETIME.
445 1718 : static TokenPtr make_lifetime (location_t locus, std::string str)
446 : {
447 : // return std::make_shared<Token> (LIFETIME, locus, str);
448 1718 : return TokenPtr (new Token (LIFETIME, locus, std::move (str)));
449 : }
450 :
451 : // Gets id of the token.
452 3774323 : TokenId get_id () const { return token_id; }
453 :
454 : // Gets location of the token.
455 850745 : location_t get_locus () const { return locus; }
456 :
457 : // Set location of the token.
458 0 : void set_locus (location_t locus) { this->locus = locus; }
459 :
460 : // Gets string description of the token.
461 440447 : const std::string &get_str () const
462 : {
463 440447 : if (token_id_is_keyword (token_id))
464 24 : return token_id_keyword_string (token_id);
465 :
466 440423 : if (!should_have_str ())
467 0 : rust_internal_error_at (
468 0 : locus, "attempting to get string for %qs, which should have no string",
469 : get_token_description ());
470 :
471 440423 : return str;
472 : }
473 :
474 : // Gets token's type hint info.
475 9916 : PrimitiveCoreType get_type_hint () const
476 : {
477 9916 : return type_hint == CORETYPE_PURE_DECIMAL ? CORETYPE_UNKNOWN : type_hint;
478 : }
479 :
480 : // Gets suffix_start of the token
481 44021 : uint16_t get_suffix_start () const { return suffix_start; }
482 :
483 : // Gets literal base of the token
484 22396 : IntegerLiteralBase get_literal_base () const { return integer_literal_base; }
485 :
486 : // diagnostics (error reporting)
487 172019 : const char *get_token_description () const
488 : {
489 171578 : return Rust::get_token_description (token_id);
490 : }
491 :
492 : // debugging
493 55 : const char *token_id_to_str () const
494 : {
495 55 : return Rust::token_id_to_str (token_id);
496 : }
497 :
498 : // debugging
499 : const char *get_type_hint_str () const;
500 :
501 : /* Returns whether the token is a literal of any type (int, float, char,
502 : * string, byte char, byte string). */
503 242650 : bool is_literal () const
504 : {
505 221165 : switch (token_id)
506 : {
507 : case INT_LITERAL:
508 : case FLOAT_LITERAL:
509 : case CHAR_LITERAL:
510 : case STRING_LITERAL:
511 : case BYTE_CHAR_LITERAL:
512 : case BYTE_STRING_LITERAL:
513 : case RAW_STRING_LITERAL:
514 : case C_STRING_LITERAL:
515 : return true;
516 4 : default:
517 4 : return false;
518 : }
519 : }
520 :
521 : // Returns whether the token should have a string.
522 667873 : bool should_have_str () const
523 : {
524 667873 : switch (token_id)
525 : {
526 : case IDENTIFIER:
527 : case LIFETIME:
528 : case INNER_DOC_COMMENT:
529 : case OUTER_DOC_COMMENT:
530 : return true;
531 238843 : default:
532 238843 : return is_literal ();
533 : }
534 : }
535 :
536 : // Returns whether the token is a pure decimal int literal
537 20984 : bool is_pure_decimal () const { return type_hint == CORETYPE_PURE_DECIMAL; }
538 :
539 : // Return the token representation as someone would find it in the original
540 : // source code file.
541 : std::string as_string () const;
542 : };
543 : } // namespace Rust
544 :
545 : namespace std {
546 : template <> struct hash<Rust::PrimitiveCoreType>
547 : {
548 201310 : size_t operator() (const Rust::PrimitiveCoreType &coretype) const noexcept
549 : {
550 201310 : return hash<std::underlying_type<Rust::PrimitiveCoreType>::type> () (
551 : static_cast<std::underlying_type<Rust::PrimitiveCoreType>::type> (
552 : coretype));
553 : }
554 : };
555 : } // namespace std
556 :
557 : #endif
|
