Line data Source code
1 : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
2 :
3 : // This file is part of GCC.
4 :
5 : // GCC is free software; you can redistribute it and/or modify it under
6 : // the terms of the GNU General Public License as published by the Free
7 : // Software Foundation; either version 3, or (at your option) any later
8 : // version.
9 :
10 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 : // for more details.
14 :
15 : // You should have received a copy of the GNU General Public License
16 : // along with GCC; see the file COPYING3. If not see
17 : // <http://www.gnu.org/licenses/>.
18 :
19 : #ifndef RUST_TOKEN_H
20 : #define RUST_TOKEN_H
21 :
22 : #include "rust-system.h"
23 : #include "rust-linemap.h"
24 : #include "rust-unicode.h"
25 : #include "rust-diagnostics.h"
26 :
27 : namespace Rust {
28 :
29 : // used by Rust::Token::make_identifier
30 : class Identifier;
31 :
32 : // "Primitive core types" in Rust - the different int and float types, as well
33 : // as some others
34 : enum PrimitiveCoreType
35 : {
36 : CORETYPE_UNKNOWN,
37 : // named primitives
38 : CORETYPE_BOOL,
39 : CORETYPE_CHAR,
40 : CORETYPE_STR,
41 : // okay technically int and uint are arch-dependent (pointer size)
42 : CORETYPE_INT,
43 : CORETYPE_UINT,
44 : // numbered number primitives
45 : CORETYPE_F32,
46 : CORETYPE_F64,
47 : CORETYPE_I8,
48 : CORETYPE_I16,
49 : CORETYPE_I32,
50 : CORETYPE_I64,
51 : CORETYPE_I128,
52 : CORETYPE_U8,
53 : CORETYPE_U16,
54 : CORETYPE_U32,
55 : CORETYPE_U64,
56 : CORETYPE_U128,
57 : // Pure decimals are used for tuple index.
58 : // Also means there is no type hint.
59 : CORETYPE_PURE_DECIMAL,
60 : // arch-dependent pointer sizes
61 : CORETYPE_ISIZE = CORETYPE_INT,
62 : CORETYPE_USIZE = CORETYPE_UINT
63 : };
64 :
65 : // RS_TOKEN(name, description)
66 : // RS_TOKEN_KEYWORD_{2015,2018}(name, identifier)
67 :
68 : // Keep RS_TOKEN_KEYWORD sorted
69 :
70 : /* note that abstract, async, become, box, do, final, macro, override, priv,
71 : * try, typeof, unsized, virtual, and yield are unused */
72 : #define RS_TOKEN_LIST \
73 : RS_TOKEN (FIRST_TOKEN, "<first-token-marker>") \
74 : RS_TOKEN (END_OF_FILE, "end of file") \
75 : RS_TOKEN (EXCLAM, "!") \
76 : RS_TOKEN (NOT_EQUAL, "!=") \
77 : RS_TOKEN (PERCENT, "%") \
78 : RS_TOKEN (PERCENT_EQ, "%=") \
79 : RS_TOKEN (AMP, "&") \
80 : RS_TOKEN (AMP_EQ, "&=") \
81 : RS_TOKEN (LOGICAL_AND, "&&") \
82 : RS_TOKEN (ASTERISK, "*") \
83 : RS_TOKEN (ASTERISK_EQ, "*=") \
84 : RS_TOKEN (PLUS, "+") \
85 : RS_TOKEN (PLUS_EQ, "+=") \
86 : RS_TOKEN (COMMA, ",") \
87 : RS_TOKEN (MINUS, "-") \
88 : RS_TOKEN (MINUS_EQ, "-=") \
89 : RS_TOKEN (RETURN_TYPE, "->") \
90 : RS_TOKEN (DOT, ".") \
91 : RS_TOKEN (DOT_DOT, "..") \
92 : RS_TOKEN (DOT_DOT_EQ, "..=") \
93 : RS_TOKEN (ELLIPSIS, "...") \
94 : RS_TOKEN (DIV, "/") \
95 : RS_TOKEN (DIV_EQ, "/=") \
96 : RS_TOKEN (COLON, ":") \
97 : RS_TOKEN (SEMICOLON, ";") \
98 : RS_TOKEN (LEFT_SHIFT, "<<") \
99 : RS_TOKEN (LEFT_SHIFT_EQ, "<<=") \
100 : RS_TOKEN (LEFT_ANGLE, "<") \
101 : RS_TOKEN (LESS_OR_EQUAL, "<=") \
102 : RS_TOKEN (EQUAL, "=") \
103 : RS_TOKEN (EQUAL_EQUAL, "==") \
104 : RS_TOKEN (MATCH_ARROW, "=>") \
105 : RS_TOKEN (RIGHT_ANGLE, ">") \
106 : RS_TOKEN (GREATER_OR_EQUAL, ">=") \
107 : RS_TOKEN (RIGHT_SHIFT, ">>") \
108 : RS_TOKEN (RIGHT_SHIFT_EQ, ">>=") \
109 : RS_TOKEN (PATTERN_BIND, "@") \
110 : RS_TOKEN (TILDE, "~") \
111 : RS_TOKEN (BACKSLASH, "\\") \
112 : RS_TOKEN (BACKTICK, "`") \
113 : RS_TOKEN (CARET, "^") \
114 : RS_TOKEN (CARET_EQ, "^=") \
115 : RS_TOKEN (PIPE, "|") \
116 : RS_TOKEN (PIPE_EQ, "|=") \
117 : RS_TOKEN (OR, "||") \
118 : RS_TOKEN (QUESTION_MARK, "?") \
119 : RS_TOKEN (HASH, "#") \
120 : /* from here on, dodgy and may not be correct. not operators and may be \
121 : * symbols */ \
122 : /* RS_TOKEN(SPACE, " ") probably too dodgy */ \
123 : /* RS_TOKEN(NEWLINE, "\n")*/ \
124 : RS_TOKEN (SCOPE_RESOLUTION, "::") /* dodgy */ \
125 : RS_TOKEN (SINGLE_QUOTE, "'") /* should i differentiate from lifetime? */ \
126 : RS_TOKEN (DOUBLE_QUOTE, "\"") \
127 : RS_TOKEN (IDENTIFIER, "identifier") \
128 : RS_TOKEN (INT_LITERAL, \
129 : "integer literal") /* do different int and float types need \
130 : different literal types? */ \
131 : RS_TOKEN (FLOAT_LITERAL, "float literal") \
132 : RS_TOKEN (STRING_LITERAL, "string literal") \
133 : RS_TOKEN (CHAR_LITERAL, "character literal") \
134 : RS_TOKEN (BYTE_STRING_LITERAL, "byte string literal") \
135 : RS_TOKEN (RAW_STRING_LITERAL, "raw string literal") \
136 : RS_TOKEN (BYTE_CHAR_LITERAL, "byte character literal") \
137 : RS_TOKEN (LIFETIME, "lifetime") /* TODO: improve token type */ \
138 : /* Have "interpolated" tokens (whatever that means)? identifer, path, type, \
139 : * pattern, */ \
140 : /* expression, statement, block, meta, item in mrustc (but not directly in \
141 : * lexer). */ \
142 : RS_TOKEN (LEFT_PAREN, "(") \
143 : RS_TOKEN (RIGHT_PAREN, ")") \
144 : RS_TOKEN (LEFT_CURLY, "{") \
145 : RS_TOKEN (RIGHT_CURLY, "}") \
146 : RS_TOKEN (LEFT_SQUARE, "[") \
147 : RS_TOKEN (RIGHT_SQUARE, "]") \
148 : /* Macros */ \
149 : RS_TOKEN (DOLLAR_SIGN, "$") \
150 : /* Doc Comments */ \
151 : RS_TOKEN (INNER_DOC_COMMENT, "#![doc]") \
152 : RS_TOKEN (OUTER_DOC_COMMENT, "#[doc]") \
153 : RS_TOKEN_KEYWORD_2015 (ABSTRACT, "abstract") /* unused */ \
154 : RS_TOKEN_KEYWORD_2015 (AS, "as") \
155 : RS_TOKEN_KEYWORD_2018 (ASYNC, "async") /* unused */ \
156 : RS_TOKEN_KEYWORD_2015 (AUTO, "auto") \
157 : RS_TOKEN_KEYWORD_2018 (AWAIT, "await") \
158 : RS_TOKEN_KEYWORD_2015 (BECOME, "become") /* unused */ \
159 : RS_TOKEN_KEYWORD_2015 (BOX, "box") /* unused */ \
160 : RS_TOKEN_KEYWORD_2015 (BREAK, "break") \
161 : RS_TOKEN_KEYWORD_2015 (CONST, "const") \
162 : RS_TOKEN_KEYWORD_2015 (CONTINUE, "continue") \
163 : RS_TOKEN_KEYWORD_2015 (CRATE, "crate") \
164 : RS_TOKEN_KEYWORD_2015 (DO, "do") /* unused */ \
165 : RS_TOKEN_KEYWORD_2018 (DYN, "dyn") \
166 : RS_TOKEN_KEYWORD_2015 (ELSE, "else") \
167 : RS_TOKEN_KEYWORD_2015 (ENUM_KW, "enum") \
168 : RS_TOKEN_KEYWORD_2015 (EXTERN_KW, "extern") \
169 : RS_TOKEN_KEYWORD_2015 (FALSE_LITERAL, "false") \
170 : RS_TOKEN_KEYWORD_2015 (FINAL_KW, "final") /* unused */ \
171 : RS_TOKEN_KEYWORD_2015 (FN_KW, "fn") \
172 : RS_TOKEN_KEYWORD_2015 (FOR, "for") \
173 : RS_TOKEN_KEYWORD_2015 (IF, "if") \
174 : RS_TOKEN_KEYWORD_2015 (IMPL, "impl") \
175 : RS_TOKEN_KEYWORD_2015 (IN, "in") \
176 : RS_TOKEN_KEYWORD_2015 (LET, "let") \
177 : RS_TOKEN_KEYWORD_2015 (LOOP, "loop") \
178 : RS_TOKEN_KEYWORD_2015 (MACRO, "macro") \
179 : RS_TOKEN_KEYWORD_2015 (MATCH_KW, "match") \
180 : RS_TOKEN_KEYWORD_2015 (MOD, "mod") \
181 : RS_TOKEN_KEYWORD_2015 (MOVE, "move") \
182 : RS_TOKEN_KEYWORD_2015 (MUT, "mut") \
183 : RS_TOKEN_KEYWORD_2015 (OVERRIDE_KW, "override") /* unused */ \
184 : RS_TOKEN_KEYWORD_2015 (PRIV, "priv") /* unused */ \
185 : RS_TOKEN_KEYWORD_2015 (PUB, "pub") \
186 : RS_TOKEN_KEYWORD_2015 (REF, "ref") \
187 : RS_TOKEN_KEYWORD_2015 (RETURN_KW, "return") \
188 : RS_TOKEN_KEYWORD_2015 ( \
189 : SELF_ALIAS, "Self") /* mrustc does not treat this as a reserved word*/ \
190 : RS_TOKEN_KEYWORD_2015 (SELF, "self") \
191 : RS_TOKEN_KEYWORD_2015 (STATIC_KW, "static") \
192 : RS_TOKEN_KEYWORD_2015 (STRUCT_KW, "struct") \
193 : RS_TOKEN_KEYWORD_2015 (SUPER, "super") \
194 : RS_TOKEN_KEYWORD_2015 (TRAIT, "trait") \
195 : RS_TOKEN_KEYWORD_2015 (TRUE_LITERAL, "true") \
196 : RS_TOKEN_KEYWORD_2015 (TRY, "try") /* unused */ \
197 : RS_TOKEN_KEYWORD_2015 (TYPE, "type") \
198 : RS_TOKEN_KEYWORD_2015 (TYPEOF, "typeof") /* unused */ \
199 : RS_TOKEN_KEYWORD_2015 (UNDERSCORE, "_") \
200 : RS_TOKEN_KEYWORD_2015 (UNSAFE, "unsafe") \
201 : RS_TOKEN_KEYWORD_2015 (UNSIZED, "unsized") /* unused */ \
202 : RS_TOKEN_KEYWORD_2015 (USE, "use") \
203 : RS_TOKEN_KEYWORD_2015 (VIRTUAL, "virtual") /* unused */ \
204 : RS_TOKEN_KEYWORD_2015 (WHERE, "where") \
205 : RS_TOKEN_KEYWORD_2015 (WHILE, "while") \
206 : RS_TOKEN_KEYWORD_2015 (YIELD, "yield") /* unused */ \
207 : RS_TOKEN (LAST_TOKEN, "<last-token-marker>")
208 :
209 : // Contains all token types. Crappy implementation via x-macros.
210 : enum TokenId
211 : {
212 : #define RS_TOKEN(name, _) name,
213 : #define RS_TOKEN_KEYWORD_2015(x, y) RS_TOKEN (x, y)
214 : #define RS_TOKEN_KEYWORD_2018 RS_TOKEN_KEYWORD_2015
215 : RS_TOKEN_LIST
216 : #undef RS_TOKEN_KEYWORD_2015
217 : #undef RS_TOKEN_KEYWORD_2018
218 : #undef RS_TOKEN
219 : };
220 :
221 : // Holds the base information for integer-literal tokens. For other tokens, its
222 : // value is 0.
223 : enum class IntegerLiteralBase : uint8_t
224 : {
225 : None = 0,
226 : Binary,
227 : Octal,
228 : Decimal,
229 : Hex,
230 : };
231 :
232 : // dodgy "TokenPtr" declaration with Token forward declaration
233 : class Token;
234 : // A smart pointer (shared_ptr) to Token.
235 : typedef std::shared_ptr<Token> TokenPtr;
236 : // A smart pointer (shared_ptr) to a constant Token.
237 : typedef std::shared_ptr<const Token> const_TokenPtr;
238 :
239 : // Hackily defined way to get token description for enum value using x-macros
240 : const char *get_token_description (TokenId id);
241 : /* Hackily defined way to get token description as a string for enum value using
242 : * x-macros */
243 : const char *token_id_to_str (TokenId id);
244 : /* checks if a token is a keyword */
245 : bool token_id_is_keyword (TokenId id);
246 : /* gets the string associated with a keyword */
247 : const std::string &token_id_keyword_string (TokenId id);
248 : // Get type hint description as a string.
249 : const char *get_type_hint_string (PrimitiveCoreType type);
250 :
251 : /* Normalize string if a token is a identifier */
252 : std::string nfc_normalize_token_string (location_t loc, TokenId id,
253 : const std::string &str);
254 :
255 : // Represents a single token. Create using factory static methods.
256 : class Token
257 : {
258 : private:
259 : // Token kind.
260 : TokenId token_id;
261 : // Token location.
262 : location_t locus;
263 : // Associated text (if any) of token.
264 : std::string str;
265 : // TODO: maybe remove issues and just store std::string as value?
266 : /* Type hint for token based on lexer data (e.g. type suffix). Does not exist
267 : * for most tokens. */
268 : PrimitiveCoreType type_hint;
269 :
270 : // Suffix start index if it exist. Otherwise has token str's length
271 : uint16_t suffix_start;
272 :
273 : // Base if it is an integer literal. Otherwise has LITERALBASE_NONE
274 : IntegerLiteralBase integer_literal_base;
275 :
276 : // Token constructor from token id and location. Has a null string.
277 586308 : Token (TokenId token_id, location_t location)
278 586308 : : token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN),
279 586308 : suffix_start (0), integer_literal_base (IntegerLiteralBase::None)
280 : {}
281 :
282 : // Token constructor from token id, location, and a string.
283 219526 : Token (TokenId token_id, location_t location, std::string paramStr)
284 219526 : : token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN),
285 219526 : integer_literal_base (IntegerLiteralBase::None)
286 : {
287 : // Normalize identifier tokens
288 219526 : str = nfc_normalize_token_string (location, token_id, std::move (paramStr));
289 219526 : suffix_start = str.length ();
290 219526 : }
291 :
292 : // Token constructor from token id, location, and a char.
293 1838 : Token (TokenId token_id, location_t location, char paramChar)
294 1838 : : token_id (token_id), locus (location), str (1, paramChar),
295 1838 : type_hint (CORETYPE_UNKNOWN), suffix_start (1),
296 1838 : integer_literal_base (IntegerLiteralBase::None)
297 : {
298 : // Do not need to normalize 1byte char
299 : }
300 :
301 : // Token constructor from token id, location, and a "codepoint".
302 275 : Token (TokenId token_id, location_t location, Codepoint paramCodepoint)
303 275 : : token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN),
304 275 : integer_literal_base (IntegerLiteralBase::None)
305 : {
306 : // Normalize identifier tokens
307 550 : str = nfc_normalize_token_string (location, token_id,
308 550 : paramCodepoint.as_string ());
309 275 : suffix_start = str.length ();
310 275 : }
311 :
312 : // Token constructor from token id, location, a string, and type hint.
313 15939 : Token (TokenId token_id, location_t location, std::string paramStr,
314 : PrimitiveCoreType parType)
315 15939 : : token_id (token_id), locus (location), type_hint (parType),
316 15939 : integer_literal_base (IntegerLiteralBase::None)
317 : {
318 : // Normalize identifier tokens
319 15939 : str = nfc_normalize_token_string (location, token_id, std::move (paramStr));
320 15939 : suffix_start = str.length ();
321 15939 : }
322 :
323 : // Token constructor from token id, location, a string, a suffix start index,
324 : // a integer base type and type hint.
325 19044 : Token (TokenId token_id, location_t location, std::string paramStr,
326 : PrimitiveCoreType parType, uint16_t suffix_start,
327 : IntegerLiteralBase base)
328 19044 : : token_id (token_id), locus (location), str (std::move (paramStr)),
329 19044 : type_hint (parType), suffix_start (suffix_start),
330 19044 : integer_literal_base (base)
331 : {
332 : // Do not need to normalize literal str
333 : }
334 :
335 : public:
336 : // No default constructor.
337 : Token () = delete;
338 : // Do not copy/assign tokens.
339 : Token (const Token &) = delete;
340 : Token &operator= (const Token &) = delete;
341 :
342 : // Allow moving tokens.
343 : Token (Token &&other) = default;
344 : Token &operator= (Token &&other) = default;
345 :
346 755031 : ~Token () = default;
347 :
348 : /* TODO: make_shared (which saves a heap allocation) does not work with the
349 : * private constructor */
350 :
351 : // Makes and returns a new TokenPtr (with null string).
352 586308 : static TokenPtr make (TokenId token_id, location_t locus)
353 : {
354 : // return std::make_shared<Token> (token_id, locus);
355 586308 : return TokenPtr (new Token (token_id, locus));
356 : }
357 :
358 : // Makes and returns a new TokenPtr of type IDENTIFIER.
359 209584 : static TokenPtr make_identifier (location_t locus, std::string str)
360 : {
361 : // return std::make_shared<Token> (IDENTIFIER, locus, str);
362 209584 : return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
363 : }
364 :
365 : static TokenPtr make_identifier (const Identifier &ident);
366 :
367 : // Makes and returns a new TokenPtr of type INT_LITERAL.
368 18671 : static TokenPtr make_int (location_t locus, std::string str,
369 : uint16_t suffix_start, IntegerLiteralBase base,
370 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
371 : {
372 : // return std::make_shared<Token> (INT_LITERAL, locus, str, type_hint,
373 : // suffix_start, base);
374 18671 : return TokenPtr (new Token (INT_LITERAL, locus, std::move (str), type_hint,
375 18671 : suffix_start, base));
376 : }
377 :
378 : // Makes and returns a new TokenPtr of type FLOAT_LITERAL.
379 : static TokenPtr
380 373 : make_float (location_t locus, std::string str, uint16_t suffix_start,
381 : PrimitiveCoreType type_hint = CORETYPE_UNKNOWN,
382 : IntegerLiteralBase base = IntegerLiteralBase::None)
383 : {
384 : // return std::make_shared<Token> (FLOAT_LITERAL, locus, str, type_hint);
385 373 : return TokenPtr (new Token (FLOAT_LITERAL, locus, std::move (str),
386 373 : type_hint, suffix_start, base));
387 : }
388 :
389 : // Makes and returns a new TokenPtr of type STRING_LITERAL.
390 15939 : static TokenPtr make_string (location_t locus, std::string str)
391 : {
392 : // return std::make_shared<Token> (STRING_LITERAL, locus, str,
393 : // CORETYPE_STR);
394 15939 : return TokenPtr (
395 15939 : new Token (STRING_LITERAL, locus, std::move (str), CORETYPE_STR));
396 : }
397 :
398 : // Makes and returns a new TokenPtr of type CHAR_LITERAL.
399 275 : static TokenPtr make_char (location_t locus, Codepoint char_lit)
400 : {
401 : // return std::make_shared<Token> (CHAR_LITERAL, locus, char_lit);
402 275 : return TokenPtr (new Token (CHAR_LITERAL, locus, char_lit));
403 : }
404 :
405 : // Makes and returns a new TokenPtr of type BYTE_CHAR_LITERAL.
406 1838 : static TokenPtr make_byte_char (location_t locus, char byte_char)
407 : {
408 : // return std::make_shared<Token> (BYTE_CHAR_LITERAL, locus, byte_char);
409 1838 : return TokenPtr (new Token (BYTE_CHAR_LITERAL, locus, byte_char));
410 : }
411 :
412 : // Makes and returns a new TokenPtr of type BYTE_STRING_LITERAL (fix).
413 96 : static TokenPtr make_byte_string (location_t locus, std::string str)
414 : {
415 : // return std::make_shared<Token> (BYTE_STRING_LITERAL, locus, str);
416 96 : return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, std::move (str)));
417 : }
418 :
419 : // Makes and returns a new TokenPtr of type RAW_STRING_LITERAL.
420 25 : static TokenPtr make_raw_string (location_t locus, std::string str)
421 : {
422 25 : return TokenPtr (new Token (RAW_STRING_LITERAL, locus, std::move (str)));
423 : }
424 :
425 : // Makes and returns a new TokenPtr of type INNER_DOC_COMMENT.
426 173 : static TokenPtr make_inner_doc_comment (location_t locus, std::string str)
427 : {
428 173 : return TokenPtr (new Token (INNER_DOC_COMMENT, locus, std::move (str)));
429 : }
430 :
431 : // Makes and returns a new TokenPtr of type OUTER_DOC_COMMENT.
432 7928 : static TokenPtr make_outer_doc_comment (location_t locus, std::string str)
433 : {
434 7928 : return TokenPtr (new Token (OUTER_DOC_COMMENT, locus, std::move (str)));
435 : }
436 :
437 : // Makes and returns a new TokenPtr of type LIFETIME.
438 1720 : static TokenPtr make_lifetime (location_t locus, std::string str)
439 : {
440 : // return std::make_shared<Token> (LIFETIME, locus, str);
441 1720 : return TokenPtr (new Token (LIFETIME, locus, std::move (str)));
442 : }
443 :
444 : // Gets id of the token.
445 3731603 : TokenId get_id () const { return token_id; }
446 :
447 : // Gets location of the token.
448 840757 : location_t get_locus () const { return locus; }
449 :
450 : // Set location of the token.
451 0 : void set_locus (location_t locus) { this->locus = locus; }
452 :
453 : // Gets string description of the token.
454 435963 : const std::string &get_str () const
455 : {
456 435963 : if (token_id_is_keyword (token_id))
457 24 : return token_id_keyword_string (token_id);
458 :
459 435939 : if (!should_have_str ())
460 0 : rust_internal_error_at (
461 0 : locus, "attempting to get string for %qs, which should have no string",
462 : get_token_description ());
463 :
464 435939 : return str;
465 : }
466 :
467 : // Gets token's type hint info.
468 9771 : PrimitiveCoreType get_type_hint () const
469 : {
470 9771 : return type_hint == CORETYPE_PURE_DECIMAL ? CORETYPE_UNKNOWN : type_hint;
471 : }
472 :
473 : // Gets suffix_start of the token
474 43405 : uint16_t get_suffix_start () const { return suffix_start; }
475 :
476 : // Gets literal base of the token
477 22091 : IntegerLiteralBase get_literal_base () const { return integer_literal_base; }
478 :
479 : // diagnostics (error reporting)
480 170888 : const char *get_token_description () const
481 : {
482 170447 : return Rust::get_token_description (token_id);
483 : }
484 :
485 : // debugging
486 55 : const char *token_id_to_str () const
487 : {
488 55 : return Rust::token_id_to_str (token_id);
489 : }
490 :
491 : // debugging
492 : const char *get_type_hint_str () const;
493 :
494 : /* Returns whether the token is a literal of any type (int, float, char,
495 : * string, byte char, byte string). */
496 240664 : bool is_literal () const
497 : {
498 219341 : switch (token_id)
499 : {
500 : case INT_LITERAL:
501 : case FLOAT_LITERAL:
502 : case CHAR_LITERAL:
503 : case STRING_LITERAL:
504 : case BYTE_CHAR_LITERAL:
505 : case BYTE_STRING_LITERAL:
506 : case RAW_STRING_LITERAL:
507 : return true;
508 4 : default:
509 4 : return false;
510 : }
511 : }
512 :
513 : // Returns whether the token should have a string.
514 661997 : bool should_have_str () const
515 : {
516 661997 : switch (token_id)
517 : {
518 : case IDENTIFIER:
519 : case LIFETIME:
520 : case INNER_DOC_COMMENT:
521 : case OUTER_DOC_COMMENT:
522 : return true;
523 236935 : default:
524 236935 : return is_literal ();
525 : }
526 : }
527 :
528 : // Returns whether the token is a pure decimal int literal
529 20697 : bool is_pure_decimal () const { return type_hint == CORETYPE_PURE_DECIMAL; }
530 :
531 : // Return the token representation as someone would find it in the original
532 : // source code file.
533 : std::string as_string () const;
534 : };
535 : } // namespace Rust
536 :
537 : namespace std {
538 : template <> struct hash<Rust::PrimitiveCoreType>
539 : {
540 198194 : size_t operator() (const Rust::PrimitiveCoreType &coretype) const noexcept
541 : {
542 198194 : return hash<std::underlying_type<Rust::PrimitiveCoreType>::type> () (
543 : static_cast<std::underlying_type<Rust::PrimitiveCoreType>::type> (
544 : coretype));
545 : }
546 : };
547 : } // namespace std
548 :
549 : #endif
|
