Line data Source code
1 : // This file is part of GCC.
2 :
3 : // GCC is free software; you can redistribute it and/or modify it under
4 : // the terms of the GNU General Public License as published by the Free
5 : // Software Foundation; either version 3, or (at your option) any later
6 : // version.
7 :
8 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
9 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 : // for more details.
12 :
13 : // You should have received a copy of the GNU General Public License
14 : // along with GCC; see the file COPYING3. If not see
15 : // <http://www.gnu.org/licenses/>.
16 :
17 : #include "rust-lex.h"
18 : #include "rust-token-converter.h"
19 : #include "bi-map.h"
20 : #include "line-map.h"
21 : #include "rust-system.h"
22 :
23 : namespace Rust {
24 :
25 : static const BiMap<PrimitiveCoreType, std::string> suffixes
26 : = {{{CORETYPE_F32, "f32"},
27 : {CORETYPE_F64, "f64"},
28 : {CORETYPE_U8, "u8"},
29 : {CORETYPE_U16, "u16"},
30 : {CORETYPE_U32, "u32"},
31 : {CORETYPE_U64, "u64"},
32 : {CORETYPE_U128, "u128"},
33 : {CORETYPE_I8, "i8"},
34 : {CORETYPE_I16, "i16"},
35 : {CORETYPE_I32, "i32"},
36 : {CORETYPE_I64, "i64"},
37 : {CORETYPE_I128, "i128"},
38 : {CORETYPE_ISIZE, "isize"},
39 : {CORETYPE_USIZE, "usize"}}};
40 :
41 : static void
42 0 : pop_group (std::vector<ProcMacro::TokenStream> &streams,
43 : ProcMacro::Delimiter delim)
44 : {
45 0 : auto g = ProcMacro::Group::make_group (streams.back (), delim);
46 0 : streams.pop_back ();
47 0 : auto tt = ProcMacro::TokenTree::make_tokentree (g);
48 :
49 0 : streams.back ().push (tt);
50 0 : }
51 :
52 : static ProcMacro::Span
53 0 : convert (location_t location)
54 : {
55 0 : return ProcMacro::Span::make_span (location, 0);
56 : }
57 :
58 : static location_t
59 0 : convert (ProcMacro::Span span)
60 : {
61 0 : return span.start;
62 : }
63 :
64 : static ProcMacro::Literal
65 0 : handle_suffix (const const_TokenPtr &token, ProcMacro::LitKind kind)
66 : {
67 0 : auto str = token->as_string ();
68 0 : auto lookup = suffixes.lookup (token->get_type_hint ());
69 0 : auto suffix = lookup.value_or ("");
70 0 : return ProcMacro::Literal::make_literal (kind, convert (token->get_locus ()),
71 0 : str, suffix);
72 0 : }
73 :
74 : ProcMacro::Literal
75 0 : convert_literal (const_TokenPtr lit)
76 : {
77 0 : auto loc = convert (lit->get_locus ());
78 0 : switch (lit->get_id ())
79 : {
80 0 : case FLOAT_LITERAL:
81 0 : return handle_suffix (lit, ProcMacro::LitKind::make_float ());
82 0 : case INT_LITERAL:
83 0 : return handle_suffix (lit, ProcMacro::LitKind::make_integer ());
84 0 : case CHAR_LITERAL:
85 0 : return ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_char (),
86 0 : loc, lit->as_string ());
87 0 : case STRING_LITERAL:
88 0 : return ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_str (),
89 0 : loc, lit->as_string ());
90 0 : case BYTE_CHAR_LITERAL:
91 0 : return ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_byte (),
92 0 : loc, lit->as_string ());
93 0 : case BYTE_STRING_LITERAL:
94 0 : return ProcMacro::Literal::make_literal (
95 0 : ProcMacro::LitKind::make_byte_str (), loc, lit->as_string ());
96 0 : default:
97 0 : rust_unreachable ();
98 : }
99 : }
100 :
101 : ProcMacro::TokenStream
102 0 : convert (const std::vector<const_TokenPtr> &tokens)
103 : {
104 0 : std::vector<ProcMacro::TokenStream> trees;
105 0 : trees.reserve (tokens.size ());
106 :
107 0 : trees.emplace_back (ProcMacro::TokenStream::make_tokenstream ());
108 0 : for (auto &token : tokens)
109 : {
110 0 : auto loc = convert (token->get_locus ());
111 0 : switch (token->get_id ())
112 : {
113 : // Literals
114 0 : case FLOAT_LITERAL:
115 0 : case INT_LITERAL:
116 0 : case CHAR_LITERAL:
117 0 : case STRING_LITERAL:
118 0 : case BYTE_CHAR_LITERAL:
119 0 : case BYTE_STRING_LITERAL:
120 0 : trees.back ().push (
121 0 : ProcMacro::TokenTree::make_tokentree (convert_literal (token)));
122 0 : break;
123 : // Ident
124 0 : case IDENTIFIER:
125 0 : case ABSTRACT:
126 0 : case AS:
127 0 : case ASYNC:
128 0 : case AUTO:
129 0 : case BECOME:
130 0 : case BOX:
131 0 : case BREAK:
132 0 : case CONST:
133 0 : case CONTINUE:
134 0 : case CRATE:
135 0 : case DO:
136 0 : case DYN:
137 0 : case ELSE:
138 0 : case ENUM_KW:
139 0 : case EXTERN_KW:
140 0 : case FINAL_KW:
141 0 : case FN_KW:
142 0 : case FOR:
143 0 : case IF:
144 0 : case IMPL:
145 0 : case IN:
146 0 : case LET:
147 0 : case LOOP:
148 0 : case MACRO:
149 0 : case MATCH_KW:
150 0 : case MOD:
151 0 : case MOVE:
152 0 : case MUT:
153 0 : case OVERRIDE_KW:
154 0 : case PRIV:
155 0 : case PUB:
156 0 : case REF:
157 0 : case RETURN_KW:
158 0 : case SELF_ALIAS:
159 0 : case SELF:
160 0 : case STATIC_KW:
161 0 : case STRUCT_KW:
162 0 : case SUPER:
163 0 : case TRAIT:
164 0 : case TRY:
165 0 : case TYPE:
166 0 : case TYPEOF:
167 0 : case UNSAFE:
168 0 : case UNSIZED:
169 0 : case USE:
170 0 : case VIRTUAL:
171 0 : case WHERE:
172 0 : case WHILE:
173 0 : case YIELD:
174 : // Underscore is not a Punct, considered as an Ident
175 0 : case UNDERSCORE:
176 : // True and false are idents, not literals
177 : // (https://doc.rust-lang.org/proc_macro/struct.Literal.html)
178 0 : case FALSE_LITERAL:
179 0 : case TRUE_LITERAL:
180 0 : trees.back ().push (ProcMacro::TokenTree::make_tokentree (
181 0 : ProcMacro::Ident::make_ident (token->as_string (), loc)));
182 0 : break;
183 : // Joint punct
184 0 : case OR:
185 0 : case PIPE_EQ:
186 0 : case CARET_EQ:
187 0 : case RIGHT_SHIFT_EQ:
188 0 : case RIGHT_SHIFT:
189 0 : case GREATER_OR_EQUAL:
190 0 : case MATCH_ARROW:
191 0 : case LESS_OR_EQUAL:
192 0 : case LEFT_SHIFT_EQ:
193 0 : case LEFT_SHIFT:
194 0 : case DIV_EQ:
195 0 : case ELLIPSIS:
196 0 : case DOT_DOT_EQ:
197 0 : case DOT_DOT:
198 0 : case RETURN_TYPE:
199 0 : case MINUS_EQ:
200 0 : case PLUS_EQ:
201 0 : case ASTERISK_EQ:
202 0 : case LOGICAL_AND:
203 0 : case AMP_EQ:
204 0 : case PERCENT_EQ:
205 0 : case SCOPE_RESOLUTION:
206 0 : case NOT_EQUAL:
207 0 : case EQUAL_EQUAL:
208 0 : {
209 0 : auto str = token->as_string ();
210 0 : auto it = str.cbegin ();
211 0 : for (; it != str.cend () - 1; it++)
212 0 : trees.back ().push (ProcMacro::TokenTree::make_tokentree (
213 0 : ProcMacro::Punct::make_punct (*it, loc, ProcMacro::JOINT)));
214 0 : trees.back ().push (ProcMacro::TokenTree::make_tokentree (
215 0 : ProcMacro::Punct::make_punct (*it, loc, ProcMacro::ALONE)));
216 0 : }
217 0 : break;
218 : // Alone punct tokens
219 0 : case EQUAL:
220 0 : case RIGHT_ANGLE:
221 0 : case LEFT_ANGLE:
222 0 : case EXCLAM:
223 0 : case TILDE:
224 0 : case PLUS:
225 0 : case MINUS:
226 0 : case ASTERISK:
227 0 : case DIV:
228 0 : case PERCENT:
229 0 : case CARET:
230 0 : case AMP:
231 0 : case PIPE:
232 0 : case PATTERN_BIND:
233 0 : case DOT:
234 0 : case COMMA:
235 0 : case SEMICOLON:
236 0 : case COLON:
237 0 : case HASH:
238 0 : case DOLLAR_SIGN:
239 0 : case QUESTION_MARK:
240 0 : case SINGLE_QUOTE:
241 0 : trees.back ().push (ProcMacro::TokenTree::make_tokentree (
242 0 : ProcMacro::Punct::make_punct (token->as_string ()[0], loc,
243 : ProcMacro::ALONE)));
244 0 : break;
245 0 : case RIGHT_PAREN:
246 0 : pop_group (trees, ProcMacro::PARENTHESIS);
247 0 : break;
248 0 : case RIGHT_CURLY:
249 0 : pop_group (trees, ProcMacro::BRACE);
250 0 : break;
251 0 : case RIGHT_SQUARE:
252 0 : pop_group (trees, ProcMacro::BRACKET);
253 0 : break;
254 0 : case LEFT_SQUARE:
255 0 : case LEFT_CURLY:
256 0 : case LEFT_PAREN:
257 0 : trees.push_back (ProcMacro::TokenStream::make_tokenstream ());
258 0 : break;
259 0 : default:
260 0 : rust_unreachable ();
261 : }
262 : }
263 0 : return trees.back ();
264 0 : }
265 :
266 : static void from_tokenstream (const ProcMacro::TokenStream &ts,
267 : std::vector<const_TokenPtr> &result);
268 :
269 : /**
270 : * Append the token corresponding to a given Ident to a vector.
271 : *
272 : * @param literal Reference to the Ident to convert.
273 : * @param result Reference to the output vector.
274 : */
275 : static void
276 0 : from_ident (const ProcMacro::Ident &ident, std::vector<const_TokenPtr> &result)
277 : {
278 0 : std::string value (ident.value.to_string ());
279 0 : if (ident.is_raw)
280 0 : value = "r#" + value;
281 :
282 0 : Lexer lexer (value, nullptr);
283 0 : auto token = lexer.build_token ();
284 0 : token->set_locus (convert (ident.span));
285 0 : result.push_back (token);
286 0 : }
287 :
288 : /**
289 : * Append the token corresponding to a given Literal to a vector.
290 : *
291 : * @param literal Reference to the Literal to convert.
292 : * @param result Reference to the vector tokens should be appended to.
293 : */
294 : static void
295 0 : from_literal (const ProcMacro::Literal &literal,
296 : std::vector<const_TokenPtr> &result)
297 : {
298 0 : auto lookup = suffixes.lookup (literal.suffix.to_string ());
299 0 : auto loc = convert (literal.span);
300 0 : auto suffix = lookup.value_or (CORETYPE_UNKNOWN);
301 : // FIXME: Add spans instead of empty locations
302 0 : switch (literal.kind.tag)
303 : {
304 0 : case ProcMacro::BYTE:
305 0 : result.push_back (
306 0 : Token::make_byte_char (loc, literal.text.to_string ()[0]));
307 0 : break;
308 0 : case ProcMacro::CHAR:
309 0 : result.push_back (Token::make_char (loc, literal.text.to_string ()[0]));
310 0 : break;
311 0 : case ProcMacro::INTEGER:
312 0 : result.push_back (
313 0 : Token::make_int (loc, literal.text.to_string (), suffix));
314 0 : break;
315 0 : case ProcMacro::FLOAT:
316 0 : result.push_back (
317 0 : Token::make_float (loc, literal.text.to_string (), suffix));
318 0 : break;
319 0 : case ProcMacro::STR:
320 0 : result.push_back (Token::make_string (loc, literal.text.to_string ()));
321 0 : break;
322 0 : case ProcMacro::BYTE_STR:
323 0 : result.push_back (
324 0 : Token::make_byte_string (loc, literal.text.to_string ()));
325 0 : break;
326 : // FIXME: Handle raw string
327 0 : case ProcMacro::STR_RAW:
328 0 : case ProcMacro::BYTE_STR_RAW:
329 0 : default:
330 0 : rust_unreachable ();
331 : }
332 0 : }
333 :
334 : /**
335 : * Accumulate through successive calls multiple Punct until one is tagged
336 : * "Alone", then append the formed token to a given result vector.
337 : *
338 : * @param punct Reference to the Punct to convert.
339 : * @param acc Reference to an accumulator for joined Punct.
340 : * @param result Reference to the output token vector.
341 : */
342 : static void
343 0 : from_punct (const ProcMacro::Punct &punct, std::vector<std::uint32_t> &acc,
344 : std::vector<const_TokenPtr> &result)
345 : {
346 0 : acc.push_back (punct.ch);
347 0 : if (ProcMacro::ALONE == punct.spacing) /* Last punct of a chain */
348 : {
349 : // TODO: UTF-8 string
350 0 : std::string whole (acc.begin (), acc.end ());
351 0 : auto lexer = Lexer (whole, nullptr);
352 0 : auto token = lexer.build_token ();
353 0 : token->set_locus (convert (punct.span));
354 0 : result.push_back (token);
355 0 : acc.clear ();
356 0 : }
357 0 : }
358 :
359 : /**
360 : * Iterate over a Group and append all inner tokens to a vector enclosed by its
361 : * delimiters.
362 : *
363 : * @param g Reference to the Group to convert.
364 : * @param result Reference to the vector tokens should be appended to.
365 : */
366 : static void
367 0 : from_group (const ProcMacro::Group &g, std::vector<const_TokenPtr> &result)
368 : {
369 0 : auto loc = convert (g.span);
370 0 : switch (g.delimiter)
371 : {
372 0 : case ProcMacro::PARENTHESIS:
373 0 : result.push_back (Token::make (LEFT_PAREN, loc));
374 0 : from_tokenstream (g.stream, result);
375 0 : result.push_back (Token::make (RIGHT_PAREN, loc));
376 0 : break;
377 0 : case ProcMacro::BRACE:
378 0 : result.push_back (Token::make (LEFT_CURLY, loc));
379 0 : from_tokenstream (g.stream, result);
380 0 : result.push_back (Token::make (RIGHT_CURLY, loc));
381 0 : break;
382 0 : case ProcMacro::BRACKET:
383 0 : result.push_back (Token::make (LEFT_SQUARE, loc));
384 0 : from_tokenstream (g.stream, result);
385 0 : result.push_back (Token::make (RIGHT_SQUARE, loc));
386 0 : break;
387 0 : case ProcMacro::NONE:
388 0 : from_tokenstream (g.stream, result);
389 0 : break;
390 0 : default:
391 0 : rust_unreachable ();
392 : }
393 0 : }
394 :
395 : /**
396 : * Dispatch TokenTree's conversion to its inner type depending on its tag.
397 : *
398 : * @param tt Reference to the TokenTree.
399 : * @param punct_accumulator Reference to an accumulator for joined Punct.
400 : * @param result Reference to the vector tokens should be appended to.
401 : */
402 : static void
403 0 : from_tokentree (const ProcMacro::TokenTree &tt,
404 : std::vector<std::uint32_t> &punct_accumulator,
405 : std::vector<const_TokenPtr> &result)
406 : {
407 0 : switch (tt.tag)
408 : {
409 0 : case ProcMacro::GROUP:
410 0 : from_group (tt.payload.group, result);
411 0 : break;
412 0 : case ProcMacro::IDENT:
413 0 : from_ident (tt.payload.ident, result);
414 0 : break;
415 0 : case ProcMacro::PUNCT:
416 0 : from_punct (tt.payload.punct, punct_accumulator, result);
417 0 : break;
418 0 : case ProcMacro::LITERAL:
419 0 : from_literal (tt.payload.literal, result);
420 0 : break;
421 0 : default:
422 0 : rust_unreachable ();
423 : }
424 0 : }
425 :
426 : /**
427 : * Iterate over a TokenStream and append all inner tokens to a vector.
428 : *
429 : * @param ts Reference to the TokenStream.
430 : * @param result Reference to the vector tokens should be appended to.
431 : */
432 : static void
433 0 : from_tokenstream (const ProcMacro::TokenStream &ts,
434 : std::vector<const_TokenPtr> &result)
435 : {
436 0 : std::vector<std::uint32_t> punct_accumulator;
437 0 : for (std::uint64_t i = 0; i < ts.size; i++)
438 : {
439 0 : from_tokentree (ts.data[i], punct_accumulator, result);
440 : }
441 0 : }
442 :
443 : std::vector<const_TokenPtr>
444 0 : convert (const ProcMacro::TokenStream &ts)
445 : {
446 0 : std::vector<const_TokenPtr> result;
447 0 : from_tokenstream (ts, result);
448 0 : return result;
449 : }
450 :
451 : } // namespace Rust
|
