LCOV - code coverage report
Current view: top level - gcc/rust/expand - rust-macro-expand.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 87.5 % 583 510
Test Date: 2026-04-20 14:57:17 Functions: 92.7 % 41 38
Legend: Lines:     hit not hit

            Line data    Source code
       1              : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
       2              : 
       3              : // This file is part of GCC.
       4              : 
       5              : // GCC is free software; you can redistribute it and/or modify it under
       6              : // the terms of the GNU General Public License as published by the Free
       7              : // Software Foundation; either version 3, or (at your option) any later
       8              : // version.
       9              : 
      10              : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      11              : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
      12              : // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      13              : // for more details.
      14              : 
      15              : // You should have received a copy of the GNU General Public License
      16              : // along with GCC; see the file COPYING3.  If not see
      17              : // <http://www.gnu.org/licenses/>.
      18              : 
      19              : #include "rust-macro-expand.h"
      20              : #include "optional.h"
      21              : #include "rust-ast-fragment.h"
      22              : #include "rust-macro-builtins.h"
      23              : #include "rust-macro-substitute-ctx.h"
      24              : #include "rust-ast-full.h"
      25              : #include "rust-ast-visitor.h"
      26              : #include "rust-diagnostics.h"
      27              : #include "rust-macro.h"
      28              : #include "rust-parse.h"
      29              : #include "rust-cfg-strip.h"
      30              : #include "rust-proc-macro.h"
      31              : #include "rust-token-tree-desugar.h"
      32              : 
      33              : namespace Rust {
      34              : 
      35              : AST::Fragment
      36         2449 : MacroExpander::expand_decl_macro (location_t invoc_locus,
      37              :                                   AST::MacroInvocData &invoc,
      38              :                                   AST::MacroRulesDefinition &rules_def,
      39              :                                   AST::InvocKind semicolon)
      40              : {
      41              :   // ensure that both invocation and rules are in a valid state
      42         2449 :   rust_assert (!invoc.is_marked_for_strip ());
      43         2449 :   rust_assert (!rules_def.is_marked_for_strip ());
      44         2449 :   rust_assert (rules_def.get_macro_rules ().size () > 0);
      45              : 
      46              :   /* probably something here about parsing invoc and rules def token trees to
      47              :    * token stream. if not, how would parser handle the captures of exprs and
      48              :    * stuff? on the other hand, token trees may be kind of useful in rules def
      49              :    * as creating a point where recursion can occur (like having
      50              :    * "compare_macro_match" and then it calling itself when it finds
      51              :    * delimiters)
      52              :    */
      53              : 
      54              :   /* find matching rule to invoc token tree, based on macro rule's matcher. if
      55              :    * none exist, error.
      56              :    * - specifically, check each matcher in order. if one fails to match, move
      57              :    * onto next. */
      58              :   /* TODO: does doing this require parsing expressions and whatever in the
      59              :    * invoc? if so, might as well save the results if referenced using $ or
      60              :    * whatever. If not, do another pass saving them. Except this is probably
      61              :    * useless as different rules could have different starting points for exprs
      62              :    * or whatever. Decision trees could avoid this, but they have their own
      63              :    * issues. */
      64              :   /* TODO: will need to modify the parser so that it can essentially "catch"
      65              :    * errors - maybe "try_parse_expr" or whatever methods. */
      66              :   // this technically creates a back-tracking parser - this will be the
      67              :   // implementation style
      68              : 
      69              :   /* then, after results are saved, generate the macro output from the
      70              :    * transcriber token tree. if i understand this correctly, the macro
      71              :    * invocation gets replaced by the transcriber tokens, except with
      72              :    * substitutions made (e.g. for $i variables) */
      73              : 
      74              :   /* TODO: it is probably better to modify AST::Token to store a pointer to a
      75              :    * Lexer::Token (rather than being converted) - i.e. not so much have
      76              :    * AST::Token as a Token but rather a TokenContainer (as it is another type
      77              :    * of TokenTree). This will prevent re-conversion of Tokens between each
      78              :    * type all the time, while still allowing the heterogenous storage of token
      79              :    * trees.
      80              :    */
      81              : 
      82         2449 :   AST::DelimTokenTree &invoc_token_tree_sugar = invoc.get_delim_tok_tree ();
      83              : 
      84              :   // We must first desugar doc comments into proper attributes
      85         2449 :   auto invoc_token_tree = AST::TokenTreeDesugar ().go (invoc_token_tree_sugar);
      86              : 
      87              :   // find matching arm
      88         2449 :   AST::MacroRule *matched_rule = nullptr;
      89         2449 :   std::map<std::string, std::unique_ptr<MatchedFragmentContainer>>
      90         2449 :     matched_fragments;
      91         3845 :   for (auto &rule : rules_def.get_rules ())
      92              :     {
      93         3837 :       sub_stack.push ();
      94         3837 :       bool did_match_rule = try_match_rule (rule, invoc_token_tree);
      95         7674 :       matched_fragments = sub_stack.pop ();
      96              : 
      97         3837 :       if (did_match_rule)
      98              :         {
      99              :           //  // Debugging
     100              :           //  for (auto &kv : matched_fragments)
     101              :           //    rust_debug ("[fragment]: %s (%ld - %s)", kv.first.c_str (),
     102              :           //            kv.second.get_fragments ().size (),
     103              :           //            kv.second.get_kind ()
     104              :           //                == MatchedFragmentContainer::Kind::Repetition
     105              :           //              ? "repetition"
     106              :           //              : "metavar");
     107              : 
     108              :           matched_rule = &rule;
     109              :           break;
     110              :         }
     111              :     }
     112              : 
     113         2449 :   if (matched_rule == nullptr)
     114              :     {
     115            8 :       if (!had_duplicate_error)
     116              :         {
     117            7 :           rich_location r (line_table, invoc_locus);
     118            7 :           r.add_range (rules_def.get_locus ());
     119            7 :           rust_error_at (r, "Failed to match any rule within macro");
     120            7 :         }
     121            8 :       had_duplicate_error = false;
     122            8 :       return AST::Fragment::create_error ();
     123              :     }
     124              : 
     125         2441 :   std::map<std::string, MatchedFragmentContainer *> matched_fragments_ptr;
     126              : 
     127         6475 :   for (auto &ent : matched_fragments)
     128         4034 :     matched_fragments_ptr.emplace (ent.first, ent.second.get ());
     129              : 
     130         2441 :   return transcribe_rule (rules_def, *matched_rule, invoc_token_tree,
     131         2441 :                           matched_fragments_ptr, semicolon, peek_context ());
     132         4890 : }
     133              : 
     134              : void
     135           45 : MacroExpander::expand_eager_invocations (AST::MacroInvocation &invoc)
     136              : {
     137           45 :   if (invoc.get_pending_eager_invocations ().empty ())
     138            0 :     return;
     139              : 
     140              :   // We have to basically create a new delimited token tree which contains the
     141              :   // result of one step of expansion. In the case of builtin macros called with
     142              :   // other macro invocations, such as `concat!("h", 'a', a!())`, we need to
     143              :   // expand `a!()` before expanding the concat macro.
     144              :   // This will, ideally, give us a new token tree containing the various
     145              :   // existing tokens + the result of the expansion of a!().
     146              :   // To do this, we "parse" the given token tree to find anything that "looks
     147              :   // like a macro invocation". Then, we get the corresponding macro invocation
     148              :   // from the `pending_eager_invocations` vector and expand it.
     149              :   // Because the `pending_eager_invocations` vector is created in the same order
     150              :   // that the DelimTokenTree is parsed, we know that the first macro invocation
     151              :   // within the DelimTokenTree corresponds to the first element in
     152              :   // `pending_eager_invocations`. The idea is thus to:
     153              :   // 1. Find a macro invocation in the token tree, noting the index of the start
     154              :   //    token and of the end token
     155              :   // 2. Get its associated invocation in `pending_eager_invocations`
     156              :   // 3. Expand that element
     157              :   // 4. Get the token tree associated with that AST fragment
     158              :   // 5. Replace the original tokens corresponding to the invocation with the new
     159              :   //    tokens from the fragment
     160              :   // pseudo-code:
     161              :   //
     162              :   // i = 0;
     163              :   // for tok in dtt:
     164              :   //   if tok is identifier && tok->next() is !:
     165              :   //     start = index(tok);
     166              :   //     l_delim = tok->next()->next();
     167              :   //     tok = skip_until_r_delim();
     168              :   //     end = index(tok);
     169              :   //
     170              :   //     new_tt = expand_eager_invoc(eagers[i++]);
     171              :   //     old_tt[start..end] = new_tt;
     172              : 
     173           45 :   auto dtt = invoc.get_invoc_data ().get_delim_tok_tree ();
     174           45 :   auto stream = dtt.to_token_stream ();
     175           45 :   std::vector<std::unique_ptr<AST::TokenTree>> new_stream;
     176           45 :   size_t current_pending = 0;
     177              : 
     178              :   // we need to create a clone of the delimited token tree as the lexer
     179              :   // expects ownership of the tokens
     180           45 :   std::vector<const_TokenPtr> dtt_clone;
     181          389 :   for (auto &tok : stream)
     182          688 :     dtt_clone.emplace_back (tok->get_tok_ptr ());
     183              : 
     184           45 :   MacroInvocLexer lex (std::move (dtt_clone));
     185           45 :   Parser<MacroInvocLexer> parser (lex);
     186              : 
     187              :   // we want to build a substitution map - basically, associating a `start` and
     188              :   // `end` index for each of the pending macro invocations
     189           45 :   std::map<std::pair<size_t, size_t>, std::unique_ptr<AST::MacroInvocation> &>
     190           45 :     substitution_map;
     191              : 
     192          389 :   for (size_t i = 0; i < stream.size (); i++)
     193              :     {
     194              :       // FIXME: Can't these offsets be figure out when we actually parse the
     195              :       // pending_eager_invocation in the first place?
     196          344 :       auto invocation = parser.parse_macro_invocation ({});
     197              : 
     198              :       // if we've managed to parse a macro invocation, we look at the current
     199              :       // offset and store them in the substitution map. Otherwise, we skip one
     200              :       // token and try parsing again
     201          344 :       if (invocation)
     202           50 :         substitution_map.insert (
     203           50 :           {{i, parser.get_token_source ().get_offs ()},
     204           50 :            invoc.get_pending_eager_invocations ()[current_pending++]});
     205              :       else
     206          294 :         parser.skip_token (stream[i]->get_id ());
     207          344 :     }
     208              : 
     209           45 :   size_t current_idx = 0;
     210           95 :   for (auto kv : substitution_map)
     211              :     {
     212           50 :       auto &to_expand = kv.second;
     213           50 :       expand_invoc (*to_expand, AST::InvocKind::Expr);
     214              : 
     215           50 :       auto fragment = take_expanded_fragment ();
     216           50 :       auto &new_tokens = fragment.get_tokens ();
     217              : 
     218           50 :       auto start = kv.first.first;
     219           50 :       auto end = kv.first.second;
     220              : 
     221              :       // We're now going to re-add the tokens to the invocation's token tree.
     222              :       // 1. Basically, what we want to do is insert all tokens up until the
     223              :       //    beginning of the macro invocation (start).
     224              :       // 2. Then, we'll insert all of the tokens resulting from the macro
     225              :       //    expansion: These are in `new_tokens`.
     226              :       // 3. Finally, we'll do that again from
     227              :       //    the end of macro and go back to 1.
     228              : 
     229          106 :       for (size_t i = current_idx; i < start; i++)
     230           56 :         new_stream.emplace_back (stream[i]->clone_token ());
     231              : 
     232          100 :       for (auto &tok : new_tokens)
     233           50 :         new_stream.emplace_back (tok->clone_token ());
     234              : 
     235           50 :       current_idx = end;
     236           50 :     }
     237              : 
     238              :   // Once all of that is done, we copy the last remaining tokens from the
     239              :   // original stream
     240          117 :   for (size_t i = current_idx; i < stream.size (); i++)
     241           72 :     new_stream.emplace_back (stream[i]->clone_token ());
     242              : 
     243           45 :   auto new_dtt
     244           45 :     = AST::DelimTokenTree (dtt.get_delim_type (), std::move (new_stream));
     245              : 
     246           45 :   invoc.get_pending_eager_invocations ().clear ();
     247           45 :   invoc.get_invoc_data ().set_delim_tok_tree (new_dtt);
     248           45 : }
     249              : 
     250              : void
     251         2852 : MacroExpander::expand_invoc (AST::MacroInvocation &invoc,
     252              :                              AST::InvocKind semicolon)
     253              : {
     254         2852 :   if (depth_exceeds_recursion_limit ())
     255              :     {
     256            0 :       rust_error_at (invoc.get_locus (), "reached recursion limit");
     257           39 :       return;
     258              :     }
     259              : 
     260         2852 :   if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin)
     261              :     {
     262              :       // Eager expansions are always expressions
     263           45 :       push_context (ContextType::EXPR);
     264           45 :       expand_eager_invocations (invoc);
     265           45 :       pop_context ();
     266              :     }
     267              : 
     268         2852 :   AST::MacroInvocData &invoc_data = invoc.get_invoc_data ();
     269              : 
     270              :   // ??
     271              :   // switch on type of macro:
     272              :   //  - '!' syntax macro (inner switch)
     273              :   //      - procedural macro - "A token-based function-like macro"
     274              :   //      - 'macro_rules' (by example/pattern-match) macro? or not? "an
     275              :   // AST-based function-like macro"
     276              :   //      - else is unreachable
     277              :   //  - attribute syntax macro (inner switch)
     278              :   //  - procedural macro attribute syntax - "A token-based attribute
     279              :   // macro"
     280              :   //      - legacy macro attribute syntax? - "an AST-based attribute macro"
     281              :   //      - non-macro attribute: mark known
     282              :   //      - else is unreachable
     283              :   //  - derive macro (inner switch)
     284              :   //      - derive or legacy derive - "token-based" vs "AST-based"
     285              :   //      - else is unreachable
     286              :   //  - derive container macro - unreachable
     287              : 
     288         2852 :   auto fragment = AST::Fragment::create_error ();
     289         2852 :   invoc_data.set_expander (this);
     290              : 
     291              :   // lookup the rules
     292         2852 :   auto rules_def = mappings.lookup_macro_invocation (invoc);
     293              : 
     294              :   // We special case the `offset_of!()` macro if the flag is here and manually
     295              :   // resolve to the builtin transcriber we have specified
     296         2852 :   auto assume_builtin_offset_of
     297         2852 :     = flag_assume_builtin_offset_of
     298           36 :       && (invoc.get_invoc_data ().get_path ().as_string () == "offset_of")
     299         2870 :       && !rules_def;
     300              : 
     301              :   // TODO: This is *massive hack* which should be removed as we progress to
     302              :   // Rust 1.71 when offset_of gets added to core
     303         2852 :   if (assume_builtin_offset_of)
     304              :     {
     305           18 :       fragment = MacroBuiltin::offset_of_handler (invoc.get_locus (),
     306              :                                                   invoc_data, semicolon)
     307           54 :                    .value_or (AST::Fragment::create_empty ());
     308              : 
     309           18 :       set_expanded_fragment (std::move (fragment));
     310              : 
     311           18 :       return;
     312              :     }
     313              : 
     314              :   // If there's no rule associated with the invocation, we can simply return
     315              :   // early. The early name resolver will have already emitted an error.
     316         2834 :   if (!rules_def)
     317              :     return;
     318              : 
     319         2813 :   auto rdef = rules_def.value ();
     320              : 
     321              :   // We store the last expanded invocation and macro definition for error
     322              :   // reporting in case the recursion limit is reached
     323         2813 :   last_invoc = *invoc.clone_macro_invocation_impl ();
     324         2813 :   last_def = *rdef;
     325              : 
     326         2813 :   if (rdef->is_builtin ())
     327          364 :     fragment = rdef
     328          364 :                  ->get_builtin_transcriber () (invoc.get_locus (), invoc_data,
     329              :                                                semicolon)
     330         1092 :                  .value_or (AST::Fragment::create_empty ());
     331              :   else
     332         2449 :     fragment
     333         4898 :       = expand_decl_macro (invoc.get_locus (), invoc_data, *rdef, semicolon);
     334              :   // fix: if the expansion is failing, we must replace the marco with an empty
     335              :   // error or node
     336              :   // makes sure that it doesn't panic on Rouge macro (it -> Lowering Phase)
     337              :   // added the parsing errors in gcc/testsuite/rust/compile/issue-4213.rs
     338         2813 :   if (fragment.is_error ())
     339              :     {
     340          112 :       fragment = AST::Fragment::create_empty ();
     341              :     }
     342         2813 :   set_expanded_fragment (std::move (fragment));
     343         2852 : }
     344              : 
     345              : void
     346            0 : MacroExpander::expand_crate ()
     347              : {
     348              :   /* fill macro/decorator map from init list? not sure where init list comes
     349              :    * from? */
     350              : 
     351              :   // TODO: does cfg apply for inner attributes? research.
     352              :   // the apparent answer (from playground test) is yes
     353              : 
     354            0 :   push_context (ContextType::ITEM);
     355              : 
     356              :   // expand attributes recursively and strip items if required
     357              :   //  AttrVisitor attr_visitor (*this);
     358            0 :   auto &items = crate.items;
     359            0 :   for (auto it = items.begin (); it != items.end ();)
     360              :     {
     361            0 :       auto &item = *it;
     362              : 
     363            0 :       auto fragment = take_expanded_fragment ();
     364            0 :       if (fragment.should_expand ())
     365              :         {
     366              :           // Remove the current expanded invocation
     367            0 :           it = items.erase (it);
     368            0 :           for (auto &node : fragment.get_nodes ())
     369              :             {
     370            0 :               it = items.insert (it, node.take_item ());
     371            0 :               it++;
     372              :             }
     373              :         }
     374            0 :       else if (item->is_marked_for_strip ())
     375            0 :         it = items.erase (it);
     376              :       else
     377            0 :         it++;
     378            0 :     }
     379              : 
     380            0 :   pop_context ();
     381              : 
     382              :   // TODO: should recursive attribute and macro expansion be done in the same
     383              :   // transversal? Or in separate ones like currently?
     384              : 
     385              :   // expand module tree recursively
     386              : 
     387              :   // post-process
     388              : 
     389              :   // extract exported macros?
     390            0 : }
     391              : 
     392              : bool
     393         6785 : MacroExpander::depth_exceeds_recursion_limit () const
     394              : {
     395         6785 :   return expansion_depth >= cfg.recursion_limit;
     396              : }
     397              : 
     398              : bool
     399         3837 : MacroExpander::try_match_rule (AST::MacroRule &match_rule,
     400              :                                AST::DelimTokenTree &invoc_token_tree)
     401              : {
     402         3837 :   MacroInvocLexer lex (invoc_token_tree.to_token_stream ());
     403         3837 :   Parser<MacroInvocLexer> parser (lex);
     404              : 
     405         3837 :   AST::MacroMatcher &matcher = match_rule.get_matcher ();
     406              : 
     407         3837 :   expansion_depth++;
     408         3837 :   if (!match_matcher (parser, matcher, false, false))
     409              :     {
     410         1396 :       expansion_depth--;
     411         1396 :       return false;
     412              :     }
     413         2441 :   expansion_depth--;
     414              : 
     415         2441 :   bool used_all_input_tokens = parser.skip_token (END_OF_FILE);
     416         2441 :   return used_all_input_tokens;
     417         3837 : }
     418              : 
     419              : bool
     420         8166 : MacroExpander::match_fragment (Parser<MacroInvocLexer> &parser,
     421              :                                AST::MacroMatchFragment &fragment)
     422              : {
     423         8166 :   switch (fragment.get_frag_spec ().get_kind ())
     424              :     {
     425         1183 :     case AST::MacroFragSpec::EXPR:
     426         2365 :       parser.parse_expr ();
     427         1183 :       break;
     428              : 
     429            2 :     case AST::MacroFragSpec::BLOCK:
     430            4 :       parser.parse_block_expr ();
     431            2 :       break;
     432              : 
     433          480 :     case AST::MacroFragSpec::IDENT:
     434          480 :       parser.parse_identifier_or_keyword_token ();
     435          480 :       break;
     436              : 
     437         3474 :     case AST::MacroFragSpec::LITERAL:
     438         6945 :       std::ignore = parser.parse_literal_expr ();
     439         3474 :       break;
     440              : 
     441            1 :     case AST::MacroFragSpec::ITEM:
     442            1 :       parser.parse_item (false);
     443            1 :       break;
     444              : 
     445         2380 :     case AST::MacroFragSpec::TY:
     446         2380 :       parser.parse_type ();
     447         2380 :       break;
     448              : 
     449           17 :     case AST::MacroFragSpec::PAT:
     450           17 :       parser.parse_pattern ();
     451           17 :       break;
     452              : 
     453            0 :     case AST::MacroFragSpec::PATH:
     454            0 :       parser.parse_path_in_expression ();
     455            0 :       break;
     456              : 
     457            0 :     case AST::MacroFragSpec::VIS:
     458            0 :       parser.parse_visibility ();
     459            0 :       break;
     460              : 
     461          301 :     case AST::MacroFragSpec::STMT:
     462          301 :       {
     463          301 :         auto restrictions = ParseRestrictions ();
     464          301 :         restrictions.consume_semi = false;
     465          301 :         parser.parse_stmt (restrictions);
     466          301 :         break;
     467              :       }
     468              : 
     469            4 :     case AST::MacroFragSpec::LIFETIME:
     470            4 :       parser.parse_lifetime_params ();
     471            4 :       break;
     472              : 
     473              :       // is meta attributes?
     474           46 :     case AST::MacroFragSpec::META:
     475           46 :       parser.parse_attribute_body ();
     476           46 :       break;
     477              : 
     478          278 :     case AST::MacroFragSpec::TT:
     479          278 :       parser.parse_token_tree ();
     480          278 :       break;
     481              : 
     482              :       // i guess we just ignore invalid and just error out
     483              :     case AST::MacroFragSpec::INVALID:
     484              :       return false;
     485              :     }
     486              : 
     487              :   // it matches if the parser did not produce errors trying to parse that type
     488              :   // of item
     489         8166 :   return !parser.has_errors ();
     490              : }
     491              : 
     492              : bool
     493         3933 : MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
     494              :                               AST::MacroMatcher &matcher, bool in_repetition,
     495              :                               bool match_delim)
     496              : {
     497         3933 :   if (depth_exceeds_recursion_limit ())
     498              :     {
     499            0 :       rust_error_at (matcher.get_match_locus (), "reached recursion limit");
     500            0 :       return false;
     501              :     }
     502              : 
     503         3933 :   auto delimiter = parser.peek_current_token ();
     504              : 
     505         7862 :   auto check_delim = [&matcher, match_delim] (AST::DelimType delim) {
     506           92 :     return !match_delim || matcher.get_delim_type () == delim;
     507         3933 :   };
     508              : 
     509              :   // this is used so we can check that we delimit the stream correctly.
     510         3933 :   switch (delimiter->get_id ())
     511              :     {
     512         3626 :     case LEFT_PAREN:
     513         3626 :       {
     514         3626 :         if (!check_delim (AST::DelimType::PARENS))
     515              :           return false;
     516              :       }
     517              :       break;
     518              : 
     519           46 :     case LEFT_SQUARE:
     520           46 :       {
     521           46 :         if (!check_delim (AST::DelimType::SQUARE))
     522              :           return false;
     523              :       }
     524              :       break;
     525              : 
     526          257 :     case LEFT_CURLY:
     527          257 :       {
     528         4190 :         if (!check_delim (AST::DelimType::CURLY))
     529              :           return false;
     530              :       }
     531              :       break;
     532              :     default:
     533              :       return false;
     534              :     }
     535         3928 :   parser.skip_token ();
     536              : 
     537         3928 :   const MacroInvocLexer &source = parser.get_token_source ();
     538              : 
     539         7856 :   std::unordered_map<std::string, location_t> duplicate_check;
     540              : 
     541        10469 :   for (auto &match : matcher.get_matches ())
     542              :     {
     543         6586 :       size_t offs_begin = source.get_offs ();
     544              : 
     545         6586 :       switch (match->get_macro_match_type ())
     546              :         {
     547         3743 :         case AST::MacroMatch::MacroMatchType::Fragment:
     548         3743 :           {
     549         3743 :             AST::MacroMatchFragment *fragment
     550         3743 :               = static_cast<AST::MacroMatchFragment *> (match.get ());
     551         3743 :             if (!match_fragment (parser, *fragment))
     552            3 :               return false;
     553              : 
     554         7482 :             auto duplicate_result = duplicate_check.insert (
     555        11223 :               std::make_pair (fragment->get_ident ().as_string (),
     556         3741 :                               fragment->get_ident ().get_locus ()));
     557              : 
     558         3741 :             if (!duplicate_result.second)
     559              :               {
     560              :                 // TODO: add range labels?
     561            1 :                 rich_location r (line_table,
     562            1 :                                  fragment->get_ident ().get_locus ());
     563            1 :                 r.add_range (duplicate_result.first->second);
     564            1 :                 rust_error_at (r, "duplicate matcher binding");
     565            1 :                 had_duplicate_error = true;
     566            1 :                 return false;
     567            1 :               }
     568              : 
     569              :             // matched fragment get the offset in the token stream
     570         3740 :             size_t offs_end = source.get_offs ();
     571         7480 :             sub_stack.insert_metavar (
     572         7480 :               MatchedFragment (fragment->get_ident ().as_string (), offs_begin,
     573        11220 :                                offs_end));
     574              :           }
     575         3740 :           break;
     576              : 
     577         1068 :         case AST::MacroMatch::MacroMatchType::Tok:
     578         1068 :           {
     579         1068 :             AST::Token *tok = static_cast<AST::Token *> (match.get ());
     580         1068 :             if (!match_token (parser, *tok))
     581              :               return false;
     582              :           }
     583              :           break;
     584              : 
     585         1712 :         case AST::MacroMatch::MacroMatchType::Repetition:
     586         1712 :           {
     587         1712 :             AST::MacroMatchRepetition *rep
     588         1712 :               = static_cast<AST::MacroMatchRepetition *> (match.get ());
     589         1712 :             if (!match_repetition (parser, *rep))
     590              :               return false;
     591              :           }
     592              :           break;
     593              : 
     594           63 :         case AST::MacroMatch::MacroMatchType::Matcher:
     595           63 :           {
     596           63 :             AST::MacroMatcher *m
     597           63 :               = static_cast<AST::MacroMatcher *> (match.get ());
     598           63 :             expansion_depth++;
     599           63 :             if (!match_matcher (parser, *m, in_repetition))
     600              :               {
     601            3 :                 expansion_depth--;
     602            3 :                 return false;
     603              :               }
     604           60 :             expansion_depth--;
     605              :           }
     606           60 :           break;
     607              :         }
     608              :     }
     609              : 
     610         3883 :   switch (delimiter->get_id ())
     611              :     {
     612         3581 :     case LEFT_PAREN:
     613         3581 :       {
     614         3581 :         if (!parser.skip_token (RIGHT_PAREN))
     615              :           return false;
     616              :       }
     617              :       break;
     618              : 
     619           46 :     case LEFT_SQUARE:
     620           46 :       {
     621           46 :         if (!parser.skip_token (RIGHT_SQUARE))
     622              :           return false;
     623              :       }
     624              :       break;
     625              : 
     626          256 :     case LEFT_CURLY:
     627          256 :       {
     628          256 :         if (!parser.skip_token (RIGHT_CURLY))
     629              :           return false;
     630              :       }
     631              :       break;
     632            0 :     default:
     633            0 :       rust_unreachable ();
     634              :     }
     635              : 
     636              :   return true;
     637         3933 : }
     638              : 
     639              : bool
     640         2694 : MacroExpander::match_token (Parser<MacroInvocLexer> &parser, AST::Token &token)
     641              : {
     642         5388 :   return parser.skip_token (token.get_tok_ptr ());
     643              : }
     644              : 
     645              : bool
     646         1726 : MacroExpander::match_n_matches (Parser<MacroInvocLexer> &parser,
     647              :                                 AST::MacroMatchRepetition &rep,
     648              :                                 size_t &match_amount, size_t lo_bound,
     649              :                                 size_t hi_bound)
     650              : {
     651         1726 :   match_amount = 0;
     652         1726 :   auto &matches = rep.get_matches ();
     653              : 
     654         1726 :   const MacroInvocLexer &source = parser.get_token_source ();
     655        10152 :   while (true)
     656              :     {
     657              :       // If the current token is a closing macro delimiter, break away.
     658              :       // TODO: Is this correct?
     659         5939 :       auto t_id = parser.peek_current_token ()->get_id ();
     660         5939 :       if (t_id == RIGHT_PAREN || t_id == RIGHT_SQUARE || t_id == RIGHT_CURLY)
     661              :         break;
     662              : 
     663              :       // Skip parsing a separator on the first match, otherwise consume it.
     664              :       // If it isn't present, this is an error
     665         4239 :       if (rep.has_sep () && match_amount > 0)
     666          357 :         if (!match_token (parser, *rep.get_sep ()))
     667              :           break;
     668              : 
     669         4224 :       sub_stack.push ();
     670         4224 :       bool valid_current_match = false;
     671         9963 :       for (auto &match : matches)
     672              :         {
     673         5739 :           size_t offs_begin = source.get_offs ();
     674         5739 :           switch (match->get_macro_match_type ())
     675              :             {
     676         4423 :             case AST::MacroMatch::MacroMatchType::Fragment:
     677         4423 :               {
     678         4423 :                 AST::MacroMatchFragment *fragment
     679         4423 :                   = static_cast<AST::MacroMatchFragment *> (match.get ());
     680         4423 :                 valid_current_match = match_fragment (parser, *fragment);
     681              : 
     682              :                 // matched fragment get the offset in the token stream
     683         4423 :                 size_t offs_end = source.get_offs ();
     684              : 
     685         4423 :                 if (valid_current_match)
     686         8834 :                   sub_stack.insert_metavar (
     687         8834 :                     MatchedFragment (fragment->get_ident ().as_string (),
     688        13251 :                                      offs_begin, offs_end));
     689              :               }
     690              :               break;
     691              : 
     692         1269 :             case AST::MacroMatch::MacroMatchType::Tok:
     693         1269 :               {
     694         1269 :                 AST::Token *tok = static_cast<AST::Token *> (match.get ());
     695         1269 :                 valid_current_match = match_token (parser, *tok);
     696              :               }
     697         1269 :               break;
     698              : 
     699           14 :             case AST::MacroMatch::MacroMatchType::Repetition:
     700           14 :               {
     701           14 :                 AST::MacroMatchRepetition *rep
     702           14 :                   = static_cast<AST::MacroMatchRepetition *> (match.get ());
     703           14 :                 valid_current_match = match_repetition (parser, *rep);
     704              :               }
     705           14 :               break;
     706              : 
     707           33 :             case AST::MacroMatch::MacroMatchType::Matcher:
     708           33 :               {
     709           33 :                 AST::MacroMatcher *m
     710           33 :                   = static_cast<AST::MacroMatcher *> (match.get ());
     711           33 :                 valid_current_match = match_matcher (parser, *m, true);
     712              :               }
     713           33 :               break;
     714              :             }
     715              :         }
     716         4224 :       auto old_stack = sub_stack.pop ();
     717              : 
     718              :       // If we've encountered an error once, stop trying to match more
     719              :       // repetitions
     720         4224 :       if (!valid_current_match)
     721              :         break;
     722              : 
     723              :       // nest metavars into repetitions
     724         8655 :       for (auto &ent : old_stack)
     725         8882 :         sub_stack.append_fragment (ent.first, std::move (ent.second));
     726              : 
     727         4214 :       match_amount++;
     728              : 
     729              :       // Break early if we notice there's too many expressions already
     730         4214 :       if (hi_bound && match_amount > hi_bound)
     731              :         break;
     732         4224 :     }
     733              : 
     734              :   // Check if the amount of matches we got is valid: Is it more than the lower
     735              :   // bound and less than the higher bound?
     736         1726 :   bool did_meet_lo_bound = match_amount >= lo_bound;
     737         1726 :   bool did_meet_hi_bound = hi_bound ? match_amount <= hi_bound : true;
     738              : 
     739              :   // If the end-result is valid, then we can clear the parse errors: Since
     740              :   // repetitions are parsed eagerly, it is okay to fail in some cases
     741         3451 :   auto res = did_meet_lo_bound && did_meet_hi_bound;
     742         1725 :   if (res)
     743         1721 :     parser.clear_errors ();
     744              : 
     745         1726 :   return res;
     746              : }
     747              : 
     748              : /*
     749              :  * Helper function for defining unmatched repetition metavars
     750              :  */
     751              : void
     752         2922 : MacroExpander::match_repetition_skipped_metavars (AST::MacroMatch &match)
     753              : {
     754              :   // We have to handle zero fragments differently: They will not have been
     755              :   // "matched" but they are still valid and should be inserted as a special
     756              :   // case. So we go through the stack map, and for every fragment which doesn't
     757              :   // exist, insert a zero-matched fragment.
     758         2922 :   switch (match.get_macro_match_type ())
     759              :     {
     760         1770 :     case AST::MacroMatch::MacroMatchType::Fragment:
     761         1770 :       match_repetition_skipped_metavars (
     762              :         static_cast<AST::MacroMatchFragment &> (match));
     763         1770 :       break;
     764           12 :     case AST::MacroMatch::MacroMatchType::Repetition:
     765           12 :       match_repetition_skipped_metavars (
     766              :         static_cast<AST::MacroMatchRepetition &> (match));
     767           12 :       break;
     768           13 :     case AST::MacroMatch::MacroMatchType::Matcher:
     769           13 :       match_repetition_skipped_metavars (
     770              :         static_cast<AST::MacroMatcher &> (match));
     771           13 :       break;
     772              :     case AST::MacroMatch::MacroMatchType::Tok:
     773              :       break;
     774              :     }
     775         2922 : }
     776              : 
     777              : void
     778         1770 : MacroExpander::match_repetition_skipped_metavars (
     779              :   AST::MacroMatchFragment &fragment)
     780              : {
     781         1770 :   auto &stack_map = sub_stack.peek ();
     782         1770 :   auto it = stack_map.find (fragment.get_ident ().as_string ());
     783              : 
     784         1770 :   if (it == stack_map.end ())
     785          114 :     sub_stack.insert_matches (fragment.get_ident ().as_string (),
     786          114 :                               MatchedFragmentContainer::zero ());
     787         1770 : }
     788              : 
     789              : void
     790         1738 : MacroExpander::match_repetition_skipped_metavars (
     791              :   AST::MacroMatchRepetition &rep)
     792              : {
     793         4654 :   for (auto &match : rep.get_matches ())
     794         2916 :     match_repetition_skipped_metavars (*match);
     795         1738 : }
     796              : 
     797              : void
     798           13 : MacroExpander::match_repetition_skipped_metavars (AST::MacroMatcher &rep)
     799              : {
     800           19 :   for (auto &match : rep.get_matches ())
     801            6 :     match_repetition_skipped_metavars (*match);
     802           13 : }
     803              : 
     804              : bool
     805         1726 : MacroExpander::match_repetition (Parser<MacroInvocLexer> &parser,
     806              :                                  AST::MacroMatchRepetition &rep)
     807              : {
     808         1726 :   size_t match_amount = 0;
     809         1726 :   bool res = false;
     810              : 
     811         1726 :   std::string lo_str;
     812         1726 :   std::string hi_str;
     813         1726 :   switch (rep.get_op ())
     814              :     {
     815         1565 :     case AST::MacroMatchRepetition::MacroRepOp::ANY:
     816         1565 :       lo_str = "0";
     817         1565 :       hi_str = "+inf";
     818         1565 :       res = match_n_matches (parser, rep, match_amount);
     819         1565 :       break;
     820          113 :     case AST::MacroMatchRepetition::MacroRepOp::ONE_OR_MORE:
     821          113 :       lo_str = "1";
     822          113 :       hi_str = "+inf";
     823          113 :       res = match_n_matches (parser, rep, match_amount, 1);
     824          113 :       break;
     825           48 :     case AST::MacroMatchRepetition::MacroRepOp::ZERO_OR_ONE:
     826           48 :       lo_str = "0";
     827           48 :       hi_str = "1";
     828           48 :       res = match_n_matches (parser, rep, match_amount, 0, 1);
     829           48 :       break;
     830            0 :     default:
     831            0 :       rust_unreachable ();
     832              :     }
     833              : 
     834         1731 :   rust_debug_loc (rep.get_match_locus (), "%s matched %lu times",
     835              :                   res ? "successfully" : "unsuccessfully",
     836              :                   (unsigned long) match_amount);
     837              : 
     838         1726 :   match_repetition_skipped_metavars (rep);
     839              : 
     840         1726 :   return res;
     841         1726 : }
     842              : 
     843              : /**
     844              :  * Helper function to refactor calling a parsing function 0 or more times
     845              :  */
     846              : static AST::Fragment
     847          737 : parse_many (Parser<MacroInvocLexer> &parser, TokenId delimiter,
     848              :             std::function<AST::SingleASTNode ()> parse_fn)
     849              : {
     850          737 :   auto &lexer = parser.get_token_source ();
     851          737 :   auto start = lexer.get_offs ();
     852              : 
     853          737 :   std::vector<AST::SingleASTNode> nodes;
     854         3136 :   while (true)
     855              :     {
     856         7746 :       if (parser.peek_current_token ()->get_id () == delimiter)
     857              :         break;
     858              : 
     859         3140 :       auto node = parse_fn ();
     860         3140 :       if (node.is_error ())
     861              :         {
     862            9 :           for (auto err : parser.get_errors ())
     863            5 :             err.emit ();
     864              : 
     865            4 :           return AST::Fragment::create_error ();
     866              :         }
     867              : 
     868         3136 :       nodes.emplace_back (std::move (node));
     869         3140 :     }
     870          733 :   auto end = lexer.get_offs ();
     871              : 
     872          733 :   return AST::Fragment (std::move (nodes), lexer.get_token_slice (start, end));
     873          737 : }
     874              : 
     875              : /**
     876              :  * Transcribe 0 or more items from a macro invocation
     877              :  *
     878              :  * @param parser Parser to extract items from
     879              :  * @param delimiter Id of the token on which parsing should stop
     880              :  */
     881              : static AST::Fragment
     882          337 : transcribe_many_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
     883              : {
     884          337 :   return parse_many (parser, delimiter, [&parser] () {
     885         2590 :     auto item = parser.parse_item (true);
     886         2590 :     if (!item)
     887            1 :       return AST::SingleASTNode (std::unique_ptr<AST::Item> (nullptr));
     888         2589 :     return AST::SingleASTNode (std::move (item.value ()));
     889          337 :   });
     890              : }
     891              : 
     892              : /**
     893              :  * Transcribe 0 or more external items from a macro invocation
     894              :  *
     895              :  * @param parser Parser to extract items from
     896              :  * @param delimiter Id of the token on which parsing should stop
     897              :  */
     898              : static AST::Fragment
     899            2 : transcribe_many_ext (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
     900              : {
     901            2 :   return parse_many (parser, delimiter, [&parser] () {
     902            3 :     auto item = parser.parse_external_item ();
     903            3 :     return AST::SingleASTNode (std::move (item));
     904            5 :   });
     905              : }
     906              : 
     907              : /**
     908              :  * Transcribe 0 or more trait items from a macro invocation
     909              :  *
     910              :  * @param parser Parser to extract items from
     911              :  * @param delimiter Id of the token on which parsing should stop
     912              :  */
     913              : static AST::Fragment
     914            1 : transcribe_many_trait_items (Parser<MacroInvocLexer> &parser,
     915              :                              TokenId &delimiter)
     916              : {
     917            1 :   return parse_many (parser, delimiter, [&parser] () {
     918            2 :     auto item = parser.parse_trait_item ();
     919            2 :     return AST::SingleASTNode (std::move (item));
     920            3 :   });
     921              : }
     922              : 
     923              : /**
     924              :  * Transcribe 0 or more impl items from a macro invocation
     925              :  *
     926              :  * @param parser Parser to extract items from
     927              :  * @param delimiter Id of the token on which parsing should stop
     928              :  */
     929              : static AST::Fragment
     930            1 : transcribe_many_impl_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
     931              : {
     932            1 :   return parse_many (parser, delimiter, [&parser] () {
     933            2 :     auto item = parser.parse_inherent_impl_item ();
     934            2 :     return AST::SingleASTNode (std::move (item));
     935            3 :   });
     936              : }
     937              : 
     938              : /**
     939              :  * Transcribe 0 or more trait impl items from a macro invocation
     940              :  *
     941              :  * @param parser Parser to extract items from
     942              :  * @param delimiter Id of the token on which parsing should stop
     943              :  */
     944              : static AST::Fragment
     945           31 : transcribe_many_trait_impl_items (Parser<MacroInvocLexer> &parser,
     946              :                                   TokenId &delimiter)
     947              : {
     948           31 :   return parse_many (parser, delimiter, [&parser] () {
     949          103 :     auto item = parser.parse_trait_impl_item ();
     950          103 :     return AST::SingleASTNode (std::move (item));
     951          134 :   });
     952              : }
     953              : 
     954              : /**
     955              :  * Transcribe 0 or more statements from a macro invocation
     956              :  *
     957              :  * @param parser Parser to extract statements from
     958              :  * @param delimiter Id of the token on which parsing should stop
     959              :  */
     960              : static AST::Fragment
     961          365 : transcribe_many_stmts (Parser<MacroInvocLexer> &parser, TokenId delimiter,
     962              :                        bool semicolon)
     963              : {
     964          365 :   auto restrictions = ParseRestrictions ();
     965          365 :   restrictions.allow_close_after_expr_stmt = true;
     966              : 
     967          365 :   return parse_many (parser, delimiter,
     968          365 :                      [&parser, restrictions, delimiter, semicolon] () {
     969          440 :                        auto stmt = parser.parse_stmt (restrictions);
     970          432 :                        if (semicolon && stmt
     971         1301 :                            && parser.peek_current_token ()->get_id ()
     972          429 :                                 == delimiter)
     973          335 :                          stmt->add_semicolon ();
     974              : 
     975          440 :                        return AST::SingleASTNode (std::move (stmt));
     976          805 :                      });
     977              : }
     978              : 
     979              : /**
     980              :  * Transcribe one expression from a macro invocation
     981              :  *
     982              :  * @param parser Parser to extract statements from
     983              :  */
     984              : static AST::Fragment
     985         1672 : transcribe_expression (Parser<MacroInvocLexer> &parser)
     986              : {
     987         1672 :   auto &lexer = parser.get_token_source ();
     988         1672 :   auto start = lexer.get_offs ();
     989              : 
     990         1672 :   auto attrs = parser.parse_outer_attributes ();
     991         1672 :   auto expr = parser.parse_expr (std::move (attrs));
     992         1677 :   for (auto error : parser.get_errors ())
     993            5 :     error.emit ();
     994         1672 :   if (!expr)
     995            1 :     return AST::Fragment::create_error ();
     996              : 
     997              :   // FIXME: make this an error for some edititons
     998         3342 :   if (parser.peek_current_token ()->get_id () == SEMICOLON)
     999              :     {
    1000            1 :       rust_warning_at (
    1001            1 :         parser.peek_current_token ()->get_locus (), 0,
    1002              :         "trailing semicolon in macro used in expression context");
    1003            1 :       parser.skip_token ();
    1004              :     }
    1005              : 
    1006         1671 :   auto end = lexer.get_offs ();
    1007              : 
    1008         3342 :   return AST::Fragment ({std::move (expr.value ())},
    1009         6684 :                         lexer.get_token_slice (start, end));
    1010         1672 : }
    1011              : 
    1012              : /**
    1013              :  * Transcribe one type from a macro invocation
    1014              :  *
    1015              :  * @param parser Parser to extract statements from
    1016              :  */
    1017              : static AST::Fragment
    1018           29 : transcribe_type (Parser<MacroInvocLexer> &parser)
    1019              : {
    1020           29 :   auto &lexer = parser.get_token_source ();
    1021           29 :   auto start = lexer.get_offs ();
    1022              : 
    1023           29 :   auto type = parser.parse_type (true);
    1024           29 :   for (auto err : parser.get_errors ())
    1025            0 :     err.emit ();
    1026           29 :   if (!type)
    1027            0 :     return AST::Fragment::create_error ();
    1028              : 
    1029           29 :   auto end = lexer.get_offs ();
    1030              : 
    1031           58 :   return AST::Fragment ({std::move (type)}, lexer.get_token_slice (start, end));
    1032           29 : }
    1033              : 
    1034              : /**
    1035              :  * Transcribe one pattern from a macro invocation
    1036              :  *
    1037              :  * @param parser Parser to extract statements from
    1038              :  */
    1039              : static AST::Fragment
    1040            3 : transcribe_pattern (Parser<MacroInvocLexer> &parser)
    1041              : {
    1042            3 :   auto &lexer = parser.get_token_source ();
    1043            3 :   auto start = lexer.get_offs ();
    1044              : 
    1045            3 :   auto pattern = parser.parse_pattern ();
    1046            6 :   for (auto err : parser.get_errors ())
    1047            3 :     err.emit ();
    1048              : 
    1049            3 :   if (!pattern)
    1050            2 :     return AST::Fragment::create_error ();
    1051              : 
    1052            1 :   auto end = lexer.get_offs ();
    1053              : 
    1054            2 :   return AST::Fragment ({std::move (pattern)},
    1055            3 :                         lexer.get_token_slice (start, end));
    1056            3 : }
    1057              : 
    1058              : static AST::Fragment
    1059         2441 : transcribe_context (MacroExpander::ContextType ctx,
    1060              :                     Parser<MacroInvocLexer> &parser, bool semicolon,
    1061              :                     AST::DelimType delimiter, TokenId last_token_id)
    1062              : {
    1063              :   // The flow-chart in order to choose a parsing function is as follows:
    1064              :   //
    1065              :   // [switch special context]
    1066              :   //     -- Item --> parser.parse_item();
    1067              :   //     -- Trait --> parser.parse_trait_item();
    1068              :   //     -- Impl --> parser.parse_impl_item();
    1069              :   //     -- Extern --> parser.parse_extern_item();
    1070              :   //     -- Pattern --> parser.parse_pattern();
    1071              :   //     -- None --> [has semicolon?]
    1072              :   //                 -- Yes --> parser.parse_stmt();
    1073              :   //                 -- No --> [switch invocation.delimiter()]
    1074              :   //                             -- { } --> parser.parse_stmt();
    1075              :   //                             -- _ --> parser.parse_expr(); // once!
    1076              : 
    1077              :   // If there is a semicolon OR we are expanding a MacroInvocationSemi, then
    1078              :   // we can parse multiple items. Otherwise, parse *one* expression
    1079              : 
    1080         2441 :   switch (ctx)
    1081              :     {
    1082          337 :     case MacroExpander::ContextType::ITEM:
    1083          337 :       return transcribe_many_items (parser, last_token_id);
    1084            1 :       break;
    1085            1 :     case MacroExpander::ContextType::TRAIT:
    1086            1 :       return transcribe_many_trait_items (parser, last_token_id);
    1087            1 :       break;
    1088            1 :     case MacroExpander::ContextType::IMPL:
    1089            1 :       return transcribe_many_impl_items (parser, last_token_id);
    1090           31 :       break;
    1091           31 :     case MacroExpander::ContextType::TRAIT_IMPL:
    1092           31 :       return transcribe_many_trait_impl_items (parser, last_token_id);
    1093            2 :       break;
    1094            2 :     case MacroExpander::ContextType::EXTERN:
    1095            2 :       return transcribe_many_ext (parser, last_token_id);
    1096           29 :       break;
    1097           29 :     case MacroExpander::ContextType::TYPE:
    1098           29 :       return transcribe_type (parser);
    1099            3 :     case MacroExpander::ContextType::PATTERN:
    1100            3 :       return transcribe_pattern (parser);
    1101          365 :       break;
    1102          365 :     case MacroExpander::ContextType::STMT:
    1103          365 :       return transcribe_many_stmts (parser, last_token_id, semicolon);
    1104         1672 :     case MacroExpander::ContextType::EXPR:
    1105         1672 :       return transcribe_expression (parser);
    1106            0 :     default:
    1107            0 :       rust_unreachable ();
    1108              :     }
    1109              : }
    1110              : 
    1111              : static std::string
    1112         2441 : tokens_to_str (std::vector<std::unique_ptr<AST::Token>> &tokens)
    1113              : {
    1114         2441 :   std::string str;
    1115         2441 :   if (!tokens.empty ())
    1116              :     {
    1117         4882 :       str += tokens[0]->as_string ();
    1118       133418 :       for (size_t i = 1; i < tokens.size (); i++)
    1119       261954 :         str += " " + tokens[i]->as_string ();
    1120              :     }
    1121              : 
    1122         2441 :   return str;
    1123              : }
    1124              : 
    1125              : AST::Fragment
    1126         2441 : MacroExpander::transcribe_rule (
    1127              :   AST::MacroRulesDefinition &definition, AST::MacroRule &match_rule,
    1128              :   AST::DelimTokenTree &invoc_token_tree,
    1129              :   std::map<std::string, MatchedFragmentContainer *> &matched_fragments,
    1130              :   AST::InvocKind invoc_kind, ContextType ctx)
    1131              : {
    1132         2441 :   bool semicolon = invoc_kind == AST::InvocKind::Semicoloned;
    1133              : 
    1134              :   // we can manipulate the token tree to substitute the dollar identifiers so
    1135              :   // that when we call parse its already substituted for us
    1136         2441 :   AST::MacroTranscriber &transcriber = match_rule.get_transcriber ();
    1137         2441 :   AST::DelimTokenTree &transcribe_tree = transcriber.get_token_tree ();
    1138              : 
    1139         2441 :   auto invoc_stream = invoc_token_tree.to_token_stream ();
    1140         2441 :   auto macro_rule_tokens = transcribe_tree.to_token_stream ();
    1141              : 
    1142         2441 :   auto substitute_context
    1143              :     = SubstituteCtx (invoc_stream, macro_rule_tokens, matched_fragments,
    1144         2441 :                      definition, invoc_token_tree.get_locus ());
    1145         2441 :   std::vector<std::unique_ptr<AST::Token>> substituted_tokens
    1146         2441 :     = substitute_context.substitute_tokens ();
    1147              : 
    1148         2441 :   rust_debug ("substituted tokens: %s",
    1149              :               tokens_to_str (substituted_tokens).c_str ());
    1150              : 
    1151              :   // parse it to an Fragment
    1152         2441 :   MacroInvocLexer lex (std::move (substituted_tokens));
    1153         2441 :   Parser<MacroInvocLexer> parser (lex);
    1154              : 
    1155         2441 :   auto last_token_id = TokenId::RIGHT_CURLY;
    1156              : 
    1157              :   // this is used so we can check that we delimit the stream correctly.
    1158         2441 :   switch (transcribe_tree.get_delim_type ())
    1159              :     {
    1160          132 :     case AST::DelimType::PARENS:
    1161          132 :       last_token_id = TokenId::RIGHT_PAREN;
    1162          132 :       rust_assert (parser.skip_token (LEFT_PAREN));
    1163              :       break;
    1164              : 
    1165         2309 :     case AST::DelimType::CURLY:
    1166         2309 :       rust_assert (parser.skip_token (LEFT_CURLY));
    1167              :       break;
    1168              : 
    1169            0 :     case AST::DelimType::SQUARE:
    1170            0 :       last_token_id = TokenId::RIGHT_SQUARE;
    1171            0 :       rust_assert (parser.skip_token (LEFT_SQUARE));
    1172              :       break;
    1173              :     }
    1174              : 
    1175              :   // see https://github.com/Rust-GCC/gccrs/issues/22
    1176              :   // TL;DR:
    1177              :   //   - Treat all macro invocations with parentheses, (), or square brackets,
    1178              :   //   [], as expressions.
    1179              :   //   - If the macro invocation has curly brackets, {}, it may be parsed as a
    1180              :   //   statement depending on the context.
    1181              :   //   - If the macro invocation has a semicolon at the end, it must be parsed
    1182              :   //   as a statement (either via ExpressionStatement or
    1183              :   //   MacroInvocationWithSemi)
    1184              : 
    1185         2441 :   auto fragment
    1186              :     = transcribe_context (ctx, parser, semicolon,
    1187         2441 :                           invoc_token_tree.get_delim_type (), last_token_id);
    1188              : 
    1189              :   // emit any errors
    1190         2441 :   if (parser.has_errors ())
    1191           11 :     return AST::Fragment::create_error ();
    1192              : 
    1193              :   // are all the tokens used?
    1194         2430 :   bool did_delimit = parser.skip_token (last_token_id);
    1195              : 
    1196         2430 :   bool reached_end_of_stream = did_delimit && parser.skip_token (END_OF_FILE);
    1197            0 :   if (!reached_end_of_stream)
    1198              :     {
    1199              :       // FIXME: rustc has some cases it accepts this with a warning due to
    1200              :       // backwards compatibility.
    1201            0 :       const_TokenPtr current_token = parser.peek_current_token ();
    1202            0 :       rust_error_at (current_token->get_locus (),
    1203              :                      "tokens here and after are unparsed");
    1204            0 :     }
    1205              : 
    1206         2430 :   return fragment;
    1207         2441 : }
    1208              : 
    1209              : AST::Fragment
    1210            0 : MacroExpander::parse_proc_macro_output (ProcMacro::TokenStream ts)
    1211              : {
    1212            0 :   MacroInvocLexer lex (convert (ts));
    1213            0 :   Parser<MacroInvocLexer> parser (lex);
    1214              : 
    1215            0 :   std::vector<AST::SingleASTNode> nodes;
    1216            0 :   switch (peek_context ())
    1217              :     {
    1218              :     case ContextType::ITEM:
    1219            0 :       while (lex.peek_token ()->get_id () != END_OF_FILE)
    1220              :         {
    1221            0 :           auto result = parser.parse_item (false);
    1222            0 :           if (!result)
    1223              :             break;
    1224            0 :           nodes.emplace_back (std::move (result.value ()));
    1225            0 :         }
    1226              :       break;
    1227              :     case ContextType::STMT:
    1228            0 :       while (lex.peek_token ()->get_id () != END_OF_FILE)
    1229              :         {
    1230            0 :           auto result = parser.parse_stmt ();
    1231            0 :           if (result == nullptr)
    1232              :             break;
    1233            0 :           nodes.emplace_back (std::move (result));
    1234            0 :         }
    1235              :       break;
    1236            0 :     case ContextType::TRAIT:
    1237            0 :     case ContextType::IMPL:
    1238            0 :     case ContextType::TRAIT_IMPL:
    1239            0 :     case ContextType::EXTERN:
    1240            0 :     case ContextType::TYPE:
    1241            0 :     case ContextType::EXPR:
    1242            0 :     default:
    1243            0 :       rust_unreachable ();
    1244              :     }
    1245              : 
    1246            0 :   if (parser.has_errors ())
    1247            0 :     return AST::Fragment::create_error ();
    1248              :   else
    1249            0 :     return {nodes, std::vector<std::unique_ptr<AST::Token>> ()};
    1250            0 : }
    1251              : 
    1252              : MatchedFragment &
    1253        11369 : MatchedFragmentContainer::get_single_fragment ()
    1254              : {
    1255        11369 :   rust_assert (is_single_fragment ());
    1256              : 
    1257        11369 :   return static_cast<MatchedFragmentContainerMetaVar &> (*this).get_fragment ();
    1258              : }
    1259              : 
    1260              : std::vector<std::unique_ptr<MatchedFragmentContainer>> &
    1261         8864 : MatchedFragmentContainer::get_fragments ()
    1262              : {
    1263         8864 :   rust_assert (!is_single_fragment ());
    1264              : 
    1265         8864 :   return static_cast<MatchedFragmentContainerRepetition &> (*this)
    1266         8864 :     .get_fragments ();
    1267              : }
    1268              : 
    1269              : void
    1270            0 : MatchedFragmentContainer::add_fragment (MatchedFragment fragment)
    1271              : {
    1272            0 :   rust_assert (!is_single_fragment ());
    1273              : 
    1274            0 :   return static_cast<MatchedFragmentContainerRepetition &> (*this)
    1275            0 :     .add_fragment (fragment);
    1276              : }
    1277              : 
    1278              : void
    1279         4441 : MatchedFragmentContainer::add_fragment (
    1280              :   std::unique_ptr<MatchedFragmentContainer> fragment)
    1281              : {
    1282         4441 :   rust_assert (!is_single_fragment ());
    1283              : 
    1284         4441 :   return static_cast<MatchedFragmentContainerRepetition &> (*this)
    1285         4441 :     .add_fragment (std::move (fragment));
    1286              : }
    1287              : 
    1288              : std::unique_ptr<MatchedFragmentContainer>
    1289           57 : MatchedFragmentContainer::zero ()
    1290              : {
    1291           57 :   return std::unique_ptr<MatchedFragmentContainer> (
    1292           57 :     new MatchedFragmentContainerRepetition ());
    1293              : }
    1294              : 
    1295              : std::unique_ptr<MatchedFragmentContainer>
    1296         8157 : MatchedFragmentContainer::metavar (MatchedFragment fragment)
    1297              : {
    1298         8157 :   return std::unique_ptr<MatchedFragmentContainer> (
    1299         8157 :     new MatchedFragmentContainerMetaVar (fragment));
    1300              : }
    1301              : 
    1302              : } // namespace Rust
        

Generated by: LCOV version 2.4-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.