LCOV - code coverage report
Current view: top level - gcc/rust/expand - rust-macro-expand.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 87.2 % 585 510
Test Date: 2026-02-28 14:20:25 Functions: 92.7 % 41 38
Legend: Lines:     hit not hit

            Line data    Source code
       1              : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
       2              : 
       3              : // This file is part of GCC.
       4              : 
       5              : // GCC is free software; you can redistribute it and/or modify it under
       6              : // the terms of the GNU General Public License as published by the Free
       7              : // Software Foundation; either version 3, or (at your option) any later
       8              : // version.
       9              : 
      10              : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      11              : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
      12              : // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      13              : // for more details.
      14              : 
      15              : // You should have received a copy of the GNU General Public License
      16              : // along with GCC; see the file COPYING3.  If not see
      17              : // <http://www.gnu.org/licenses/>.
      18              : 
      19              : #include "rust-macro-expand.h"
      20              : #include "optional.h"
      21              : #include "rust-ast-fragment.h"
      22              : #include "rust-macro-builtins.h"
      23              : #include "rust-macro-substitute-ctx.h"
      24              : #include "rust-ast-full.h"
      25              : #include "rust-ast-visitor.h"
      26              : #include "rust-diagnostics.h"
      27              : #include "rust-macro.h"
      28              : #include "rust-parse.h"
      29              : #include "rust-cfg-strip.h"
      30              : #include "rust-proc-macro.h"
      31              : #include "rust-token-tree-desugar.h"
      32              : 
      33              : namespace Rust {
      34              : 
      35              : AST::Fragment
      36         2444 : MacroExpander::expand_decl_macro (location_t invoc_locus,
      37              :                                   AST::MacroInvocData &invoc,
      38              :                                   AST::MacroRulesDefinition &rules_def,
      39              :                                   AST::InvocKind semicolon)
      40              : {
      41              :   // ensure that both invocation and rules are in a valid state
      42         2444 :   rust_assert (!invoc.is_marked_for_strip ());
      43         2444 :   rust_assert (!rules_def.is_marked_for_strip ());
      44         2444 :   rust_assert (rules_def.get_macro_rules ().size () > 0);
      45              : 
      46              :   /* probably something here about parsing invoc and rules def token trees to
      47              :    * token stream. if not, how would parser handle the captures of exprs and
      48              :    * stuff? on the other hand, token trees may be kind of useful in rules def
      49              :    * as creating a point where recursion can occur (like having
      50              :    * "compare_macro_match" and then it calling itself when it finds
      51              :    * delimiters)
      52              :    */
      53              : 
      54              :   /* find matching rule to invoc token tree, based on macro rule's matcher. if
      55              :    * none exist, error.
      56              :    * - specifically, check each matcher in order. if one fails to match, move
      57              :    * onto next. */
      58              :   /* TODO: does doing this require parsing expressions and whatever in the
      59              :    * invoc? if so, might as well save the results if referenced using $ or
      60              :    * whatever. If not, do another pass saving them. Except this is probably
      61              :    * useless as different rules could have different starting points for exprs
      62              :    * or whatever. Decision trees could avoid this, but they have their own
      63              :    * issues. */
      64              :   /* TODO: will need to modify the parser so that it can essentially "catch"
      65              :    * errors - maybe "try_parse_expr" or whatever methods. */
      66              :   // this technically creates a back-tracking parser - this will be the
      67              :   // implementation style
      68              : 
      69              :   /* then, after results are saved, generate the macro output from the
      70              :    * transcriber token tree. if i understand this correctly, the macro
      71              :    * invocation gets replaced by the transcriber tokens, except with
      72              :    * substitutions made (e.g. for $i variables) */
      73              : 
      74              :   /* TODO: it is probably better to modify AST::Token to store a pointer to a
      75              :    * Lexer::Token (rather than being converted) - i.e. not so much have
      76              :    * AST::Token as a Token but rather a TokenContainer (as it is another type
      77              :    * of TokenTree). This will prevent re-conversion of Tokens between each
      78              :    * type all the time, while still allowing the heterogenous storage of token
      79              :    * trees.
      80              :    */
      81              : 
      82         2444 :   AST::DelimTokenTree &invoc_token_tree_sugar = invoc.get_delim_tok_tree ();
      83              : 
      84              :   // We must first desugar doc comments into proper attributes
      85         2444 :   auto invoc_token_tree = AST::TokenTreeDesugar ().go (invoc_token_tree_sugar);
      86              : 
      87              :   // find matching arm
      88         2444 :   AST::MacroRule *matched_rule = nullptr;
      89         2444 :   std::map<std::string, std::unique_ptr<MatchedFragmentContainer>>
      90         2444 :     matched_fragments;
      91         3840 :   for (auto &rule : rules_def.get_rules ())
      92              :     {
      93         3832 :       sub_stack.push ();
      94         3832 :       bool did_match_rule = try_match_rule (rule, invoc_token_tree);
      95         7664 :       matched_fragments = sub_stack.pop ();
      96              : 
      97         3832 :       if (did_match_rule)
      98              :         {
      99              :           //  // Debugging
     100              :           //  for (auto &kv : matched_fragments)
     101              :           //    rust_debug ("[fragment]: %s (%ld - %s)", kv.first.c_str (),
     102              :           //            kv.second.get_fragments ().size (),
     103              :           //            kv.second.get_kind ()
     104              :           //                == MatchedFragmentContainer::Kind::Repetition
     105              :           //              ? "repetition"
     106              :           //              : "metavar");
     107              : 
     108              :           matched_rule = &rule;
     109              :           break;
     110              :         }
     111              :     }
     112              : 
     113         2444 :   if (matched_rule == nullptr)
     114              :     {
     115            8 :       if (!had_duplicate_error)
     116              :         {
     117            7 :           rich_location r (line_table, invoc_locus);
     118            7 :           r.add_range (rules_def.get_locus ());
     119            7 :           rust_error_at (r, "Failed to match any rule within macro");
     120            7 :         }
     121            8 :       had_duplicate_error = false;
     122            8 :       return AST::Fragment::create_error ();
     123              :     }
     124              : 
     125         2436 :   std::map<std::string, MatchedFragmentContainer *> matched_fragments_ptr;
     126              : 
     127         6464 :   for (auto &ent : matched_fragments)
     128         4028 :     matched_fragments_ptr.emplace (ent.first, ent.second.get ());
     129              : 
     130         2436 :   return transcribe_rule (rules_def, *matched_rule, invoc_token_tree,
     131         2436 :                           matched_fragments_ptr, semicolon, peek_context ());
     132         4880 : }
     133              : 
     134              : void
     135           43 : MacroExpander::expand_eager_invocations (AST::MacroInvocation &invoc)
     136              : {
     137           43 :   if (invoc.get_pending_eager_invocations ().empty ())
     138            0 :     return;
     139              : 
     140              :   // We have to basically create a new delimited token tree which contains the
     141              :   // result of one step of expansion. In the case of builtin macros called with
     142              :   // other macro invocations, such as `concat!("h", 'a', a!())`, we need to
     143              :   // expand `a!()` before expanding the concat macro.
     144              :   // This will, ideally, give us a new token tree containing the various
     145              :   // existing tokens + the result of the expansion of a!().
     146              :   // To do this, we "parse" the given token tree to find anything that "looks
     147              :   // like a macro invocation". Then, we get the corresponding macro invocation
     148              :   // from the `pending_eager_invocations` vector and expand it.
     149              :   // Because the `pending_eager_invocations` vector is created in the same order
     150              :   // that the DelimTokenTree is parsed, we know that the first macro invocation
     151              :   // within the DelimTokenTree corresponds to the first element in
     152              :   // `pending_eager_invocations`. The idea is thus to:
     153              :   // 1. Find a macro invocation in the token tree, noting the index of the start
     154              :   //    token and of the end token
     155              :   // 2. Get its associated invocation in `pending_eager_invocations`
     156              :   // 3. Expand that element
     157              :   // 4. Get the token tree associated with that AST fragment
     158              :   // 5. Replace the original tokens corresponding to the invocation with the new
     159              :   //    tokens from the fragment
     160              :   // pseudo-code:
     161              :   //
     162              :   // i = 0;
     163              :   // for tok in dtt:
     164              :   //   if tok is identifier && tok->next() is !:
     165              :   //     start = index(tok);
     166              :   //     l_delim = tok->next()->next();
     167              :   //     tok = skip_until_r_delim();
     168              :   //     end = index(tok);
     169              :   //
     170              :   //     new_tt = expand_eager_invoc(eagers[i++]);
     171              :   //     old_tt[start..end] = new_tt;
     172              : 
     173           43 :   auto dtt = invoc.get_invoc_data ().get_delim_tok_tree ();
     174           43 :   auto stream = dtt.to_token_stream ();
     175           43 :   std::vector<std::unique_ptr<AST::TokenTree>> new_stream;
     176           43 :   size_t current_pending = 0;
     177              : 
     178              :   // we need to create a clone of the delimited token tree as the lexer
     179              :   // expects ownership of the tokens
     180           43 :   std::vector<const_TokenPtr> dtt_clone;
     181          361 :   for (auto &tok : stream)
     182          636 :     dtt_clone.emplace_back (tok->get_tok_ptr ());
     183              : 
     184           43 :   MacroInvocLexer lex (std::move (dtt_clone));
     185           43 :   Parser<MacroInvocLexer> parser (lex);
     186              : 
     187              :   // we want to build a substitution map - basically, associating a `start` and
     188              :   // `end` index for each of the pending macro invocations
     189           43 :   std::map<std::pair<size_t, size_t>, std::unique_ptr<AST::MacroInvocation> &>
     190           43 :     substitution_map;
     191              : 
     192          361 :   for (size_t i = 0; i < stream.size (); i++)
     193              :     {
     194              :       // FIXME: Can't these offsets be figure out when we actually parse the
     195              :       // pending_eager_invocation in the first place?
     196          318 :       auto invocation = parser.parse_macro_invocation ({});
     197              : 
     198              :       // if we've managed to parse a macro invocation, we look at the current
     199              :       // offset and store them in the substitution map. Otherwise, we skip one
     200              :       // token and try parsing again
     201          318 :       if (invocation)
     202           46 :         substitution_map.insert (
     203           46 :           {{i, parser.get_token_source ().get_offs ()},
     204           46 :            invoc.get_pending_eager_invocations ()[current_pending++]});
     205              :       else
     206          272 :         parser.skip_token (stream[i]->get_id ());
     207          318 :     }
     208              : 
     209           43 :   size_t current_idx = 0;
     210           89 :   for (auto kv : substitution_map)
     211              :     {
     212           46 :       auto &to_expand = kv.second;
     213           46 :       expand_invoc (*to_expand, AST::InvocKind::Expr);
     214              : 
     215           46 :       auto fragment = take_expanded_fragment ();
     216           46 :       auto &new_tokens = fragment.get_tokens ();
     217              : 
     218           46 :       auto start = kv.first.first;
     219           46 :       auto end = kv.first.second;
     220              : 
     221              :       // We're now going to re-add the tokens to the invocation's token tree.
     222              :       // 1. Basically, what we want to do is insert all tokens up until the
     223              :       //    beginning of the macro invocation (start).
     224              :       // 2. Then, we'll insert all of the tokens resulting from the macro
     225              :       //    expansion: These are in `new_tokens`.
     226              :       // 3. Finally, we'll do that again from
     227              :       //    the end of macro and go back to 1.
     228              : 
     229           98 :       for (size_t i = current_idx; i < start; i++)
     230           52 :         new_stream.emplace_back (stream[i]->clone_token ());
     231              : 
     232           92 :       for (auto &tok : new_tokens)
     233           46 :         new_stream.emplace_back (tok->clone_token ());
     234              : 
     235           46 :       current_idx = end;
     236           46 :     }
     237              : 
     238              :   // Once all of that is done, we copy the last remaining tokens from the
     239              :   // original stream
     240          113 :   for (size_t i = current_idx; i < stream.size (); i++)
     241           70 :     new_stream.emplace_back (stream[i]->clone_token ());
     242              : 
     243           43 :   auto new_dtt
     244           43 :     = AST::DelimTokenTree (dtt.get_delim_type (), std::move (new_stream));
     245              : 
     246           43 :   invoc.get_pending_eager_invocations ().clear ();
     247           43 :   invoc.get_invoc_data ().set_delim_tok_tree (new_dtt);
     248           86 : }
     249              : 
     250              : void
     251         2839 : MacroExpander::expand_invoc (AST::MacroInvocation &invoc,
     252              :                              AST::InvocKind semicolon)
     253              : {
     254         2839 :   if (depth_exceeds_recursion_limit ())
     255              :     {
     256            0 :       rust_error_at (invoc.get_locus (), "reached recursion limit");
     257           39 :       return;
     258              :     }
     259              : 
     260         2839 :   if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin)
     261              :     {
     262              :       // Eager expansions are always expressions
     263           43 :       push_context (ContextType::EXPR);
     264           43 :       expand_eager_invocations (invoc);
     265           43 :       pop_context ();
     266              :     }
     267              : 
     268         2839 :   AST::MacroInvocData &invoc_data = invoc.get_invoc_data ();
     269              : 
     270              :   // ??
     271              :   // switch on type of macro:
     272              :   //  - '!' syntax macro (inner switch)
     273              :   //      - procedural macro - "A token-based function-like macro"
     274              :   //      - 'macro_rules' (by example/pattern-match) macro? or not? "an
     275              :   // AST-based function-like macro"
     276              :   //      - else is unreachable
     277              :   //  - attribute syntax macro (inner switch)
     278              :   //  - procedural macro attribute syntax - "A token-based attribute
     279              :   // macro"
     280              :   //      - legacy macro attribute syntax? - "an AST-based attribute macro"
     281              :   //      - non-macro attribute: mark known
     282              :   //      - else is unreachable
     283              :   //  - derive macro (inner switch)
     284              :   //      - derive or legacy derive - "token-based" vs "AST-based"
     285              :   //      - else is unreachable
     286              :   //  - derive container macro - unreachable
     287              : 
     288         2839 :   auto fragment = AST::Fragment::create_error ();
     289         2839 :   invoc_data.set_expander (this);
     290              : 
     291              :   // lookup the rules
     292         2839 :   auto rules_def = mappings.lookup_macro_invocation (invoc);
     293              : 
     294              :   // We special case the `offset_of!()` macro if the flag is here and manually
     295              :   // resolve to the builtin transcriber we have specified
     296         2839 :   auto assume_builtin_offset_of
     297         2839 :     = flag_assume_builtin_offset_of
     298           36 :       && (invoc.get_invoc_data ().get_path ().as_string () == "offset_of")
     299         2857 :       && !rules_def;
     300              : 
     301              :   // TODO: This is *massive hack* which should be removed as we progress to
     302              :   // Rust 1.71 when offset_of gets added to core
     303         2839 :   if (assume_builtin_offset_of)
     304              :     {
     305           18 :       fragment = MacroBuiltin::offset_of_handler (invoc.get_locus (),
     306              :                                                   invoc_data, semicolon)
     307           54 :                    .value_or (AST::Fragment::create_empty ());
     308              : 
     309           18 :       set_expanded_fragment (std::move (fragment));
     310              : 
     311           18 :       return;
     312              :     }
     313              : 
     314              :   // If there's no rule associated with the invocation, we can simply return
     315              :   // early. The early name resolver will have already emitted an error.
     316         2821 :   if (!rules_def)
     317              :     return;
     318              : 
     319         2800 :   auto rdef = rules_def.value ();
     320              : 
     321              :   // We store the last expanded invocation and macro definition for error
     322              :   // reporting in case the recursion limit is reached
     323         2800 :   last_invoc = *invoc.clone_macro_invocation_impl ();
     324         2800 :   last_def = *rdef;
     325              : 
     326         2800 :   if (rdef->is_builtin ())
     327          356 :     fragment = rdef
     328          356 :                  ->get_builtin_transcriber () (invoc.get_locus (), invoc_data,
     329              :                                                semicolon)
     330         1068 :                  .value_or (AST::Fragment::create_empty ());
     331              :   else
     332         2444 :     fragment
     333         4888 :       = expand_decl_macro (invoc.get_locus (), invoc_data, *rdef, semicolon);
     334              :   // fix: if the expansion is failing, we must replace the marco with an empty
     335              :   // error or node
     336              :   // makes sure that it doesn't panic on Rouge macro (it -> Lowering Phase)
     337              :   // added the parsing errors in gcc/testsuite/rust/compile/issue-4213.rs
     338         2800 :   if (fragment.is_error ())
     339              :     {
     340          112 :       fragment = AST::Fragment::create_empty ();
     341              :     }
     342         2800 :   set_expanded_fragment (std::move (fragment));
     343         2839 : }
     344              : 
     345              : void
     346            0 : MacroExpander::expand_crate ()
     347              : {
     348            0 :   NodeId scope_node_id = crate.get_node_id ();
     349            0 :   resolver->get_macro_scope ().push (scope_node_id);
     350              : 
     351              :   /* fill macro/decorator map from init list? not sure where init list comes
     352              :    * from? */
     353              : 
     354              :   // TODO: does cfg apply for inner attributes? research.
     355              :   // the apparent answer (from playground test) is yes
     356              : 
     357            0 :   push_context (ContextType::ITEM);
     358              : 
     359              :   // expand attributes recursively and strip items if required
     360              :   //  AttrVisitor attr_visitor (*this);
     361            0 :   auto &items = crate.items;
     362            0 :   for (auto it = items.begin (); it != items.end ();)
     363              :     {
     364            0 :       auto &item = *it;
     365              : 
     366            0 :       auto fragment = take_expanded_fragment ();
     367            0 :       if (fragment.should_expand ())
     368              :         {
     369              :           // Remove the current expanded invocation
     370            0 :           it = items.erase (it);
     371            0 :           for (auto &node : fragment.get_nodes ())
     372              :             {
     373            0 :               it = items.insert (it, node.take_item ());
     374            0 :               it++;
     375              :             }
     376              :         }
     377            0 :       else if (item->is_marked_for_strip ())
     378            0 :         it = items.erase (it);
     379              :       else
     380            0 :         it++;
     381            0 :     }
     382              : 
     383            0 :   pop_context ();
     384              : 
     385              :   // TODO: should recursive attribute and macro expansion be done in the same
     386              :   // transversal? Or in separate ones like currently?
     387              : 
     388              :   // expand module tree recursively
     389              : 
     390              :   // post-process
     391              : 
     392              :   // extract exported macros?
     393            0 : }
     394              : 
     395              : bool
     396         6766 : MacroExpander::depth_exceeds_recursion_limit () const
     397              : {
     398         6766 :   return expansion_depth >= cfg.recursion_limit;
     399              : }
     400              : 
     401              : bool
     402         3832 : MacroExpander::try_match_rule (AST::MacroRule &match_rule,
     403              :                                AST::DelimTokenTree &invoc_token_tree)
     404              : {
     405         3832 :   MacroInvocLexer lex (invoc_token_tree.to_token_stream ());
     406         3832 :   Parser<MacroInvocLexer> parser (lex);
     407              : 
     408         3832 :   AST::MacroMatcher &matcher = match_rule.get_matcher ();
     409              : 
     410         3832 :   expansion_depth++;
     411         3832 :   if (!match_matcher (parser, matcher, false, false))
     412              :     {
     413         1396 :       expansion_depth--;
     414         1396 :       return false;
     415              :     }
     416         2436 :   expansion_depth--;
     417              : 
     418         2436 :   bool used_all_input_tokens = parser.skip_token (END_OF_FILE);
     419         2436 :   return used_all_input_tokens;
     420         3832 : }
     421              : 
     422              : bool
     423         8160 : MacroExpander::match_fragment (Parser<MacroInvocLexer> &parser,
     424              :                                AST::MacroMatchFragment &fragment)
     425              : {
     426         8160 :   switch (fragment.get_frag_spec ().get_kind ())
     427              :     {
     428         1179 :     case AST::MacroFragSpec::EXPR:
     429         2357 :       parser.parse_expr ();
     430         1179 :       break;
     431              : 
     432            2 :     case AST::MacroFragSpec::BLOCK:
     433            4 :       parser.parse_block_expr ();
     434            2 :       break;
     435              : 
     436          479 :     case AST::MacroFragSpec::IDENT:
     437          479 :       parser.parse_identifier_or_keyword_token ();
     438          479 :       break;
     439              : 
     440         3474 :     case AST::MacroFragSpec::LITERAL:
     441         6945 :       std::ignore = parser.parse_literal_expr ();
     442         3474 :       break;
     443              : 
     444            1 :     case AST::MacroFragSpec::ITEM:
     445            1 :       parser.parse_item (false);
     446            1 :       break;
     447              : 
     448         2380 :     case AST::MacroFragSpec::TY:
     449         2380 :       parser.parse_type ();
     450         2380 :       break;
     451              : 
     452           17 :     case AST::MacroFragSpec::PAT:
     453           17 :       parser.parse_pattern ();
     454           17 :       break;
     455              : 
     456            0 :     case AST::MacroFragSpec::PATH:
     457            0 :       parser.parse_path_in_expression ();
     458            0 :       break;
     459              : 
     460            0 :     case AST::MacroFragSpec::VIS:
     461            0 :       parser.parse_visibility ();
     462            0 :       break;
     463              : 
     464          301 :     case AST::MacroFragSpec::STMT:
     465          301 :       {
     466          301 :         auto restrictions = ParseRestrictions ();
     467          301 :         restrictions.consume_semi = false;
     468          301 :         parser.parse_stmt (restrictions);
     469          301 :         break;
     470              :       }
     471              : 
     472            4 :     case AST::MacroFragSpec::LIFETIME:
     473            4 :       parser.parse_lifetime_params ();
     474            4 :       break;
     475              : 
     476              :       // is meta attributes?
     477           46 :     case AST::MacroFragSpec::META:
     478           46 :       parser.parse_attribute_body ();
     479           46 :       break;
     480              : 
     481          277 :     case AST::MacroFragSpec::TT:
     482          277 :       parser.parse_token_tree ();
     483          277 :       break;
     484              : 
     485              :       // i guess we just ignore invalid and just error out
     486              :     case AST::MacroFragSpec::INVALID:
     487              :       return false;
     488              :     }
     489              : 
     490              :   // it matches if the parser did not produce errors trying to parse that type
     491              :   // of item
     492         8160 :   return !parser.has_errors ();
     493              : }
     494              : 
     495              : bool
     496         3927 : MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
     497              :                               AST::MacroMatcher &matcher, bool in_repetition,
     498              :                               bool match_delim)
     499              : {
     500         3927 :   if (depth_exceeds_recursion_limit ())
     501              :     {
     502            0 :       rust_error_at (matcher.get_match_locus (), "reached recursion limit");
     503            0 :       return false;
     504              :     }
     505              : 
     506         3927 :   auto delimiter = parser.peek_current_token ();
     507              : 
     508         7850 :   auto check_delim = [&matcher, match_delim] (AST::DelimType delim) {
     509           91 :     return !match_delim || matcher.get_delim_type () == delim;
     510         3927 :   };
     511              : 
     512              :   // this is used so we can check that we delimit the stream correctly.
     513         3927 :   switch (delimiter->get_id ())
     514              :     {
     515         3621 :     case LEFT_PAREN:
     516         3621 :       {
     517         3621 :         if (!check_delim (AST::DelimType::PARENS))
     518              :           return false;
     519              :       }
     520              :       break;
     521              : 
     522           46 :     case LEFT_SQUARE:
     523           46 :       {
     524           46 :         if (!check_delim (AST::DelimType::SQUARE))
     525              :           return false;
     526              :       }
     527              :       break;
     528              : 
     529          256 :     case LEFT_CURLY:
     530          256 :       {
     531         4183 :         if (!check_delim (AST::DelimType::CURLY))
     532              :           return false;
     533              :       }
     534              :       break;
     535              :     default:
     536              :       return false;
     537              :     }
     538         3922 :   parser.skip_token ();
     539              : 
     540         3922 :   const MacroInvocLexer &source = parser.get_token_source ();
     541              : 
     542         7844 :   std::unordered_map<std::string, location_t> duplicate_check;
     543              : 
     544        10451 :   for (auto &match : matcher.get_matches ())
     545              :     {
     546         6574 :       size_t offs_begin = source.get_offs ();
     547              : 
     548         6574 :       switch (match->get_macro_match_type ())
     549              :         {
     550         3738 :         case AST::MacroMatch::MacroMatchType::Fragment:
     551         3738 :           {
     552         3738 :             AST::MacroMatchFragment *fragment
     553         3738 :               = static_cast<AST::MacroMatchFragment *> (match.get ());
     554         3738 :             if (!match_fragment (parser, *fragment))
     555            3 :               return false;
     556              : 
     557         7472 :             auto duplicate_result = duplicate_check.insert (
     558        11208 :               std::make_pair (fragment->get_ident ().as_string (),
     559         3736 :                               fragment->get_ident ().get_locus ()));
     560              : 
     561         3736 :             if (!duplicate_result.second)
     562              :               {
     563              :                 // TODO: add range labels?
     564            1 :                 rich_location r (line_table,
     565            1 :                                  fragment->get_ident ().get_locus ());
     566            1 :                 r.add_range (duplicate_result.first->second);
     567            1 :                 rust_error_at (r, "duplicate matcher binding");
     568            1 :                 had_duplicate_error = true;
     569            1 :                 return false;
     570            1 :               }
     571              : 
     572              :             // matched fragment get the offset in the token stream
     573         3735 :             size_t offs_end = source.get_offs ();
     574         7470 :             sub_stack.insert_metavar (
     575         7470 :               MatchedFragment (fragment->get_ident ().as_string (), offs_begin,
     576        11205 :                                offs_end));
     577              :           }
     578         3735 :           break;
     579              : 
     580         1067 :         case AST::MacroMatch::MacroMatchType::Tok:
     581         1067 :           {
     582         1067 :             AST::Token *tok = static_cast<AST::Token *> (match.get ());
     583         1067 :             if (!match_token (parser, *tok))
     584              :               return false;
     585              :           }
     586              :           break;
     587              : 
     588         1707 :         case AST::MacroMatch::MacroMatchType::Repetition:
     589         1707 :           {
     590         1707 :             AST::MacroMatchRepetition *rep
     591         1707 :               = static_cast<AST::MacroMatchRepetition *> (match.get ());
     592         1707 :             if (!match_repetition (parser, *rep))
     593              :               return false;
     594              :           }
     595              :           break;
     596              : 
     597           62 :         case AST::MacroMatch::MacroMatchType::Matcher:
     598           62 :           {
     599           62 :             AST::MacroMatcher *m
     600           62 :               = static_cast<AST::MacroMatcher *> (match.get ());
     601           62 :             expansion_depth++;
     602           62 :             if (!match_matcher (parser, *m, in_repetition))
     603              :               {
     604            3 :                 expansion_depth--;
     605            3 :                 return false;
     606              :               }
     607           59 :             expansion_depth--;
     608              :           }
     609           59 :           break;
     610              :         }
     611              :     }
     612              : 
     613         3877 :   switch (delimiter->get_id ())
     614              :     {
     615         3576 :     case LEFT_PAREN:
     616         3576 :       {
     617         3576 :         if (!parser.skip_token (RIGHT_PAREN))
     618              :           return false;
     619              :       }
     620              :       break;
     621              : 
     622           46 :     case LEFT_SQUARE:
     623           46 :       {
     624           46 :         if (!parser.skip_token (RIGHT_SQUARE))
     625              :           return false;
     626              :       }
     627              :       break;
     628              : 
     629          255 :     case LEFT_CURLY:
     630          255 :       {
     631          255 :         if (!parser.skip_token (RIGHT_CURLY))
     632              :           return false;
     633              :       }
     634              :       break;
     635            0 :     default:
     636            0 :       rust_unreachable ();
     637              :     }
     638              : 
     639              :   return true;
     640         3927 : }
     641              : 
     642              : bool
     643         2692 : MacroExpander::match_token (Parser<MacroInvocLexer> &parser, AST::Token &token)
     644              : {
     645         5384 :   return parser.skip_token (token.get_tok_ptr ());
     646              : }
     647              : 
     648              : bool
     649         1721 : MacroExpander::match_n_matches (Parser<MacroInvocLexer> &parser,
     650              :                                 AST::MacroMatchRepetition &rep,
     651              :                                 size_t &match_amount, size_t lo_bound,
     652              :                                 size_t hi_bound)
     653              : {
     654         1721 :   match_amount = 0;
     655         1721 :   auto &matches = rep.get_matches ();
     656              : 
     657         1721 :   const MacroInvocLexer &source = parser.get_token_source ();
     658        10145 :   while (true)
     659              :     {
     660              :       // If the current token is a closing macro delimiter, break away.
     661              :       // TODO: Is this correct?
     662         5933 :       auto t_id = parser.peek_current_token ()->get_id ();
     663         5933 :       if (t_id == RIGHT_PAREN || t_id == RIGHT_SQUARE || t_id == RIGHT_CURLY)
     664              :         break;
     665              : 
     666              :       // Skip parsing a separator on the first match, otherwise consume it.
     667              :       // If it isn't present, this is an error
     668         4238 :       if (rep.has_sep () && match_amount > 0)
     669          357 :         if (!match_token (parser, *rep.get_sep ()))
     670              :           break;
     671              : 
     672         4223 :       sub_stack.push ();
     673         4223 :       bool valid_current_match = false;
     674         9960 :       for (auto &match : matches)
     675              :         {
     676         5737 :           size_t offs_begin = source.get_offs ();
     677         5737 :           switch (match->get_macro_match_type ())
     678              :             {
     679         4422 :             case AST::MacroMatch::MacroMatchType::Fragment:
     680         4422 :               {
     681         4422 :                 AST::MacroMatchFragment *fragment
     682         4422 :                   = static_cast<AST::MacroMatchFragment *> (match.get ());
     683         4422 :                 valid_current_match = match_fragment (parser, *fragment);
     684              : 
     685              :                 // matched fragment get the offset in the token stream
     686         4422 :                 size_t offs_end = source.get_offs ();
     687              : 
     688         4422 :                 if (valid_current_match)
     689         8832 :                   sub_stack.insert_metavar (
     690         8832 :                     MatchedFragment (fragment->get_ident ().as_string (),
     691        13248 :                                      offs_begin, offs_end));
     692              :               }
     693              :               break;
     694              : 
     695         1268 :             case AST::MacroMatch::MacroMatchType::Tok:
     696         1268 :               {
     697         1268 :                 AST::Token *tok = static_cast<AST::Token *> (match.get ());
     698         1268 :                 valid_current_match = match_token (parser, *tok);
     699              :               }
     700         1268 :               break;
     701              : 
     702           14 :             case AST::MacroMatch::MacroMatchType::Repetition:
     703           14 :               {
     704           14 :                 AST::MacroMatchRepetition *rep
     705           14 :                   = static_cast<AST::MacroMatchRepetition *> (match.get ());
     706           14 :                 valid_current_match = match_repetition (parser, *rep);
     707              :               }
     708           14 :               break;
     709              : 
     710           33 :             case AST::MacroMatch::MacroMatchType::Matcher:
     711           33 :               {
     712           33 :                 AST::MacroMatcher *m
     713           33 :                   = static_cast<AST::MacroMatcher *> (match.get ());
     714           33 :                 valid_current_match = match_matcher (parser, *m, true);
     715              :               }
     716           33 :               break;
     717              :             }
     718              :         }
     719         4223 :       auto old_stack = sub_stack.pop ();
     720              : 
     721              :       // If we've encountered an error once, stop trying to match more
     722              :       // repetitions
     723         4223 :       if (!valid_current_match)
     724              :         break;
     725              : 
     726              :       // nest metavars into repetitions
     727         8653 :       for (auto &ent : old_stack)
     728         8880 :         sub_stack.append_fragment (ent.first, std::move (ent.second));
     729              : 
     730         4213 :       match_amount++;
     731              : 
     732              :       // Break early if we notice there's too many expressions already
     733         4213 :       if (hi_bound && match_amount > hi_bound)
     734              :         break;
     735         4223 :     }
     736              : 
     737              :   // Check if the amount of matches we got is valid: Is it more than the lower
     738              :   // bound and less than the higher bound?
     739         1721 :   bool did_meet_lo_bound = match_amount >= lo_bound;
     740         1721 :   bool did_meet_hi_bound = hi_bound ? match_amount <= hi_bound : true;
     741              : 
     742              :   // If the end-result is valid, then we can clear the parse errors: Since
     743              :   // repetitions are parsed eagerly, it is okay to fail in some cases
     744         3441 :   auto res = did_meet_lo_bound && did_meet_hi_bound;
     745         1720 :   if (res)
     746         1716 :     parser.clear_errors ();
     747              : 
     748         1721 :   return res;
     749              : }
     750              : 
     751              : /*
     752              :  * Helper function for defining unmatched repetition metavars
     753              :  */
     754              : void
     755         2916 : MacroExpander::match_repetition_skipped_metavars (AST::MacroMatch &match)
     756              : {
     757              :   // We have to handle zero fragments differently: They will not have been
     758              :   // "matched" but they are still valid and should be inserted as a special
     759              :   // case. So we go through the stack map, and for every fragment which doesn't
     760              :   // exist, insert a zero-matched fragment.
     761         2916 :   switch (match.get_macro_match_type ())
     762              :     {
     763         1769 :     case AST::MacroMatch::MacroMatchType::Fragment:
     764         1769 :       match_repetition_skipped_metavars (
     765              :         static_cast<AST::MacroMatchFragment &> (match));
     766         1769 :       break;
     767           12 :     case AST::MacroMatch::MacroMatchType::Repetition:
     768           12 :       match_repetition_skipped_metavars (
     769              :         static_cast<AST::MacroMatchRepetition &> (match));
     770           12 :       break;
     771           13 :     case AST::MacroMatch::MacroMatchType::Matcher:
     772           13 :       match_repetition_skipped_metavars (
     773              :         static_cast<AST::MacroMatcher &> (match));
     774           13 :       break;
     775              :     case AST::MacroMatch::MacroMatchType::Tok:
     776              :       break;
     777              :     }
     778         2916 : }
     779              : 
     780              : void
     781         1769 : MacroExpander::match_repetition_skipped_metavars (
     782              :   AST::MacroMatchFragment &fragment)
     783              : {
     784         1769 :   auto &stack_map = sub_stack.peek ();
     785         1769 :   auto it = stack_map.find (fragment.get_ident ().as_string ());
     786              : 
     787         1769 :   if (it == stack_map.end ())
     788          114 :     sub_stack.insert_matches (fragment.get_ident ().as_string (),
     789          114 :                               MatchedFragmentContainer::zero ());
     790         1769 : }
     791              : 
     792              : void
     793         1733 : MacroExpander::match_repetition_skipped_metavars (
     794              :   AST::MacroMatchRepetition &rep)
     795              : {
     796         4643 :   for (auto &match : rep.get_matches ())
     797         2910 :     match_repetition_skipped_metavars (*match);
     798         1733 : }
     799              : 
     800              : void
     801           13 : MacroExpander::match_repetition_skipped_metavars (AST::MacroMatcher &rep)
     802              : {
     803           19 :   for (auto &match : rep.get_matches ())
     804            6 :     match_repetition_skipped_metavars (*match);
     805           13 : }
     806              : 
     807              : bool
     808         1721 : MacroExpander::match_repetition (Parser<MacroInvocLexer> &parser,
     809              :                                  AST::MacroMatchRepetition &rep)
     810              : {
     811         1721 :   size_t match_amount = 0;
     812         1721 :   bool res = false;
     813              : 
     814         1721 :   std::string lo_str;
     815         1721 :   std::string hi_str;
     816         1721 :   switch (rep.get_op ())
     817              :     {
     818         1564 :     case AST::MacroMatchRepetition::MacroRepOp::ANY:
     819         1564 :       lo_str = "0";
     820         1564 :       hi_str = "+inf";
     821         1564 :       res = match_n_matches (parser, rep, match_amount);
     822         1564 :       break;
     823          113 :     case AST::MacroMatchRepetition::MacroRepOp::ONE_OR_MORE:
     824          113 :       lo_str = "1";
     825          113 :       hi_str = "+inf";
     826          113 :       res = match_n_matches (parser, rep, match_amount, 1);
     827          113 :       break;
     828           44 :     case AST::MacroMatchRepetition::MacroRepOp::ZERO_OR_ONE:
     829           44 :       lo_str = "0";
     830           44 :       hi_str = "1";
     831           44 :       res = match_n_matches (parser, rep, match_amount, 0, 1);
     832           44 :       break;
     833            0 :     default:
     834            0 :       rust_unreachable ();
     835              :     }
     836              : 
     837         1726 :   rust_debug_loc (rep.get_match_locus (), "%s matched %lu times",
     838              :                   res ? "successfully" : "unsuccessfully",
     839              :                   (unsigned long) match_amount);
     840              : 
     841         1721 :   match_repetition_skipped_metavars (rep);
     842              : 
     843         1721 :   return res;
     844         1721 : }
     845              : 
     846              : /**
     847              :  * Helper function to refactor calling a parsing function 0 or more times
     848              :  */
     849              : static AST::Fragment
     850          736 : parse_many (Parser<MacroInvocLexer> &parser, TokenId delimiter,
     851              :             std::function<AST::SingleASTNode ()> parse_fn)
     852              : {
     853          736 :   auto &lexer = parser.get_token_source ();
     854          736 :   auto start = lexer.get_offs ();
     855              : 
     856          736 :   std::vector<AST::SingleASTNode> nodes;
     857         3135 :   while (true)
     858              :     {
     859         7742 :       if (parser.peek_current_token ()->get_id () == delimiter)
     860              :         break;
     861              : 
     862         3139 :       auto node = parse_fn ();
     863         3139 :       if (node.is_error ())
     864              :         {
     865            9 :           for (auto err : parser.get_errors ())
     866            5 :             err.emit ();
     867              : 
     868            4 :           return AST::Fragment::create_error ();
     869              :         }
     870              : 
     871         3135 :       nodes.emplace_back (std::move (node));
     872         3139 :     }
     873          732 :   auto end = lexer.get_offs ();
     874              : 
     875          732 :   return AST::Fragment (std::move (nodes), lexer.get_token_slice (start, end));
     876          736 : }
     877              : 
     878              : /**
     879              :  * Transcribe 0 or more items from a macro invocation
     880              :  *
     881              :  * @param parser Parser to extract items from
     882              :  * @param delimiter Id of the token on which parsing should stop
     883              :  */
     884              : static AST::Fragment
     885          336 : transcribe_many_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
     886              : {
     887          336 :   return parse_many (parser, delimiter, [&parser] () {
     888         2589 :     auto item = parser.parse_item (true);
     889         2589 :     if (!item)
     890            1 :       return AST::SingleASTNode (std::unique_ptr<AST::Item> (nullptr));
     891         2588 :     return AST::SingleASTNode (std::move (item.value ()));
     892          336 :   });
     893              : }
     894              : 
     895              : /**
     896              :  * Transcribe 0 or more external items from a macro invocation
     897              :  *
     898              :  * @param parser Parser to extract items from
     899              :  * @param delimiter Id of the token on which parsing should stop
     900              :  */
     901              : static AST::Fragment
     902            2 : transcribe_many_ext (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
     903              : {
     904            2 :   return parse_many (parser, delimiter, [&parser] () {
     905            3 :     auto item = parser.parse_external_item ();
     906            3 :     return AST::SingleASTNode (std::move (item));
     907            5 :   });
     908              : }
     909              : 
     910              : /**
     911              :  * Transcribe 0 or more trait items from a macro invocation
     912              :  *
     913              :  * @param parser Parser to extract items from
     914              :  * @param delimiter Id of the token on which parsing should stop
     915              :  */
     916              : static AST::Fragment
     917            1 : transcribe_many_trait_items (Parser<MacroInvocLexer> &parser,
     918              :                              TokenId &delimiter)
     919              : {
     920            1 :   return parse_many (parser, delimiter, [&parser] () {
     921            2 :     auto item = parser.parse_trait_item ();
     922            2 :     return AST::SingleASTNode (std::move (item));
     923            3 :   });
     924              : }
     925              : 
     926              : /**
     927              :  * Transcribe 0 or more impl items from a macro invocation
     928              :  *
     929              :  * @param parser Parser to extract items from
     930              :  * @param delimiter Id of the token on which parsing should stop
     931              :  */
     932              : static AST::Fragment
     933            1 : transcribe_many_impl_items (Parser<MacroInvocLexer> &parser, TokenId &delimiter)
     934              : {
     935            1 :   return parse_many (parser, delimiter, [&parser] () {
     936            2 :     auto item = parser.parse_inherent_impl_item ();
     937            2 :     return AST::SingleASTNode (std::move (item));
     938            3 :   });
     939              : }
     940              : 
     941              : /**
     942              :  * Transcribe 0 or more trait impl items from a macro invocation
     943              :  *
     944              :  * @param parser Parser to extract items from
     945              :  * @param delimiter Id of the token on which parsing should stop
     946              :  */
     947              : static AST::Fragment
     948           31 : transcribe_many_trait_impl_items (Parser<MacroInvocLexer> &parser,
     949              :                                   TokenId &delimiter)
     950              : {
     951           31 :   return parse_many (parser, delimiter, [&parser] () {
     952          103 :     auto item = parser.parse_trait_impl_item ();
     953          103 :     return AST::SingleASTNode (std::move (item));
     954          134 :   });
     955              : }
     956              : 
     957              : /**
     958              :  * Transcribe 0 or more statements from a macro invocation
     959              :  *
     960              :  * @param parser Parser to extract statements from
     961              :  * @param delimiter Id of the token on which parsing should stop
     962              :  */
     963              : static AST::Fragment
     964          365 : transcribe_many_stmts (Parser<MacroInvocLexer> &parser, TokenId delimiter,
     965              :                        bool semicolon)
     966              : {
     967          365 :   auto restrictions = ParseRestrictions ();
     968          365 :   restrictions.allow_close_after_expr_stmt = true;
     969              : 
     970          365 :   return parse_many (parser, delimiter,
     971          365 :                      [&parser, restrictions, delimiter, semicolon] () {
     972          440 :                        auto stmt = parser.parse_stmt (restrictions);
     973          432 :                        if (semicolon && stmt
     974         1301 :                            && parser.peek_current_token ()->get_id ()
     975          429 :                                 == delimiter)
     976          335 :                          stmt->add_semicolon ();
     977              : 
     978          440 :                        return AST::SingleASTNode (std::move (stmt));
     979          805 :                      });
     980              : }
     981              : 
     982              : /**
     983              :  * Transcribe one expression from a macro invocation
     984              :  *
     985              :  * @param parser Parser to extract statements from
     986              :  */
     987              : static AST::Fragment
     988         1668 : transcribe_expression (Parser<MacroInvocLexer> &parser)
     989              : {
     990         1668 :   auto &lexer = parser.get_token_source ();
     991         1668 :   auto start = lexer.get_offs ();
     992              : 
     993         1668 :   auto attrs = parser.parse_outer_attributes ();
     994         1668 :   auto expr = parser.parse_expr (std::move (attrs));
     995         1673 :   for (auto error : parser.get_errors ())
     996            5 :     error.emit ();
     997         1668 :   if (!expr)
     998            1 :     return AST::Fragment::create_error ();
     999              : 
    1000              :   // FIXME: make this an error for some edititons
    1001         3334 :   if (parser.peek_current_token ()->get_id () == SEMICOLON)
    1002              :     {
    1003            1 :       rust_warning_at (
    1004            1 :         parser.peek_current_token ()->get_locus (), 0,
    1005              :         "trailing semicolon in macro used in expression context");
    1006            1 :       parser.skip_token ();
    1007              :     }
    1008              : 
    1009         1667 :   auto end = lexer.get_offs ();
    1010              : 
    1011         3334 :   return AST::Fragment ({std::move (expr.value ())},
    1012         6668 :                         lexer.get_token_slice (start, end));
    1013         1668 : }
    1014              : 
    1015              : /**
    1016              :  * Transcribe one type from a macro invocation
    1017              :  *
    1018              :  * @param parser Parser to extract statements from
    1019              :  */
    1020              : static AST::Fragment
    1021           29 : transcribe_type (Parser<MacroInvocLexer> &parser)
    1022              : {
    1023           29 :   auto &lexer = parser.get_token_source ();
    1024           29 :   auto start = lexer.get_offs ();
    1025              : 
    1026           29 :   auto type = parser.parse_type (true);
    1027           29 :   for (auto err : parser.get_errors ())
    1028            0 :     err.emit ();
    1029           29 :   if (!type)
    1030            0 :     return AST::Fragment::create_error ();
    1031              : 
    1032           29 :   auto end = lexer.get_offs ();
    1033              : 
    1034           58 :   return AST::Fragment ({std::move (type)}, lexer.get_token_slice (start, end));
    1035           29 : }
    1036              : 
    1037              : /**
    1038              :  * Transcribe one pattern from a macro invocation
    1039              :  *
    1040              :  * @param parser Parser to extract statements from
    1041              :  */
    1042              : static AST::Fragment
    1043            3 : transcribe_pattern (Parser<MacroInvocLexer> &parser)
    1044              : {
    1045            3 :   auto &lexer = parser.get_token_source ();
    1046            3 :   auto start = lexer.get_offs ();
    1047              : 
    1048            3 :   auto pattern = parser.parse_pattern ();
    1049            6 :   for (auto err : parser.get_errors ())
    1050            3 :     err.emit ();
    1051              : 
    1052            3 :   if (!pattern)
    1053            2 :     return AST::Fragment::create_error ();
    1054              : 
    1055            1 :   auto end = lexer.get_offs ();
    1056              : 
    1057            2 :   return AST::Fragment ({std::move (pattern)},
    1058            3 :                         lexer.get_token_slice (start, end));
    1059            3 : }
    1060              : 
    1061              : static AST::Fragment
    1062         2436 : transcribe_context (MacroExpander::ContextType ctx,
    1063              :                     Parser<MacroInvocLexer> &parser, bool semicolon,
    1064              :                     AST::DelimType delimiter, TokenId last_token_id)
    1065              : {
    1066              :   // The flow-chart in order to choose a parsing function is as follows:
    1067              :   //
    1068              :   // [switch special context]
    1069              :   //     -- Item --> parser.parse_item();
    1070              :   //     -- Trait --> parser.parse_trait_item();
    1071              :   //     -- Impl --> parser.parse_impl_item();
    1072              :   //     -- Extern --> parser.parse_extern_item();
    1073              :   //     -- Pattern --> parser.parse_pattern();
    1074              :   //     -- None --> [has semicolon?]
    1075              :   //                 -- Yes --> parser.parse_stmt();
    1076              :   //                 -- No --> [switch invocation.delimiter()]
    1077              :   //                             -- { } --> parser.parse_stmt();
    1078              :   //                             -- _ --> parser.parse_expr(); // once!
    1079              : 
    1080              :   // If there is a semicolon OR we are expanding a MacroInvocationSemi, then
    1081              :   // we can parse multiple items. Otherwise, parse *one* expression
    1082              : 
    1083         2436 :   switch (ctx)
    1084              :     {
    1085          336 :     case MacroExpander::ContextType::ITEM:
    1086          336 :       return transcribe_many_items (parser, last_token_id);
    1087            1 :       break;
    1088            1 :     case MacroExpander::ContextType::TRAIT:
    1089            1 :       return transcribe_many_trait_items (parser, last_token_id);
    1090            1 :       break;
    1091            1 :     case MacroExpander::ContextType::IMPL:
    1092            1 :       return transcribe_many_impl_items (parser, last_token_id);
    1093           31 :       break;
    1094           31 :     case MacroExpander::ContextType::TRAIT_IMPL:
    1095           31 :       return transcribe_many_trait_impl_items (parser, last_token_id);
    1096            2 :       break;
    1097            2 :     case MacroExpander::ContextType::EXTERN:
    1098            2 :       return transcribe_many_ext (parser, last_token_id);
    1099           29 :       break;
    1100           29 :     case MacroExpander::ContextType::TYPE:
    1101           29 :       return transcribe_type (parser);
    1102            3 :     case MacroExpander::ContextType::PATTERN:
    1103            3 :       return transcribe_pattern (parser);
    1104          365 :       break;
    1105          365 :     case MacroExpander::ContextType::STMT:
    1106          365 :       return transcribe_many_stmts (parser, last_token_id, semicolon);
    1107         1668 :     case MacroExpander::ContextType::EXPR:
    1108         1668 :       return transcribe_expression (parser);
    1109            0 :     default:
    1110            0 :       rust_unreachable ();
    1111              :     }
    1112              : }
    1113              : 
    1114              : static std::string
    1115         2436 : tokens_to_str (std::vector<std::unique_ptr<AST::Token>> &tokens)
    1116              : {
    1117         2436 :   std::string str;
    1118         2436 :   if (!tokens.empty ())
    1119              :     {
    1120         4872 :       str += tokens[0]->as_string ();
    1121       133371 :       for (size_t i = 1; i < tokens.size (); i++)
    1122       261870 :         str += " " + tokens[i]->as_string ();
    1123              :     }
    1124              : 
    1125         2436 :   return str;
    1126              : }
    1127              : 
    1128              : AST::Fragment
    1129         2436 : MacroExpander::transcribe_rule (
    1130              :   AST::MacroRulesDefinition &definition, AST::MacroRule &match_rule,
    1131              :   AST::DelimTokenTree &invoc_token_tree,
    1132              :   std::map<std::string, MatchedFragmentContainer *> &matched_fragments,
    1133              :   AST::InvocKind invoc_kind, ContextType ctx)
    1134              : {
    1135         2436 :   bool semicolon = invoc_kind == AST::InvocKind::Semicoloned;
    1136              : 
    1137              :   // we can manipulate the token tree to substitute the dollar identifiers so
    1138              :   // that when we call parse its already substituted for us
    1139         2436 :   AST::MacroTranscriber &transcriber = match_rule.get_transcriber ();
    1140         2436 :   AST::DelimTokenTree &transcribe_tree = transcriber.get_token_tree ();
    1141              : 
    1142         2436 :   auto invoc_stream = invoc_token_tree.to_token_stream ();
    1143         2436 :   auto macro_rule_tokens = transcribe_tree.to_token_stream ();
    1144              : 
    1145         2436 :   auto substitute_context
    1146              :     = SubstituteCtx (invoc_stream, macro_rule_tokens, matched_fragments,
    1147         2436 :                      definition, invoc_token_tree.get_locus ());
    1148         2436 :   std::vector<std::unique_ptr<AST::Token>> substituted_tokens
    1149         2436 :     = substitute_context.substitute_tokens ();
    1150              : 
    1151         2436 :   rust_debug ("substituted tokens: %s",
    1152              :               tokens_to_str (substituted_tokens).c_str ());
    1153              : 
    1154              :   // parse it to an Fragment
    1155         2436 :   MacroInvocLexer lex (std::move (substituted_tokens));
    1156         2436 :   Parser<MacroInvocLexer> parser (lex);
    1157              : 
    1158         2436 :   auto last_token_id = TokenId::RIGHT_CURLY;
    1159              : 
    1160              :   // this is used so we can check that we delimit the stream correctly.
    1161         2436 :   switch (transcribe_tree.get_delim_type ())
    1162              :     {
    1163          132 :     case AST::DelimType::PARENS:
    1164          132 :       last_token_id = TokenId::RIGHT_PAREN;
    1165          132 :       rust_assert (parser.skip_token (LEFT_PAREN));
    1166              :       break;
    1167              : 
    1168         2304 :     case AST::DelimType::CURLY:
    1169         2304 :       rust_assert (parser.skip_token (LEFT_CURLY));
    1170              :       break;
    1171              : 
    1172            0 :     case AST::DelimType::SQUARE:
    1173            0 :       last_token_id = TokenId::RIGHT_SQUARE;
    1174            0 :       rust_assert (parser.skip_token (LEFT_SQUARE));
    1175              :       break;
    1176              :     }
    1177              : 
    1178              :   // see https://github.com/Rust-GCC/gccrs/issues/22
    1179              :   // TL;DR:
    1180              :   //   - Treat all macro invocations with parentheses, (), or square brackets,
    1181              :   //   [], as expressions.
    1182              :   //   - If the macro invocation has curly brackets, {}, it may be parsed as a
    1183              :   //   statement depending on the context.
    1184              :   //   - If the macro invocation has a semicolon at the end, it must be parsed
    1185              :   //   as a statement (either via ExpressionStatement or
    1186              :   //   MacroInvocationWithSemi)
    1187              : 
    1188         2436 :   auto fragment
    1189              :     = transcribe_context (ctx, parser, semicolon,
    1190         2436 :                           invoc_token_tree.get_delim_type (), last_token_id);
    1191              : 
    1192              :   // emit any errors
    1193         2436 :   if (parser.has_errors ())
    1194           11 :     return AST::Fragment::create_error ();
    1195              : 
    1196              :   // are all the tokens used?
    1197         2425 :   bool did_delimit = parser.skip_token (last_token_id);
    1198              : 
    1199         2425 :   bool reached_end_of_stream = did_delimit && parser.skip_token (END_OF_FILE);
    1200            0 :   if (!reached_end_of_stream)
    1201              :     {
    1202              :       // FIXME: rustc has some cases it accepts this with a warning due to
    1203              :       // backwards compatibility.
    1204            0 :       const_TokenPtr current_token = parser.peek_current_token ();
    1205            0 :       rust_error_at (current_token->get_locus (),
    1206              :                      "tokens here and after are unparsed");
    1207            0 :     }
    1208              : 
    1209         2425 :   return fragment;
    1210         4872 : }
    1211              : 
    1212              : AST::Fragment
    1213            0 : MacroExpander::parse_proc_macro_output (ProcMacro::TokenStream ts)
    1214              : {
    1215            0 :   MacroInvocLexer lex (convert (ts));
    1216            0 :   Parser<MacroInvocLexer> parser (lex);
    1217              : 
    1218            0 :   std::vector<AST::SingleASTNode> nodes;
    1219            0 :   switch (peek_context ())
    1220              :     {
    1221              :     case ContextType::ITEM:
    1222            0 :       while (lex.peek_token ()->get_id () != END_OF_FILE)
    1223              :         {
    1224            0 :           auto result = parser.parse_item (false);
    1225            0 :           if (!result)
    1226              :             break;
    1227            0 :           nodes.emplace_back (std::move (result.value ()));
    1228            0 :         }
    1229              :       break;
    1230              :     case ContextType::STMT:
    1231            0 :       while (lex.peek_token ()->get_id () != END_OF_FILE)
    1232              :         {
    1233            0 :           auto result = parser.parse_stmt ();
    1234            0 :           if (result == nullptr)
    1235              :             break;
    1236            0 :           nodes.emplace_back (std::move (result));
    1237            0 :         }
    1238              :       break;
    1239            0 :     case ContextType::TRAIT:
    1240            0 :     case ContextType::IMPL:
    1241            0 :     case ContextType::TRAIT_IMPL:
    1242            0 :     case ContextType::EXTERN:
    1243            0 :     case ContextType::TYPE:
    1244            0 :     case ContextType::EXPR:
    1245            0 :     default:
    1246            0 :       rust_unreachable ();
    1247              :     }
    1248              : 
    1249            0 :   if (parser.has_errors ())
    1250            0 :     return AST::Fragment::create_error ();
    1251              :   else
    1252            0 :     return {nodes, std::vector<std::unique_ptr<AST::Token>> ()};
    1253            0 : }
    1254              : 
    1255              : MatchedFragment &
    1256        11366 : MatchedFragmentContainer::get_single_fragment ()
    1257              : {
    1258        11366 :   rust_assert (is_single_fragment ());
    1259              : 
    1260        11366 :   return static_cast<MatchedFragmentContainerMetaVar &> (*this).get_fragment ();
    1261              : }
    1262              : 
    1263              : std::vector<std::unique_ptr<MatchedFragmentContainer>> &
    1264         8862 : MatchedFragmentContainer::get_fragments ()
    1265              : {
    1266         8862 :   rust_assert (!is_single_fragment ());
    1267              : 
    1268         8862 :   return static_cast<MatchedFragmentContainerRepetition &> (*this)
    1269         8862 :     .get_fragments ();
    1270              : }
    1271              : 
    1272              : void
    1273            0 : MatchedFragmentContainer::add_fragment (MatchedFragment fragment)
    1274              : {
    1275            0 :   rust_assert (!is_single_fragment ());
    1276              : 
    1277            0 :   return static_cast<MatchedFragmentContainerRepetition &> (*this)
    1278            0 :     .add_fragment (fragment);
    1279              : }
    1280              : 
    1281              : void
    1282         4440 : MatchedFragmentContainer::add_fragment (
    1283              :   std::unique_ptr<MatchedFragmentContainer> fragment)
    1284              : {
    1285         4440 :   rust_assert (!is_single_fragment ());
    1286              : 
    1287         4440 :   return static_cast<MatchedFragmentContainerRepetition &> (*this)
    1288         4440 :     .add_fragment (std::move (fragment));
    1289              : }
    1290              : 
    1291              : std::unique_ptr<MatchedFragmentContainer>
    1292           57 : MatchedFragmentContainer::zero ()
    1293              : {
    1294           57 :   return std::unique_ptr<MatchedFragmentContainer> (
    1295           57 :     new MatchedFragmentContainerRepetition ());
    1296              : }
    1297              : 
    1298              : std::unique_ptr<MatchedFragmentContainer>
    1299         8151 : MatchedFragmentContainer::metavar (MatchedFragment fragment)
    1300              : {
    1301         8151 :   return std::unique_ptr<MatchedFragmentContainer> (
    1302         8151 :     new MatchedFragmentContainerMetaVar (fragment));
    1303              : }
    1304              : 
    1305              : } // namespace Rust
        

Generated by: LCOV version 2.4-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.