LCOV - code coverage report
Current view: top level - gcc - tree-inline.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 88.4 % 3293 2912
Test Date: 2026-11-14 16:39:50 Functions: 96.6 % 88 85
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Tree inlining.
       2              :    Copyright (C) 2001-2026 Free Software Foundation, Inc.
       3              :    Contributed by Alexandre Oliva <aoliva@redhat.com>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify
       8              : it under the terms of the GNU General Public License as published by
       9              : the Free Software Foundation; either version 3, or (at your option)
      10              : any later version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful,
      13              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      14              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15              : GNU General Public License for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "target.h"
      26              : #include "rtl.h"
      27              : #include "tree.h"
      28              : #include "gimple.h"
      29              : #include "cfghooks.h"
      30              : #include "tree-pass.h"
      31              : #include "ssa.h"
      32              : #include "cgraph.h"
      33              : #include "tree-pretty-print.h"
      34              : #include "diagnostic-core.h"
      35              : #include "gimple-predict.h"
      36              : #include "fold-const.h"
      37              : #include "stor-layout.h"
      38              : #include "calls.h"
      39              : #include "tree-inline.h"
      40              : #include "langhooks.h"
      41              : #include "cfganal.h"
      42              : #include "tree-iterator.h"
      43              : #include "intl.h"
      44              : #include "gimple-iterator.h"
      45              : #include "gimple-fold.h"
      46              : #include "tree-eh.h"
      47              : #include "gimplify.h"
      48              : #include "gimplify-me.h"
      49              : #include "gimple-walk.h"
      50              : #include "tree-cfg.h"
      51              : #include "tree-into-ssa.h"
      52              : #include "tree-dfa.h"
      53              : #include "tree-ssa.h"
      54              : #include "except.h"
      55              : #include "debug.h"
      56              : #include "value-prof.h"
      57              : #include "cfgloop.h"
      58              : #include "builtins.h"
      59              : #include "stringpool.h"
      60              : #include "attribs.h"
      61              : #include "sreal.h"
      62              : #include "tree-cfgcleanup.h"
      63              : #include "tree-ssa-live.h"
      64              : #include "alloc-pool.h"
      65              : #include "symbol-summary.h"
      66              : #include "symtab-thunks.h"
      67              : #include "symtab-clones.h"
      68              : #include "asan.h"
      69              : 
      70              : /* I'm not real happy about this, but we need to handle gimple and
      71              :    non-gimple trees.  */
      72              : 
      73              : /* Inlining, Cloning, Versioning, Parallelization
      74              : 
      75              :    Inlining: a function body is duplicated, but the PARM_DECLs are
      76              :    remapped into VAR_DECLs, and non-void RETURN_EXPRs become
      77              :    MODIFY_EXPRs that store to a dedicated returned-value variable.
      78              :    The duplicated eh_region info of the copy will later be appended
      79              :    to the info for the caller; the eh_region info in copied throwing
      80              :    statements and RESX statements are adjusted accordingly.
      81              : 
      82              :    Cloning: (only in C++) We have one body for a con/de/structor, and
      83              :    multiple function decls, each with a unique parameter list.
      84              :    Duplicate the body, using the given splay tree; some parameters
      85              :    will become constants (like 0 or 1).
      86              : 
      87              :    Versioning: a function body is duplicated and the result is a new
      88              :    function rather than into blocks of an existing function as with
      89              :    inlining.  Some parameters will become constants.
      90              : 
      91              :    Parallelization: a region of a function is duplicated resulting in
      92              :    a new function.  Variables may be replaced with complex expressions
      93              :    to enable shared variable semantics.
      94              : 
      95              :    All of these will simultaneously lookup any callgraph edges.  If
      96              :    we're going to inline the duplicated function body, and the given
      97              :    function has some cloned callgraph nodes (one for each place this
      98              :    function will be inlined) those callgraph edges will be duplicated.
      99              :    If we're cloning the body, those callgraph edges will be
     100              :    updated to point into the new body.  (Note that the original
     101              :    callgraph node and edge list will not be altered.)
     102              : 
     103              :    See the CALL_EXPR handling case in copy_tree_body_r ().  */
     104              : 
     105              : /* To Do:
     106              : 
     107              :    o In order to make inlining-on-trees work, we pessimized
     108              :      function-local static constants.  In particular, they are now
     109              :      always output, even when not addressed.  Fix this by treating
     110              :      function-local static constants just like global static
     111              :      constants; the back-end already knows not to output them if they
     112              :      are not needed.
     113              : 
     114              :    o Provide heuristics to clamp inlining of recursive template
     115              :      calls?  */
     116              : 
     117              : 
     118              : /* Weights that estimate_num_insns uses to estimate the size of the
     119              :    produced code.  */
     120              : 
     121              : eni_weights eni_size_weights;
     122              : 
     123              : /* Weights that estimate_num_insns uses to estimate the time necessary
     124              :    to execute the produced code.  */
     125              : 
     126              : eni_weights eni_time_weights;
     127              : 
     128              : /* Prototypes.  */
     129              : 
     130              : static tree declare_return_variable (copy_body_data *, tree, tree,
     131              :                                      basic_block);
     132              : static void remap_block (tree *, copy_body_data *);
     133              : static void copy_bind_expr (tree *, int *, copy_body_data *);
     134              : static void declare_inline_vars (tree, tree);
     135              : static void remap_save_expr (tree *, hash_map<tree, tree> *, int *);
     136              : static void prepend_lexical_block (tree current_block, tree new_block);
     137              : static tree copy_result_decl_to_var (tree, copy_body_data *);
     138              : static tree copy_decl_maybe_to_var (tree, copy_body_data *);
     139              : static gimple_seq remap_gimple_stmt (gimple *, copy_body_data *);
     140              : static void insert_init_stmt (copy_body_data *, basic_block, gimple *);
     141              : 
     142              : /* Insert a tree->tree mapping for ID.  Despite the name suggests
     143              :    that the trees should be variables, it is used for more than that.  */
     144              : 
     145              : void
     146    697818841 : insert_decl_map (copy_body_data *id, tree key, tree value)
     147              : {
     148    697818841 :   id->decl_map->put (key, value);
     149              : 
     150              :   /* Always insert an identity map as well.  If we see this same new
     151              :      node again, we won't want to duplicate it a second time.  */
     152    697818841 :   if (key != value && value)
     153    213167617 :     id->decl_map->put (value, value);
     154    697818841 : }
     155              : 
     156              : /* If nonzero, we're remapping the contents of inlined debug
     157              :    statements.  If negative, an error has occurred, such as a
     158              :    reference to a variable that isn't available in the inlined
     159              :    context.  */
     160              : static int processing_debug_stmt = 0;
     161              : 
     162              : /* Construct new SSA name for old NAME. ID is the inline context.  */
     163              : 
     164              : static tree
     165     71736790 : remap_ssa_name (tree name, copy_body_data *id)
     166              : {
     167     71736790 :   tree new_tree, var;
     168     71736790 :   tree *n;
     169              : 
     170     71736790 :   gcc_assert (TREE_CODE (name) == SSA_NAME);
     171              : 
     172     71736790 :   n = id->decl_map->get (name);
     173     71736790 :   if (n)
     174              :     {
     175              :       /* When we perform edge redirection as part of CFG copy, IPA-SRA can
     176              :          remove an unused LHS from a call statement.  Such LHS can however
     177              :          still appear in debug statements, but their value is lost in this
     178              :          function and we do not want to map them.  */
     179     52691656 :       if (id->killed_new_ssa_names
     180     52691656 :           && id->killed_new_ssa_names->contains (*n))
     181              :         {
     182          386 :           gcc_assert (processing_debug_stmt);
     183          386 :           processing_debug_stmt = -1;
     184          386 :           return name;
     185              :         }
     186              : 
     187     52691270 :       return unshare_expr (*n);
     188              :     }
     189              : 
     190     19045134 :   if (processing_debug_stmt)
     191              :     {
     192        99365 :       if (SSA_NAME_IS_DEFAULT_DEF (name)
     193        99322 :           && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
     194        99275 :           && id->entry_bb == NULL
     195       198639 :           && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
     196              :         {
     197        99274 :           gimple *def_temp;
     198        99274 :           gimple_stmt_iterator gsi;
     199        99274 :           tree val = SSA_NAME_VAR (name);
     200              : 
     201        99274 :           n = id->decl_map->get (val);
     202        99274 :           if (n != NULL)
     203        99274 :             val = *n;
     204        99274 :           if (TREE_CODE (val) != PARM_DECL
     205        99274 :               && !(VAR_P (val) && DECL_ABSTRACT_ORIGIN (val)))
     206              :             {
     207            0 :               processing_debug_stmt = -1;
     208            0 :               return name;
     209              :             }
     210        99274 :           n = id->decl_map->get (val);
     211        99274 :           if (n && TREE_CODE (*n) == DEBUG_EXPR_DECL)
     212              :             return *n;
     213        34699 :           tree vexpr = build_debug_expr_decl (TREE_TYPE (name));
     214              :           /* FIXME: Is setting the mode really necessary? */
     215        34699 :           SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (name)));
     216        34699 :           def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
     217        34699 :           gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
     218        34699 :           gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
     219        34699 :           insert_decl_map (id, val, vexpr);
     220        34699 :           return vexpr;
     221              :         }
     222              : 
     223           91 :       processing_debug_stmt = -1;
     224           91 :       return name;
     225              :     }
     226              : 
     227              :   /* Remap anonymous SSA names or SSA names of anonymous decls.  */
     228     18945769 :   var = SSA_NAME_VAR (name);
     229      3764591 :   if (!var
     230      3764591 :       || (!SSA_NAME_IS_DEFAULT_DEF (name)
     231      3359417 :           && VAR_P (var)
     232      3149444 :           && !VAR_DECL_IS_VIRTUAL_OPERAND (var)
     233      3149444 :           && DECL_ARTIFICIAL (var)
     234       579133 :           && DECL_IGNORED_P (var)
     235       161474 :           && !DECL_NAME (var)))
     236              :     {
     237     15186622 :       const pt_solution *pt;
     238     15186622 :       new_tree = make_ssa_name (remap_type (TREE_TYPE (name), id));
     239     15186622 :       if (!var && SSA_NAME_IDENTIFIER (name))
     240      1969612 :         SET_SSA_NAME_VAR_OR_IDENTIFIER (new_tree, SSA_NAME_IDENTIFIER (name));
     241     15186622 :       insert_decl_map (id, name, new_tree);
     242     30373244 :       SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
     243     15186622 :         = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
     244              :       /* Range info can be transferred, if local addresses are removed.  */
     245     15186622 :       if (SSA_NAME_RANGE_INFO (name))
     246      8741841 :         duplicate_ssa_name_range_info (new_tree, name, true);
     247              :       /* At least IPA points-to info can be directly transferred.  */
     248     15186622 :       if (id->src_cfun->gimple_df
     249     15186622 :           && id->src_cfun->gimple_df->ipa_pta
     250        15022 :           && POINTER_TYPE_P (TREE_TYPE (name))
     251         5191 :           && (pt = ssa_name_pt_solution (name))
     252     15189215 :           && !pt->anything)
     253         2572 :         set_ssa_name_pt_solution (new_tree, *pt);
     254              :       return new_tree;
     255              :     }
     256              : 
     257              :   /* Do not set DEF_STMT yet as statement is not copied yet. We do that
     258              :      in copy_bb.  */
     259      3759147 :   new_tree = remap_decl (var, id);
     260              : 
     261              :   /* We might've substituted constant or another SSA_NAME for
     262              :      the variable.
     263              : 
     264              :      Replace the SSA name representing RESULT_DECL by variable during
     265              :      inlining:  this saves us from need to introduce PHI node in a case
     266              :      return value is just partly initialized.  */
     267       388806 :   if ((VAR_P (new_tree) || TREE_CODE (new_tree) == PARM_DECL)
     268      4147953 :       && (!SSA_NAME_VAR (name)
     269      3759147 :           || TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL
     270            0 :           || !id->transform_return_to_modify))
     271              :     {
     272      3759147 :       const pt_solution *pt;
     273      3759147 :       new_tree = make_ssa_name (new_tree);
     274      3759147 :       insert_decl_map (id, name, new_tree);
     275      7518294 :       SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
     276      3759147 :         = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
     277              : 
     278      3759147 :       tree zero = NULL_TREE;
     279      3759147 :       if (SSA_NAME_IS_DEFAULT_DEF (name))
     280              :         {
     281              :           /* By inlining function having uninitialized variable, we might
     282              :              extend the lifetime (variable might get reused).  This cause
     283              :              ICE in the case we end up extending lifetime of SSA name across
     284              :              abnormal edge, but also increase register pressure.
     285              : 
     286              :              We simply initialize all uninitialized vars by 0 except
     287              :              for case we are inlining to very first BB.  We can avoid
     288              :              this for all BBs that are not inside strongly connected
     289              :              regions of the CFG, but this is expensive to test.  */
     290       405174 :           if (id->entry_bb
     291        45077 :               && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name)
     292            0 :               && (!SSA_NAME_VAR (name)
     293            0 :                   || TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL)
     294       405174 :               && (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun),
     295            0 :                                              0)->dest
     296            0 :                   || EDGE_COUNT (id->entry_bb->preds) != 1))
     297              :             {
     298            0 :               gimple_stmt_iterator gsi = gsi_last_bb (id->entry_bb);
     299            0 :               gimple *init_stmt;
     300            0 :               zero = build_zero_cst (TREE_TYPE (new_tree));
     301              : 
     302            0 :               init_stmt = gimple_build_assign (new_tree, zero);
     303            0 :               gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT);
     304            0 :               SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0;
     305              :             }
     306              :           else
     307              :             {
     308       405174 :               SSA_NAME_DEF_STMT (new_tree) = gimple_build_nop ();
     309       405174 :               set_ssa_default_def (cfun, SSA_NAME_VAR (new_tree), new_tree);
     310              :             }
     311              :         }
     312              :       /* Range info can be transferred if local addresses are removed
     313              :          and no uninitialized name was processed and set to zero.  */
     314      3759147 :       if (!zero && SSA_NAME_RANGE_INFO (name))
     315      1928452 :         duplicate_ssa_name_range_info (new_tree, name, true);
     316              :       /* At least IPA points-to info can be directly transferred.  */
     317      3759147 :       if (id->src_cfun->gimple_df
     318      3759147 :           && id->src_cfun->gimple_df->ipa_pta
     319         2518 :           && POINTER_TYPE_P (TREE_TYPE (name))
     320          560 :           && (pt = ssa_name_pt_solution (name))
     321      3759424 :           && !pt->anything)
     322          277 :         set_ssa_name_pt_solution (new_tree, *pt);
     323              :     }
     324              :   else
     325            0 :     insert_decl_map (id, name, new_tree);
     326              :   return new_tree;
     327              : }
     328              : 
     329              : /* Remap DECL during the copying of the BLOCK tree for the function.  */
     330              : 
     331              : tree
     332    355218806 : remap_decl (tree decl, copy_body_data *id)
     333              : {
     334    355218806 :   tree *n;
     335              : 
     336              :   /* We only remap local variables in the current function.  */
     337              : 
     338              :   /* See if we have remapped this declaration.  */
     339              : 
     340    355218806 :   n = id->decl_map->get (decl);
     341              : 
     342    355218806 :   if (!n && processing_debug_stmt)
     343              :     {
     344       677738 :       processing_debug_stmt = -1;
     345       677738 :       return decl;
     346              :     }
     347              : 
     348              :   /* When remapping a type within copy_gimple_seq_and_replace_locals, all
     349              :      necessary DECLs have already been remapped and we do not want to duplicate
     350              :      a decl coming from outside of the sequence we are copying.  */
     351    130929579 :   if (!n
     352    130929579 :       && id->prevent_decl_creation_for_types
     353            0 :       && id->remapping_type_depth > 0
     354            0 :       && (VAR_P (decl) || TREE_CODE (decl) == PARM_DECL))
     355              :     return decl;
     356              : 
     357              :   /* If we didn't already have an equivalent for this declaration, create one
     358              :      now.  */
     359    354541068 :   if (!n)
     360              :     {
     361              :       /* Make a copy of the variable or label.  */
     362    130929579 :       tree t = id->copy_decl (decl, id);
     363              : 
     364              :       /* Remember it, so that if we encounter this local entity again
     365              :          we can reuse this copy.  Do this early because remap_type may
     366              :          need this decl for TYPE_STUB_DECL.  */
     367    130929579 :       insert_decl_map (id, decl, t);
     368              : 
     369    130929579 :       if (!DECL_P (t) || t == decl)
     370              :         return t;
     371              : 
     372              :       /* Remap types, if necessary.  */
     373    128569485 :       TREE_TYPE (t) = remap_type (TREE_TYPE (t), id);
     374    128569485 :       if (TREE_CODE (t) == TYPE_DECL)
     375              :         {
     376      1607754 :           DECL_ORIGINAL_TYPE (t) = remap_type (DECL_ORIGINAL_TYPE (t), id);
     377              : 
     378              :           /* Preserve the invariant that DECL_ORIGINAL_TYPE != TREE_TYPE,
     379              :              which is enforced in gen_typedef_die when DECL_ABSTRACT_ORIGIN
     380              :              is not set on the TYPE_DECL, for example in LTO mode.  */
     381      1607754 :           if (DECL_ORIGINAL_TYPE (t) == TREE_TYPE (t))
     382              :             {
     383           10 :               tree x = build_variant_type_copy (TREE_TYPE (t));
     384           10 :               TYPE_STUB_DECL (x) = TYPE_STUB_DECL (TREE_TYPE (t));
     385           10 :               TYPE_NAME (x) = TYPE_NAME (TREE_TYPE (t));
     386           10 :               DECL_ORIGINAL_TYPE (t) = x;
     387              :             }
     388              :         }
     389              : 
     390              :       /* Remap sizes as necessary.  */
     391    128569485 :       walk_tree (&DECL_SIZE (t), copy_tree_body_r, id, NULL);
     392    128569485 :       walk_tree (&DECL_SIZE_UNIT (t), copy_tree_body_r, id, NULL);
     393              : 
     394              :       /* If fields, do likewise for offset and qualifier.  */
     395    128569485 :       if (TREE_CODE (t) == FIELD_DECL)
     396              :         {
     397          773 :           walk_tree (&DECL_FIELD_OFFSET (t), copy_tree_body_r, id, NULL);
     398          773 :           if (TREE_CODE (DECL_CONTEXT (t)) == QUAL_UNION_TYPE)
     399            0 :             walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL);
     400              :         }
     401              : 
     402              :       return t;
     403              :     }
     404              : 
     405    223611489 :   if (id->do_not_unshare)
     406    141511691 :     return *n;
     407              :   else
     408     82099798 :     return unshare_expr (*n);
     409              : }
     410              : 
     411              : static tree
     412       104561 : remap_type_1 (tree type, copy_body_data *id)
     413              : {
     414       104561 :   tree new_tree, t;
     415              : 
     416              :   /* We do need a copy.  build and register it now.  If this is a pointer or
     417              :      reference type, remap the designated type and make a new pointer or
     418              :      reference type.  */
     419       104561 :   if (TREE_CODE (type) == POINTER_TYPE)
     420              :     {
     421        28450 :       new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id),
     422        28450 :                                          TYPE_MODE (type),
     423        28450 :                                          TYPE_REF_CAN_ALIAS_ALL (type));
     424        28450 :       if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
     425        15521 :         new_tree = build_type_attribute_qual_variant (new_tree,
     426        15521 :                                                       TYPE_ATTRIBUTES (type),
     427        15521 :                                                       TYPE_QUALS (type));
     428        28450 :       insert_decl_map (id, type, new_tree);
     429        28450 :       return new_tree;
     430              :     }
     431        76111 :   else if (TREE_CODE (type) == REFERENCE_TYPE)
     432              :     {
     433         8061 :       new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id),
     434         8061 :                                             TYPE_MODE (type),
     435         8061 :                                             TYPE_REF_CAN_ALIAS_ALL (type));
     436         8061 :       if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
     437         5481 :         new_tree = build_type_attribute_qual_variant (new_tree,
     438         5481 :                                                       TYPE_ATTRIBUTES (type),
     439         5481 :                                                       TYPE_QUALS (type));
     440         8061 :       insert_decl_map (id, type, new_tree);
     441         8061 :       return new_tree;
     442              :     }
     443              :   else
     444        68050 :     new_tree = copy_node (type);
     445              : 
     446        68050 :   insert_decl_map (id, type, new_tree);
     447              : 
     448              :   /* This is a new type, not a copy of an old type.  Need to reassociate
     449              :      variants.  We can handle everything except the main variant lazily.  */
     450        68050 :   t = TYPE_MAIN_VARIANT (type);
     451        68050 :   if (type != t)
     452              :     {
     453           79 :       t = remap_type (t, id);
     454           79 :       TYPE_MAIN_VARIANT (new_tree) = t;
     455           79 :       TYPE_NEXT_VARIANT (new_tree) = TYPE_NEXT_VARIANT (t);
     456           79 :       TYPE_NEXT_VARIANT (t) = new_tree;
     457              :     }
     458              :   else
     459              :     {
     460        67971 :       TYPE_MAIN_VARIANT (new_tree) = new_tree;
     461        67971 :       TYPE_NEXT_VARIANT (new_tree) = NULL;
     462              :     }
     463              : 
     464        68050 :   if (TYPE_STUB_DECL (type))
     465          182 :     TYPE_STUB_DECL (new_tree) = remap_decl (TYPE_STUB_DECL (type), id);
     466              : 
     467              :   /* Lazily create pointer and reference types.  */
     468        68050 :   TYPE_POINTER_TO (new_tree) = NULL;
     469        68050 :   TYPE_REFERENCE_TO (new_tree) = NULL;
     470              : 
     471              :   /* Copy all types that may contain references to local variables; be sure to
     472              :      preserve sharing in between type and its main variant when possible.  */
     473        68050 :   switch (TREE_CODE (new_tree))
     474              :     {
     475        32689 :     case INTEGER_TYPE:
     476        32689 :     case REAL_TYPE:
     477        32689 :     case FIXED_POINT_TYPE:
     478        32689 :     case ENUMERAL_TYPE:
     479        32689 :     case BOOLEAN_TYPE:
     480        32689 :       if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
     481              :         {
     482            0 :           gcc_checking_assert (TYPE_MIN_VALUE (type) == TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (type)));
     483            0 :           gcc_checking_assert (TYPE_MAX_VALUE (type) == TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (type)));
     484              : 
     485            0 :           TYPE_MIN_VALUE (new_tree) = TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (new_tree));
     486            0 :           TYPE_MAX_VALUE (new_tree) = TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (new_tree));
     487              :         }
     488              :       else
     489              :         {
     490        32689 :           t = TYPE_MIN_VALUE (new_tree);
     491        32689 :           if (t && TREE_CODE (t) != INTEGER_CST)
     492            0 :             walk_tree (&TYPE_MIN_VALUE (new_tree), copy_tree_body_r, id, NULL);
     493              : 
     494        32689 :           t = TYPE_MAX_VALUE (new_tree);
     495        32689 :           if (t && TREE_CODE (t) != INTEGER_CST)
     496        32689 :             walk_tree (&TYPE_MAX_VALUE (new_tree), copy_tree_body_r, id, NULL);
     497              :         }
     498              :       return new_tree;
     499              : 
     500           16 :     case FUNCTION_TYPE:
     501           16 :       if (TYPE_MAIN_VARIANT (new_tree) != new_tree
     502           16 :           && TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
     503            0 :         TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
     504              :       else
     505           16 :         TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
     506           16 :       if (TYPE_MAIN_VARIANT (new_tree) != new_tree
     507           16 :           && TYPE_ARG_TYPES (type) == TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (type)))
     508            0 :         TYPE_ARG_TYPES (new_tree) = TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (new_tree));
     509              :       else
     510           16 :         walk_tree (&TYPE_ARG_TYPES (new_tree), copy_tree_body_r, id, NULL);
     511              :       return new_tree;
     512              : 
     513        35114 :     case ARRAY_TYPE:
     514        35114 :       if (TYPE_MAIN_VARIANT (new_tree) != new_tree
     515        35114 :           && TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
     516           29 :         TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
     517              :       else
     518        35085 :         TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
     519              : 
     520        35114 :       if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
     521              :         {
     522           62 :           gcc_checking_assert (TYPE_DOMAIN (type)
     523              :                                == TYPE_DOMAIN (TYPE_MAIN_VARIANT (type)));
     524           62 :           TYPE_DOMAIN (new_tree) = TYPE_DOMAIN (TYPE_MAIN_VARIANT (new_tree));
     525              :         }
     526              :       else
     527              :         {
     528        35052 :           TYPE_DOMAIN (new_tree) = remap_type (TYPE_DOMAIN (new_tree), id);
     529              :           /* For array bounds where we have decided not to copy over the bounds
     530              :              variable which isn't used in OpenMP/OpenACC region, change them to
     531              :              an uninitialized VAR_DECL temporary.  */
     532        35052 :           if (id->adjust_array_error_bounds
     533         3751 :               && TYPE_DOMAIN (new_tree)
     534         3751 :               && TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) == error_mark_node
     535        37951 :               && TYPE_MAX_VALUE (TYPE_DOMAIN (type)) != error_mark_node)
     536              :             {
     537         2899 :               tree v = create_tmp_var (TREE_TYPE (TYPE_DOMAIN (new_tree)));
     538         2899 :               DECL_ATTRIBUTES (v)
     539         2899 :                 = tree_cons (get_identifier ("omp dummy var"), NULL_TREE,
     540         2899 :                              DECL_ATTRIBUTES (v));
     541         2899 :               TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) = v;
     542              :             }
     543              :         }
     544              :       break;
     545              : 
     546          231 :     case RECORD_TYPE:
     547          231 :     case UNION_TYPE:
     548          231 :     case QUAL_UNION_TYPE:
     549          231 :       if (TYPE_MAIN_VARIANT (type) != type
     550          231 :           && TYPE_FIELDS (type) == TYPE_FIELDS (TYPE_MAIN_VARIANT (type)))
     551           17 :         TYPE_FIELDS (new_tree) = TYPE_FIELDS (TYPE_MAIN_VARIANT (new_tree));
     552              :       else
     553              :         {
     554          214 :           tree f, nf = NULL;
     555              : 
     556          987 :           for (f = TYPE_FIELDS (new_tree); f ; f = DECL_CHAIN (f))
     557              :             {
     558          773 :               t = remap_decl (f, id);
     559          773 :               DECL_CONTEXT (t) = new_tree;
     560          773 :               DECL_CHAIN (t) = nf;
     561          773 :               nf = t;
     562              :             }
     563          214 :           TYPE_FIELDS (new_tree) = nreverse (nf);
     564              :         }
     565              :       break;
     566              : 
     567            0 :     case OFFSET_TYPE:
     568            0 :     default:
     569              :       /* Shouldn't have been thought variable sized.  */
     570            0 :       gcc_unreachable ();
     571              :     }
     572              : 
     573              :   /* All variants of type share the same size, so use the already remapped data.  */
     574        35345 :   if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
     575              :     {
     576           79 :       tree s = TYPE_SIZE (type);
     577           79 :       tree mvs = TYPE_SIZE (TYPE_MAIN_VARIANT (type));
     578           79 :       tree su = TYPE_SIZE_UNIT (type);
     579           79 :       tree mvsu = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (type));
     580           79 :       gcc_checking_assert ((TREE_CODE (s) == PLACEHOLDER_EXPR
     581              :                             && (TREE_CODE (mvs) == PLACEHOLDER_EXPR))
     582              :                            || s == mvs);
     583           79 :       gcc_checking_assert ((TREE_CODE (su) == PLACEHOLDER_EXPR
     584              :                             && (TREE_CODE (mvsu) == PLACEHOLDER_EXPR))
     585              :                            || su == mvsu);
     586           79 :       TYPE_SIZE (new_tree) = TYPE_SIZE (TYPE_MAIN_VARIANT (new_tree));
     587           79 :       TYPE_SIZE_UNIT (new_tree) = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (new_tree));
     588              :     }
     589              :   else
     590              :     {
     591        35266 :       walk_tree (&TYPE_SIZE (new_tree), copy_tree_body_r, id, NULL);
     592        35266 :       walk_tree (&TYPE_SIZE_UNIT (new_tree), copy_tree_body_r, id, NULL);
     593              :     }
     594              : 
     595              :   return new_tree;
     596              : }
     597              : 
     598              : /* Helper function for remap_type_2, called through walk_tree.  */
     599              : 
     600              : static tree
     601        31050 : remap_type_3 (tree *tp, int *walk_subtrees, void *data)
     602              : {
     603        31050 :   copy_body_data *id = (copy_body_data *) data;
     604              : 
     605        31050 :   if (TYPE_P (*tp))
     606            0 :     *walk_subtrees = 0;
     607              : 
     608        31050 :   else if (DECL_P (*tp) && remap_decl (*tp, id) != *tp)
     609            9 :     return *tp;
     610              : 
     611              :   return NULL_TREE;
     612              : }
     613              : 
     614              : /* Return true if TYPE needs to be remapped because remap_decl on any
     615              :    needed embedded decl returns something other than that decl.  */
     616              : 
     617              : static bool
     618        51969 : remap_type_2 (tree type, copy_body_data *id)
     619              : {
     620        62694 :   tree t;
     621              : 
     622              : #define RETURN_TRUE_IF_VAR(T) \
     623              :   do                                                            \
     624              :     {                                                           \
     625              :       tree _t = (T);                                            \
     626              :       if (_t)                                                   \
     627              :         {                                                       \
     628              :           if (DECL_P (_t) && remap_decl (_t, id) != _t)         \
     629              :             return true;                                        \
     630              :           if (!TYPE_SIZES_GIMPLIFIED (type)                     \
     631              :               && walk_tree (&_t, remap_type_3, id, NULL))   \
     632              :             return true;                                        \
     633              :         }                                                       \
     634              :     }                                                           \
     635              :   while (0)
     636              : 
     637        62694 :   switch (TREE_CODE (type))
     638              :     {
     639        10725 :     case POINTER_TYPE:
     640        10725 :     case REFERENCE_TYPE:
     641        10725 :     case FUNCTION_TYPE:
     642        10725 :     case METHOD_TYPE:
     643        10725 :       return remap_type_2 (TREE_TYPE (type), id);
     644              : 
     645        34935 :     case INTEGER_TYPE:
     646        34935 :     case REAL_TYPE:
     647        34935 :     case FIXED_POINT_TYPE:
     648        34935 :     case ENUMERAL_TYPE:
     649        34935 :     case BOOLEAN_TYPE:
     650        34935 :       RETURN_TRUE_IF_VAR (TYPE_MIN_VALUE (type));
     651        34935 :       RETURN_TRUE_IF_VAR (TYPE_MAX_VALUE (type));
     652        23038 :       return false;
     653              : 
     654        16559 :     case ARRAY_TYPE:
     655        16559 :       if (remap_type_2 (TREE_TYPE (type), id)
     656        32464 :           || (TYPE_DOMAIN (type) && remap_type_2 (TYPE_DOMAIN (type), id)))
     657              :         return true;
     658              :       break;
     659              : 
     660          357 :     case RECORD_TYPE:
     661          357 :     case UNION_TYPE:
     662          357 :     case QUAL_UNION_TYPE:
     663         3561 :       for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
     664         3204 :         if (TREE_CODE (t) == FIELD_DECL)
     665              :           {
     666         1242 :             RETURN_TRUE_IF_VAR (DECL_FIELD_OFFSET (t));
     667         1242 :             RETURN_TRUE_IF_VAR (DECL_SIZE (t));
     668         1242 :             RETURN_TRUE_IF_VAR (DECL_SIZE_UNIT (t));
     669         1242 :             if (TREE_CODE (type) == QUAL_UNION_TYPE)
     670            0 :               RETURN_TRUE_IF_VAR (DECL_QUALIFIER (t));
     671              :           }
     672              :       break;
     673              : 
     674              :     default:
     675              :       return false;
     676              :     }
     677              : 
     678         7802 :   RETURN_TRUE_IF_VAR (TYPE_SIZE (type));
     679         7793 :   RETURN_TRUE_IF_VAR (TYPE_SIZE_UNIT (type));
     680         7057 :   return false;
     681              : #undef RETURN_TRUE_IF_VAR
     682              : }
     683              : 
     684              : tree
     685   1414057772 : remap_type (tree type, copy_body_data *id)
     686              : {
     687   1414057772 :   tree *node;
     688   1414057772 :   tree tmp;
     689              : 
     690   1414057772 :   if (type == NULL)
     691              :     return type;
     692              : 
     693              :   /* See if we have remapped this type.  */
     694   1413703382 :   node = id->decl_map->get (type);
     695   1413703382 :   if (node)
     696    931840138 :     return *node;
     697              : 
     698              :   /* The type only needs remapping if it's variably modified.  */
     699    481863244 :   if (! variably_modified_type_p (type, id->src_fn)
     700              :       /* Don't remap if copy_decl method doesn't always return a new
     701              :          decl and for all embedded decls returns the passed in decl.  */
     702    481863244 :       || (id->dont_remap_vla_if_no_change && !remap_type_2 (type, id)))
     703              :     {
     704    481758746 :       insert_decl_map (id, type, type);
     705    481758746 :       return type;
     706              :     }
     707              : 
     708       104498 :   id->remapping_type_depth++;
     709       104498 :   tmp = remap_type_1 (type, id);
     710       104498 :   id->remapping_type_depth--;
     711              : 
     712       104498 :   return tmp;
     713              : }
     714              : 
     715              : /* Decide if DECL can be put into BLOCK_NONLOCAL_VARs.  */
     716              : 
     717              : static bool
     718     50877091 : can_be_nonlocal (tree decl, copy_body_data *id)
     719              : {
     720              :   /* We cannot duplicate function decls.  */
     721     50877091 :   if (TREE_CODE (decl) == FUNCTION_DECL)
     722              :     return true;
     723              : 
     724              :   /* Local static vars must be non-local or we get multiple declaration
     725              :      problems.  */
     726     50858228 :   if (VAR_P (decl) && !auto_var_in_fn_p (decl, id->src_fn))
     727       231675 :     return true;
     728              : 
     729              :   return false;
     730              : }
     731              : 
     732              : static tree
     733     52590864 : remap_decls (tree decls, vec<tree, va_gc> **nonlocalized_list,
     734              :              copy_body_data *id)
     735              : {
     736     52590864 :   tree old_var;
     737     52590864 :   tree new_decls = NULL_TREE;
     738              : 
     739              :   /* Remap its variables.  */
     740     98993705 :   for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
     741              :     {
     742     46402841 :       tree new_var;
     743              : 
     744     46402841 :       if (can_be_nonlocal (old_var, id))
     745              :         {
     746              :           /* We need to add this variable to the local decls as otherwise
     747              :              nothing else will do so.  */
     748       163244 :           if (VAR_P (old_var) && ! DECL_EXTERNAL (old_var) && cfun)
     749       144125 :             add_local_decl (cfun, old_var);
     750       100403 :           if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
     751       135457 :               && !DECL_IGNORED_P (old_var)
     752       298458 :               && nonlocalized_list)
     753        84401 :             vec_safe_push (*nonlocalized_list, old_var);
     754       163244 :           continue;
     755              :         }
     756              : 
     757              :       /* Remap the variable.  */
     758     46239597 :       new_var = remap_decl (old_var, id);
     759              : 
     760              :       /* If we didn't remap this variable, we can't mess with its
     761              :          TREE_CHAIN.  If we remapped this variable to the return slot, it's
     762              :          already declared somewhere else, so don't declare it here.  */
     763              : 
     764     46239597 :       if (new_var == old_var || new_var == id->retvar)
     765              :         ;
     766     41959415 :       else if (!new_var)
     767              :         {
     768            0 :           if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
     769            0 :               && !DECL_IGNORED_P (old_var)
     770            0 :               && nonlocalized_list)
     771            0 :             vec_safe_push (*nonlocalized_list, old_var);
     772              :         }
     773              :       else
     774              :         {
     775     41959415 :           gcc_assert (DECL_P (new_var));
     776     41959415 :           DECL_CHAIN (new_var) = new_decls;
     777     41959415 :           new_decls = new_var;
     778              : 
     779              :           /* Also copy value-expressions.  */
     780     41959415 :           if (VAR_P (new_var) && DECL_HAS_VALUE_EXPR_P (new_var))
     781              :             {
     782      1621134 :               tree tem = DECL_VALUE_EXPR (new_var);
     783      1621134 :               bool old_regimplify = id->regimplify;
     784      1621134 :               id->remapping_type_depth++;
     785      1621134 :               walk_tree (&tem, copy_tree_body_r, id, NULL);
     786      1621134 :               id->remapping_type_depth--;
     787      1621134 :               id->regimplify = old_regimplify;
     788      1621134 :               SET_DECL_VALUE_EXPR (new_var, tem);
     789              :             }
     790              :         }
     791              :     }
     792              : 
     793     52590864 :   return nreverse (new_decls);
     794              : }
     795              : 
     796              : /* Copy the BLOCK to contain remapped versions of the variables
     797              :    therein.  And hook the new block into the block-tree.  */
     798              : 
     799              : static void
     800     47254865 : remap_block (tree *block, copy_body_data *id)
     801              : {
     802     47254865 :   tree old_block;
     803     47254865 :   tree new_block;
     804              : 
     805              :   /* Make the new block.  */
     806     47254865 :   old_block = *block;
     807     47254865 :   new_block = make_node (BLOCK);
     808     47254865 :   TREE_USED (new_block) = TREE_USED (old_block);
     809     47254865 :   BLOCK_ABSTRACT_ORIGIN (new_block) = BLOCK_ORIGIN (old_block);
     810     47254865 :   BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block);
     811     47254865 :   BLOCK_NONLOCALIZED_VARS (new_block)
     812     47336977 :     = vec_safe_copy (BLOCK_NONLOCALIZED_VARS (old_block));
     813     47254865 :   *block = new_block;
     814              : 
     815              :   /* Remap its variables.  */
     816     94509730 :   BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block),
     817     47254865 :                                         &BLOCK_NONLOCALIZED_VARS (new_block),
     818              :                                         id);
     819              : 
     820              :   /* Remember the remapped block.  */
     821     47254865 :   insert_decl_map (id, old_block, new_block);
     822     47254865 : }
     823              : 
     824              : /* Copy the whole block tree and root it in id->block.  */
     825              : 
     826              : static tree
     827     26684537 : remap_blocks (tree block, copy_body_data *id)
     828              : {
     829     26684537 :   tree t;
     830     26684537 :   tree new_tree = block;
     831              : 
     832     26684537 :   if (!block)
     833              :     return NULL;
     834              : 
     835     26684537 :   remap_block (&new_tree, id);
     836     26684537 :   gcc_assert (new_tree != block);
     837     48496342 :   for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
     838     21811805 :     prepend_lexical_block (new_tree, remap_blocks (t, id));
     839              :   /* Blocks are in arbitrary order, but make things slightly prettier and do
     840              :      not swap order when producing a copy.  */
     841     26684537 :   BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree));
     842     26684537 :   return new_tree;
     843              : }
     844              : 
     845              : /* Remap the block tree rooted at BLOCK to nothing.  */
     846              : 
     847              : static void
     848        75063 : remap_blocks_to_null (tree block, copy_body_data *id)
     849              : {
     850        75063 :   tree t;
     851        75063 :   insert_decl_map (id, block, NULL_TREE);
     852       123218 :   for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
     853        48155 :     remap_blocks_to_null (t, id);
     854        75063 : }
     855              : 
     856              : /* Remap the location info pointed to by LOCUS.  */
     857              : 
     858              : static location_t
     859     26666311 : remap_location (location_t locus, copy_body_data *id)
     860              : {
     861     26666311 :   if (LOCATION_BLOCK (locus))
     862              :     {
     863     10780342 :       tree *n = id->decl_map->get (LOCATION_BLOCK (locus));
     864     10780342 :       gcc_assert (n);
     865     10780342 :       if (*n)
     866     10747302 :         return set_block (locus, *n);
     867              :     }
     868              : 
     869     15919009 :   locus = LOCATION_LOCUS (locus);
     870              : 
     871     15919009 :   if (locus != UNKNOWN_LOCATION && id->block)
     872           15 :     return set_block (locus, id->block);
     873              : 
     874              :   return locus;
     875              : }
     876              : 
     877              : static void
     878     37767602 : copy_statement_list (tree *tp)
     879              : {
     880     37767602 :   tree_stmt_iterator oi, ni;
     881     37767602 :   tree new_tree;
     882              : 
     883     37767602 :   new_tree = alloc_stmt_list ();
     884     37767602 :   ni = tsi_start (new_tree);
     885     37767602 :   oi = tsi_start (*tp);
     886     37767602 :   TREE_TYPE (new_tree) = TREE_TYPE (*tp);
     887     37767602 :   *tp = new_tree;
     888              : 
     889    124057058 :   for (; !tsi_end_p (oi); tsi_next (&oi))
     890              :     {
     891     86289456 :       tree stmt = tsi_stmt (oi);
     892     86289456 :       if (TREE_CODE (stmt) == STATEMENT_LIST)
     893              :         /* This copy is not redundant; tsi_link_after will smash this
     894              :            STATEMENT_LIST into the end of the one we're building, and we
     895              :            don't want to do that with the original.  */
     896        77695 :         copy_statement_list (&stmt);
     897     86289456 :       tsi_link_after (&ni, stmt, TSI_CONTINUE_LINKING);
     898              :     }
     899     37767602 : }
     900              : 
     901              : static void
     902     20570611 : copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id)
     903              : {
     904     20570611 :   tree block = BIND_EXPR_BLOCK (*tp);
     905              :   /* Copy (and replace) the statement.  */
     906     20570611 :   copy_tree_r (tp, walk_subtrees, NULL);
     907     20570611 :   if (block)
     908              :     {
     909     20570033 :       remap_block (&block, id);
     910     20570033 :       BIND_EXPR_BLOCK (*tp) = block;
     911              :     }
     912              : 
     913     20570611 :   if (BIND_EXPR_VARS (*tp))
     914              :     /* This will remap a lot of the same decls again, but this should be
     915              :        harmless.  */
     916      5335831 :     BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id);
     917     20570611 : }
     918              : 
     919              : 
     920              : /* Create a new gimple_seq by remapping all the statements in BODY
     921              :    using the inlining information in ID.  */
     922              : 
     923              : static gimple_seq
     924           67 : remap_gimple_seq (gimple_seq body, copy_body_data *id)
     925              : {
     926           67 :   gimple_stmt_iterator si;
     927           67 :   gimple_seq new_body = NULL;
     928              : 
     929           67 :   for (si = gsi_start (body); !gsi_end_p (si); gsi_next (&si))
     930              :     {
     931            0 :       gimple_seq new_stmts = remap_gimple_stmt (gsi_stmt (si), id);
     932            0 :       gimple_seq_add_seq (&new_body, new_stmts);
     933              :     }
     934              : 
     935           67 :   return new_body;
     936              : }
     937              : 
     938              : 
     939              : /* Copy a GIMPLE_BIND statement STMT, remapping all the symbols in its
     940              :    block using the mapping information in ID.  */
     941              : 
     942              : static gimple *
     943            0 : copy_gimple_bind (gbind *stmt, copy_body_data *id)
     944              : {
     945            0 :   gimple *new_bind;
     946            0 :   tree new_block, new_vars;
     947            0 :   gimple_seq body, new_body;
     948              : 
     949              :   /* Copy the statement.  Note that we purposely don't use copy_stmt
     950              :      here because we need to remap statements as we copy.  */
     951            0 :   body = gimple_bind_body (stmt);
     952            0 :   new_body = remap_gimple_seq (body, id);
     953              : 
     954            0 :   new_block = gimple_bind_block (stmt);
     955            0 :   if (new_block)
     956            0 :     remap_block (&new_block, id);
     957              : 
     958              :   /* This will remap a lot of the same decls again, but this should be
     959              :      harmless.  */
     960            0 :   new_vars = gimple_bind_vars (stmt);
     961            0 :   if (new_vars)
     962            0 :     new_vars = remap_decls (new_vars, NULL, id);
     963              : 
     964            0 :   new_bind = gimple_build_bind (new_vars, new_body, new_block);
     965              : 
     966            0 :   return new_bind;
     967              : }
     968              : 
     969              : /* Return true if DECL is a parameter or a SSA_NAME for a parameter.  */
     970              : 
     971              : static bool
     972        31946 : is_parm (tree decl)
     973              : {
     974        31946 :   if (TREE_CODE (decl) == SSA_NAME)
     975              :     {
     976        30064 :       decl = SSA_NAME_VAR (decl);
     977              :       if (!decl)
     978              :         return false;
     979              :     }
     980              : 
     981        19385 :   return (TREE_CODE (decl) == PARM_DECL);
     982              : }
     983              : 
     984              : /* Copy the TREE_THIS_NOTRAP flag from OLD to T if it is appropriate to do so.
     985              :    T and OLD must be both either INDIRECT_REF or MEM_REF.  */
     986              : 
     987              : static void
     988     21685080 : maybe_copy_this_notrap (copy_body_data *id, tree t, tree old)
     989              : {
     990     21685080 :   gcc_assert (TREE_CODE (t) == TREE_CODE (old));
     991              : 
     992              :   /* We cannot blindly propagate the TREE_THIS_NOTRAP flag if we have remapped
     993              :      a parameter as the property might be valid only for the parameter itself,
     994              :      typically when it is passed by reference.  But we propagate the flag when
     995              :      this is the dereference of an entire object done in a type that has self-
     996              :      referential size, to avoid the static size check in tree_could_trap_p.  */
     997     21685080 :   if (TREE_THIS_NOTRAP (old)
     998     21685080 :       && (!is_parm (TREE_OPERAND (old, 0))
     999        13614 :           || (!id->transform_parameter && is_parm (TREE_OPERAND (t, 0)))
    1000         9994 :           || ((TREE_CODE (t) == INDIRECT_REF
    1001         9994 :                || integer_zerop (TREE_OPERAND (t, 1)))
    1002         9994 :               && TREE_CODE (TREE_OPERAND (t, 0)) == ADDR_EXPR
    1003         9329 :               && DECL_P (TREE_OPERAND (TREE_OPERAND (t, 0), 0))
    1004         9329 :               && type_contains_placeholder_p (TREE_TYPE (t)))))
    1005        18332 :     TREE_THIS_NOTRAP (t) = 1;
    1006     21685080 : }
    1007              : 
    1008              : /* Remap the dependence CLIQUE from the source to the destination function
    1009              :    as specified in ID.  */
    1010              : 
    1011              : static unsigned short
    1012      2838204 : remap_dependence_clique (copy_body_data *id, unsigned short clique)
    1013              : {
    1014      2838204 :   if (clique == 0 || processing_debug_stmt)
    1015              :     return 0;
    1016      2803881 :   if (!id->dependence_map)
    1017       733457 :     id->dependence_map = new hash_map<dependence_hash, unsigned short>;
    1018      2803881 :   bool existed;
    1019      2803881 :   unsigned short &newc = id->dependence_map->get_or_insert (clique, &existed);
    1020      2803881 :   if (!existed)
    1021              :     {
    1022              :       /* Clique 1 is reserved for local ones set by PTA.  */
    1023      1100819 :       if (cfun->last_clique == 0)
    1024       340306 :         cfun->last_clique = 1;
    1025      2201638 :       newc = get_new_clique (cfun);
    1026              :     }
    1027      2803881 :   return newc;
    1028              : }
    1029              : 
    1030              : /* Remap the GIMPLE operand pointed to by *TP.  DATA is really a
    1031              :    'struct walk_stmt_info *'.  DATA->INFO is a 'copy_body_data *'.
    1032              :    WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
    1033              :    recursing into the children nodes of *TP.  */
    1034              : 
    1035              : static tree
    1036    212509986 : remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
    1037              : {
    1038    212509986 :   struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
    1039    212509986 :   copy_body_data *id = (copy_body_data *) wi_p->info;
    1040    212509986 :   tree fn = id->src_fn;
    1041              : 
    1042              :   /* For recursive invocations this is no longer the LHS itself.  */
    1043    212509986 :   bool is_lhs = wi_p->is_lhs;
    1044    212509986 :   wi_p->is_lhs = false;
    1045              : 
    1046    212509986 :   if (TREE_CODE (*tp) == SSA_NAME)
    1047              :     {
    1048     67631228 :       *tp = remap_ssa_name (*tp, id);
    1049     67631228 :       *walk_subtrees = 0;
    1050     67631228 :       if (is_lhs)
    1051     16821869 :         SSA_NAME_DEF_STMT (*tp) = wi_p->stmt;
    1052              :       return NULL;
    1053              :     }
    1054    144878758 :   else if (auto_var_in_fn_p (*tp, fn))
    1055              :     {
    1056              :       /* Local variables and labels need to be replaced by equivalent
    1057              :          variables.  We don't want to copy static variables; there's
    1058              :          only one of those, no matter how many times we inline the
    1059              :          containing function.  Similarly for globals from an outer
    1060              :          function.  */
    1061     47331685 :       tree new_decl;
    1062              : 
    1063              :       /* Remap the declaration.  */
    1064     47331685 :       new_decl = remap_decl (*tp, id);
    1065     47331685 :       gcc_assert (new_decl);
    1066              :       /* Replace this variable with the copy.  */
    1067     47331685 :       STRIP_TYPE_NOPS (new_decl);
    1068              :       /* ???  The C++ frontend uses void * pointer zero to initialize
    1069              :          any other type.  This confuses the middle-end type verification.
    1070              :          As cloned bodies do not go through gimplification again the fixup
    1071              :          there doesn't trigger.  */
    1072     47331685 :       if (TREE_CODE (new_decl) == INTEGER_CST
    1073     47331685 :           && !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
    1074            0 :         new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
    1075     47331685 :       *tp = new_decl;
    1076     47331685 :       *walk_subtrees = 0;
    1077              :     }
    1078     97547073 :   else if (TREE_CODE (*tp) == STATEMENT_LIST)
    1079            0 :     gcc_unreachable ();
    1080     97547073 :   else if (TREE_CODE (*tp) == SAVE_EXPR)
    1081            0 :     gcc_unreachable ();
    1082     97547073 :   else if (TREE_CODE (*tp) == LABEL_DECL
    1083     97547073 :            && (!DECL_CONTEXT (*tp)
    1084          461 :                || decl_function_context (*tp) == id->src_fn))
    1085              :     /* These may need to be remapped for EH handling.  */
    1086            0 :     *tp = remap_decl (*tp, id);
    1087     97547073 :   else if (TREE_CODE (*tp) == FIELD_DECL)
    1088              :     {
    1089              :       /* If the enclosing record type is variably_modified_type_p, the field
    1090              :          has already been remapped.  Otherwise, it need not be.  */
    1091     16924391 :       tree *n = id->decl_map->get (*tp);
    1092     16924391 :       if (n)
    1093           83 :         *tp = *n;
    1094     16924391 :       *walk_subtrees = 0;
    1095              :     }
    1096     80622682 :   else if (TYPE_P (*tp))
    1097              :     /* Types may need remapping as well.  */
    1098            0 :     *tp = remap_type (*tp, id);
    1099     80622682 :   else if (CONSTANT_CLASS_P (*tp))
    1100              :     {
    1101              :       /* If this is a constant, we have to copy the node iff the type
    1102              :          will be remapped.  copy_tree_r will not copy a constant.  */
    1103     13968856 :       tree new_type = remap_type (TREE_TYPE (*tp), id);
    1104              : 
    1105     13968856 :       if (new_type == TREE_TYPE (*tp))
    1106     13965667 :         *walk_subtrees = 0;
    1107              : 
    1108         3189 :       else if (TREE_CODE (*tp) == INTEGER_CST)
    1109         3189 :         *tp = wide_int_to_tree (new_type, wi::to_wide (*tp));
    1110              :       else
    1111              :         {
    1112            0 :           *tp = copy_node (*tp);
    1113            0 :           TREE_TYPE (*tp) = new_type;
    1114              :         }
    1115              :     }
    1116              :   else
    1117              :     {
    1118              :       /* Otherwise, just copy the node.  Note that copy_tree_r already
    1119              :          knows not to copy VAR_DECLs, etc., so this is safe.  */
    1120              : 
    1121     66653826 :       if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
    1122              :         {
    1123              :           /* We need to re-canonicalize MEM_REFs from inline substitutions
    1124              :              that can happen when a pointer argument is an ADDR_EXPR.
    1125              :              Recurse here manually to allow that.  */
    1126     14302520 :           tree ptr = TREE_OPERAND (*tp, 0);
    1127     14302520 :           tree type = remap_type (TREE_TYPE (*tp), id);
    1128     14302520 :           tree old = *tp;
    1129     14302520 :           walk_tree (&ptr, remap_gimple_op_r, data, NULL);
    1130     14302520 :           *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
    1131     14302520 :           TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
    1132     14302520 :           TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
    1133     14302520 :           copy_warning (*tp, old);
    1134     14302520 :           if (MR_DEPENDENCE_CLIQUE (old) != 0)
    1135              :             {
    1136      2751903 :               MR_DEPENDENCE_CLIQUE (*tp)
    1137      2751903 :                 = remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
    1138      2751903 :               MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
    1139              :             }
    1140     14302520 :           maybe_copy_this_notrap (id, *tp, old);
    1141     14302520 :           REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
    1142     14302520 :           *walk_subtrees = 0;
    1143     14302520 :           return NULL;
    1144              :         }
    1145              : 
    1146              :       /* Here is the "usual case".  Copy this tree node, and then
    1147              :          tweak some special cases.  */
    1148     52351306 :       copy_tree_r (tp, walk_subtrees, NULL);
    1149              : 
    1150     52351306 :       if (TREE_CODE (*tp) != OMP_CLAUSE)
    1151     52351306 :         TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
    1152              : 
    1153     52351306 :       if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
    1154              :         {
    1155              :           /* The copied TARGET_EXPR has never been expanded, even if the
    1156              :              original node was expanded already.  */
    1157            0 :           TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
    1158            0 :           TREE_OPERAND (*tp, 3) = NULL_TREE;
    1159              :         }
    1160     52351306 :       else if (TREE_CODE (*tp) == ADDR_EXPR)
    1161              :         {
    1162              :           /* Variable substitution need not be simple.  In particular,
    1163              :              the MEM_REF substitution above.  Make sure that
    1164              :              TREE_CONSTANT and friends are up-to-date.  */
    1165     16108920 :           int invariant = is_gimple_min_invariant (*tp);
    1166     16108920 :           walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL);
    1167     16108920 :           recompute_tree_invariant_for_addr_expr (*tp);
    1168              : 
    1169              :           /* If this used to be invariant, but is not any longer,
    1170              :              then regimplification is probably needed.  */
    1171     16108920 :           if (invariant && !is_gimple_min_invariant (*tp))
    1172         4628 :             id->regimplify = true;
    1173              : 
    1174     16108920 :           *walk_subtrees = 0;
    1175              :         }
    1176     36242386 :       else if (TREE_CODE (*tp) == OMP_NEXT_VARIANT)
    1177              :         {
    1178              :           /* Neither operand is interesting, and walking the selector
    1179              :              causes problems because it's not an expression.  */
    1180          288 :           gcc_assert (TREE_CODE (TREE_OPERAND (*tp, 0)) == INTEGER_CST);
    1181          288 :           *walk_subtrees = 0;
    1182              :         }
    1183              :     }
    1184              : 
    1185              :   /* Update the TREE_BLOCK for the cloned expr.  */
    1186    130576238 :   if (EXPR_P (*tp))
    1187              :     {
    1188     34546460 :       tree new_block = id->remapping_type_depth == 0 ? id->block : NULL;
    1189     34546460 :       tree old_block = TREE_BLOCK (*tp);
    1190     34546460 :       if (old_block)
    1191              :         {
    1192     19937182 :           tree *n;
    1193     19937182 :           n = id->decl_map->get (TREE_BLOCK (*tp));
    1194     19937182 :           if (n)
    1195     19932161 :             new_block = *n;
    1196              :         }
    1197     34546460 :       TREE_SET_BLOCK (*tp, new_block);
    1198              :     }
    1199              : 
    1200              :   /* Keep iterating.  */
    1201              :   return NULL_TREE;
    1202              : }
    1203              : 
    1204              : 
    1205              : /* Called from copy_body_id via walk_tree.  DATA is really a
    1206              :    `copy_body_data *'.  */
    1207              : 
    1208              : tree
    1209   1432405920 : copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
    1210              : {
    1211   1432405920 :   copy_body_data *id = (copy_body_data *) data;
    1212   1432405920 :   tree fn = id->src_fn;
    1213   1432405920 :   tree new_block;
    1214              : 
    1215              :   /* Begin by recognizing trees that we'll completely rewrite for the
    1216              :      inlining context.  Our output for these trees is completely
    1217              :      different from out input (e.g. RETURN_EXPR is deleted, and morphs
    1218              :      into an edge).  Further down, we'll handle trees that get
    1219              :      duplicated and/or tweaked.  */
    1220              : 
    1221              :   /* When requested, RETURN_EXPRs should be transformed to just the
    1222              :      contained MODIFY_EXPR.  The branch semantics of the return will
    1223              :      be handled elsewhere by manipulating the CFG rather than a statement.  */
    1224   1432405920 :   if (TREE_CODE (*tp) == RETURN_EXPR && id->transform_return_to_modify)
    1225              :     {
    1226            0 :       tree assignment = TREE_OPERAND (*tp, 0);
    1227              : 
    1228              :       /* If we're returning something, just turn that into an
    1229              :          assignment into the equivalent of the original RESULT_DECL.
    1230              :          If the "assignment" is just the result decl, the result
    1231              :          decl has already been set (e.g. a recent "foo (&result_decl,
    1232              :          ...)"); just toss the entire RETURN_EXPR.  */
    1233            0 :       if (assignment && TREE_CODE (assignment) == MODIFY_EXPR)
    1234              :         {
    1235              :           /* Replace the RETURN_EXPR with (a copy of) the
    1236              :              MODIFY_EXPR hanging underneath.  */
    1237            0 :           *tp = copy_node (assignment);
    1238              :         }
    1239              :       else /* Else the RETURN_EXPR returns no value.  */
    1240              :         {
    1241            0 :           *tp = NULL;
    1242            0 :           return (tree) (void *)1;
    1243              :         }
    1244            0 :     }
    1245   1432405920 :   else if (TREE_CODE (*tp) == SSA_NAME)
    1246              :     {
    1247      4058027 :       *tp = remap_ssa_name (*tp, id);
    1248      4058027 :       *walk_subtrees = 0;
    1249      4058027 :       return NULL;
    1250              :     }
    1251              : 
    1252              :   /* Local variables and labels need to be replaced by equivalent
    1253              :      variables.  We don't want to copy static variables; there's only
    1254              :      one of those, no matter how many times we inline the containing
    1255              :      function.  Similarly for globals from an outer function.  */
    1256   1428347893 :   else if (auto_var_in_fn_p (*tp, fn))
    1257              :     {
    1258    175848316 :       tree new_decl;
    1259              : 
    1260              :       /* Remap the declaration.  */
    1261    175848316 :       new_decl = remap_decl (*tp, id);
    1262    175848316 :       gcc_assert (new_decl);
    1263              :       /* Replace this variable with the copy.  */
    1264    175848316 :       STRIP_TYPE_NOPS (new_decl);
    1265    175848316 :       *tp = new_decl;
    1266    175848316 :       *walk_subtrees = 0;
    1267              :     }
    1268   1252499577 :   else if (TREE_CODE (*tp) == STATEMENT_LIST)
    1269     37689709 :     copy_statement_list (tp);
    1270   1214809868 :   else if (TREE_CODE (*tp) == SAVE_EXPR
    1271   1214119491 :            || TREE_CODE (*tp) == TARGET_EXPR)
    1272     15698316 :     remap_save_expr (tp, id->decl_map, walk_subtrees);
    1273   1199111552 :   else if (TREE_CODE (*tp) == LABEL_DECL
    1274   1199111552 :            && (! DECL_CONTEXT (*tp)
    1275            7 :                || decl_function_context (*tp) == id->src_fn))
    1276              :     /* These may need to be remapped for EH handling.  */
    1277            0 :     *tp = remap_decl (*tp, id);
    1278   1199111552 :   else if (TREE_CODE (*tp) == BIND_EXPR)
    1279     20570611 :     copy_bind_expr (tp, walk_subtrees, id);
    1280              :   /* Types may need remapping as well.  */
    1281   1178540941 :   else if (TYPE_P (*tp))
    1282       223369 :     *tp = remap_type (*tp, id);
    1283              : 
    1284              :   /* If this is a constant, we have to copy the node iff the type will be
    1285              :      remapped.  copy_tree_r will not copy a constant.  */
    1286   1178317572 :   else if (CONSTANT_CLASS_P (*tp))
    1287              :     {
    1288    361673153 :       tree new_type = remap_type (TREE_TYPE (*tp), id);
    1289              : 
    1290    361673153 :       if (new_type == TREE_TYPE (*tp))
    1291    361671998 :         *walk_subtrees = 0;
    1292              : 
    1293         1155 :       else if (TREE_CODE (*tp) == INTEGER_CST)
    1294         1155 :         *tp = wide_int_to_tree (new_type, wi::to_wide (*tp));
    1295              :       else
    1296              :         {
    1297            0 :           *tp = copy_node (*tp);
    1298            0 :           TREE_TYPE (*tp) = new_type;
    1299              :         }
    1300              :     }
    1301              : 
    1302              :   /* Otherwise, just copy the node.  Note that copy_tree_r already
    1303              :      knows not to copy VAR_DECLs, etc., so this is safe.  */
    1304              :   else
    1305              :     {
    1306              :       /* Here we handle trees that are not completely rewritten.
    1307              :          First we detect some inlining-induced bogosities for
    1308              :          discarding.  */
    1309    816644419 :       if (TREE_CODE (*tp) == MODIFY_EXPR
    1310     10960876 :           && TREE_OPERAND (*tp, 0) == TREE_OPERAND (*tp, 1)
    1311    816644440 :           && (auto_var_in_fn_p (TREE_OPERAND (*tp, 0), fn)))
    1312              :         {
    1313              :           /* Some assignments VAR = VAR; don't generate any rtl code
    1314              :              and thus don't count as variable modification.  Avoid
    1315              :              keeping bogosities like 0 = 0.  */
    1316            0 :           tree decl = TREE_OPERAND (*tp, 0), value;
    1317            0 :           tree *n;
    1318              : 
    1319            0 :           n = id->decl_map->get (decl);
    1320            0 :           if (n)
    1321              :             {
    1322            0 :               value = *n;
    1323            0 :               STRIP_TYPE_NOPS (value);
    1324            0 :               if (TREE_CONSTANT (value) || TREE_READONLY (value))
    1325              :                 {
    1326            0 :                   *tp = build_empty_stmt (EXPR_LOCATION (*tp));
    1327            0 :                   return copy_tree_body_r (tp, walk_subtrees, data);
    1328              :                 }
    1329              :             }
    1330              :         }
    1331    816644419 :       else if (INDIRECT_REF_P (*tp))
    1332              :         {
    1333              :           /* Get rid of *& from inline substitutions that can happen when a
    1334              :              pointer argument is an ADDR_EXPR.  */
    1335     44361250 :           tree decl = TREE_OPERAND (*tp, 0);
    1336     44361250 :           tree *n = id->decl_map->get (decl);
    1337     44361250 :           if (n)
    1338              :             {
    1339              :               /* If we happen to get an ADDR_EXPR in n->value, strip
    1340              :                  it manually here as we'll eventually get ADDR_EXPRs
    1341              :                  which lie about their types pointed to.  In this case
    1342              :                  build_fold_indirect_ref wouldn't strip the INDIRECT_REF,
    1343              :                  but we absolutely rely on that.  As fold_indirect_ref
    1344              :                  does other useful transformations, try that first, though.  */
    1345      7310361 :               tree type = TREE_TYPE (*tp);
    1346      7310361 :               tree ptr = id->do_not_unshare ? *n : unshare_expr (*n);
    1347      7310361 :               tree old = *tp;
    1348      7310361 :               *tp = id->do_not_fold ? NULL : gimple_fold_indirect_ref (ptr);
    1349      7310361 :               if (! *tp)
    1350              :                 {
    1351      7310226 :                   type = remap_type (type, id);
    1352      7310226 :                   if (TREE_CODE (ptr) == ADDR_EXPR && !id->do_not_fold)
    1353              :                     {
    1354            0 :                       *tp
    1355            0 :                         = fold_indirect_ref_1 (EXPR_LOCATION (ptr), type, ptr);
    1356              :                       /* ???  We should either assert here or build
    1357              :                          a VIEW_CONVERT_EXPR instead of blindly leaking
    1358              :                          incompatible types to our IL.  */
    1359            0 :                       if (! *tp)
    1360            0 :                         *tp = TREE_OPERAND (ptr, 0);
    1361              :                     }
    1362              :                   else
    1363              :                     {
    1364      7310226 :                       *tp = build1 (INDIRECT_REF, type, ptr);
    1365      7310226 :                       TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
    1366      7310226 :                       TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
    1367      7310226 :                       TREE_READONLY (*tp) = TREE_READONLY (old);
    1368      7310226 :                       maybe_copy_this_notrap (id, *tp, old);
    1369              :                     }
    1370              :                 }
    1371      7310361 :               *walk_subtrees = 0;
    1372      7310361 :               return NULL;
    1373              :             }
    1374              :         }
    1375    772283169 :       else if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
    1376              :         {
    1377              :           /* We need to re-canonicalize MEM_REFs from inline substitutions
    1378              :              that can happen when a pointer argument is an ADDR_EXPR.
    1379              :              Recurse here manually to allow that.  */
    1380        72334 :           tree ptr = TREE_OPERAND (*tp, 0);
    1381        72334 :           tree type = remap_type (TREE_TYPE (*tp), id);
    1382        72334 :           tree old = *tp;
    1383        72334 :           walk_tree (&ptr, copy_tree_body_r, data, NULL);
    1384        72334 :           *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
    1385        72334 :           TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
    1386        72334 :           TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
    1387        72334 :           copy_warning (*tp, old);
    1388        72334 :           if (MR_DEPENDENCE_CLIQUE (old) != 0)
    1389              :             {
    1390         5878 :               MR_DEPENDENCE_CLIQUE (*tp)
    1391         5878 :                 = remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
    1392         5878 :               MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
    1393              :             }
    1394        72334 :           maybe_copy_this_notrap (id, *tp, old);
    1395        72334 :           REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
    1396        72334 :           *walk_subtrees = 0;
    1397        72334 :           return NULL;
    1398              :         }
    1399              : 
    1400              :       /* Here is the "usual case".  Copy this tree node, and then
    1401              :          tweak some special cases.  */
    1402    809261724 :       copy_tree_r (tp, walk_subtrees, NULL);
    1403              : 
    1404              :       /* If EXPR has block defined, map it to newly constructed block.
    1405              :          When inlining we want EXPRs without block appear in the block
    1406              :          of function call if we are not remapping a type.  */
    1407    809261724 :       if (EXPR_P (*tp))
    1408              :         {
    1409    706022525 :           new_block = id->remapping_type_depth == 0 ? id->block : NULL;
    1410    706022525 :           if (TREE_BLOCK (*tp))
    1411              :             {
    1412        14023 :               tree *n;
    1413        14023 :               n = id->decl_map->get (TREE_BLOCK (*tp));
    1414        14023 :               if (n)
    1415        14023 :                 new_block = *n;
    1416              :             }
    1417    706022525 :           TREE_SET_BLOCK (*tp, new_block);
    1418              :         }
    1419              : 
    1420    809261724 :       if (TREE_CODE (*tp) != OMP_CLAUSE)
    1421    809261633 :         TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
    1422              : 
    1423              :       /* The copied TARGET_EXPR has never been expanded, even if the
    1424              :          original node was expanded already.  */
    1425    809261724 :       if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
    1426              :         {
    1427            0 :           TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
    1428            0 :           TREE_OPERAND (*tp, 3) = NULL_TREE;
    1429              :         }
    1430              : 
    1431              :       /* Variable substitution need not be simple.  In particular, the
    1432              :          INDIRECT_REF substitution above.  Make sure that TREE_CONSTANT
    1433              :          and friends are up-to-date.  */
    1434    809261724 :       else if (TREE_CODE (*tp) == ADDR_EXPR)
    1435              :         {
    1436     80072409 :           int invariant = is_gimple_min_invariant (*tp);
    1437     80072409 :           walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
    1438              : 
    1439              :           /* Handle the case where we substituted an INDIRECT_REF
    1440              :              into the operand of the ADDR_EXPR.  */
    1441     80072409 :           if (INDIRECT_REF_P (TREE_OPERAND (*tp, 0))
    1442     80072409 :               && !id->do_not_fold)
    1443              :             {
    1444          195 :               tree t = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
    1445          195 :               if (TREE_TYPE (t) != TREE_TYPE (*tp))
    1446          195 :                 t = fold_convert (remap_type (TREE_TYPE (*tp), id), t);
    1447          195 :               *tp = t;
    1448              :             }
    1449              :           else
    1450     80072214 :             recompute_tree_invariant_for_addr_expr (*tp);
    1451              : 
    1452              :           /* If this used to be invariant, but is not any longer,
    1453              :              then regimplification is probably needed.  */
    1454     80072409 :           if (invariant && !is_gimple_min_invariant (*tp))
    1455           19 :             id->regimplify = true;
    1456              : 
    1457     80072409 :           *walk_subtrees = 0;
    1458              :         }
    1459    729189315 :       else if (TREE_CODE (*tp) == OMP_CLAUSE
    1460    729189315 :                && (OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_AFFINITY
    1461           76 :                    || OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_DEPEND))
    1462              :         {
    1463           30 :           tree t = OMP_CLAUSE_DECL (*tp);
    1464           30 :           if (t && OMP_ITERATOR_DECL_P (t))
    1465              :             {
    1466           18 :               *walk_subtrees = 0;
    1467           18 :               OMP_CLAUSE_DECL (*tp) = copy_node (t);
    1468           18 :               t = OMP_CLAUSE_DECL (*tp);
    1469           18 :               TREE_PURPOSE (t) = copy_node (TREE_PURPOSE (t));
    1470          108 :               for (int i = 0; i <= 4; i++)
    1471           90 :                 walk_tree (&TREE_VEC_ELT (TREE_PURPOSE (t), i),
    1472              :                            copy_tree_body_r, id, NULL);
    1473           18 :               if (OMP_ITERATOR_BLOCK (TREE_PURPOSE (t)))
    1474           18 :                 remap_block (&OMP_ITERATOR_BLOCK (TREE_PURPOSE (t)), id);
    1475           18 :               walk_tree (&TREE_VALUE (t), copy_tree_body_r, id, NULL);
    1476              :             }
    1477              :         }
    1478              :     }
    1479              : 
    1480              :   /* Keep iterating.  */
    1481              :   return NULL_TREE;
    1482              : }
    1483              : 
    1484              : /* Helper for remap_gimple_stmt.  Given an EH region number for the
    1485              :    source function, map that to the duplicate EH region number in
    1486              :    the destination function.  */
    1487              : 
    1488              : static int
    1489       102195 : remap_eh_region_nr (int old_nr, copy_body_data *id)
    1490              : {
    1491       102195 :   eh_region old_r, new_r;
    1492              : 
    1493       102195 :   old_r = get_eh_region_from_number_fn (id->src_cfun, old_nr);
    1494       102195 :   new_r = static_cast<eh_region> (*id->eh_map->get (old_r));
    1495              : 
    1496       102195 :   return new_r->index;
    1497              : }
    1498              : 
    1499              : /* Similar, but operate on INTEGER_CSTs.  */
    1500              : 
    1501              : static tree
    1502         9735 : remap_eh_region_tree_nr (tree old_t_nr, copy_body_data *id)
    1503              : {
    1504         9735 :   int old_nr, new_nr;
    1505              : 
    1506         9735 :   old_nr = tree_to_shwi (old_t_nr);
    1507         9735 :   new_nr = remap_eh_region_nr (old_nr, id);
    1508              : 
    1509         9735 :   return build_int_cst (integer_type_node, new_nr);
    1510              : }
    1511              : 
    1512              : /* Helper for copy_bb.  Remap statement STMT using the inlining
    1513              :    information in ID.  Return the new statement copy.  */
    1514              : 
    1515              : static gimple_seq
    1516     98598216 : remap_gimple_stmt (gimple *stmt, copy_body_data *id)
    1517              : {
    1518     98598216 :   gimple *copy = NULL;
    1519     98598216 :   struct walk_stmt_info wi;
    1520     98598216 :   bool skip_first = false;
    1521     98598216 :   gimple_seq stmts = NULL;
    1522              : 
    1523     98598216 :   if (is_gimple_debug (stmt)
    1524     98598216 :       && (gimple_debug_nonbind_marker_p (stmt)
    1525     14038228 :           ? !DECL_STRUCT_FUNCTION (id->dst_fn)->debug_nonbind_markers
    1526     47993531 :           : !opt_for_fn (id->dst_fn, flag_var_tracking_assignments)))
    1527              :     return NULL;
    1528              : 
    1529     98566342 :   if (!is_gimple_debug (stmt)
    1530     36566457 :       && id->param_body_adjs
    1531    102285089 :       && id->param_body_adjs->m_dead_stmts.contains (stmt))
    1532              :     {
    1533         6832 :       tree *dval = id->param_body_adjs->m_dead_stmt_debug_equiv.get (stmt);
    1534         6832 :       if (!dval)
    1535              :         return NULL;
    1536              : 
    1537         3340 :       gcc_assert (is_gimple_assign (stmt));
    1538         3340 :       tree lhs = gimple_assign_lhs (stmt);
    1539         3340 :       tree *dvar = id->param_body_adjs->m_dead_ssa_debug_equiv.get (lhs);
    1540         3340 :       gdebug *bind = gimple_build_debug_bind (*dvar, *dval, stmt);
    1541         3340 :       if (id->reset_location)
    1542            0 :         gimple_set_location (bind, input_location);
    1543         3340 :       id->debug_stmts.safe_push (bind);
    1544         3340 :       gimple_seq_add_stmt_without_update (&stmts, bind);
    1545         3340 :       return stmts;
    1546              :     }
    1547              : 
    1548              :   /* Begin by recognizing trees that we'll completely rewrite for the
    1549              :      inlining context.  Our output for these trees is completely
    1550              :      different from our input (e.g. RETURN_EXPR is deleted and morphs
    1551              :      into an edge).  Further down, we'll handle trees that get
    1552              :      duplicated and/or tweaked.  */
    1553              : 
    1554              :   /* When requested, GIMPLE_RETURN should be transformed to just the
    1555              :      contained GIMPLE_ASSIGN.  The branch semantics of the return will
    1556              :      be handled elsewhere by manipulating the CFG rather than the
    1557              :      statement.  */
    1558     98559510 :   if (gimple_code (stmt) == GIMPLE_RETURN && id->transform_return_to_modify)
    1559              :     {
    1560      4607572 :       tree retval = gimple_return_retval (as_a <greturn *> (stmt));
    1561              : 
    1562              :       /* If we're returning something, just turn that into an
    1563              :          assignment to the equivalent of the original RESULT_DECL.
    1564              :          If RETVAL is just the result decl, the result decl has
    1565              :          already been set (e.g. a recent "foo (&result_decl, ...)");
    1566              :          just toss the entire GIMPLE_RETURN.  Likewise for when the
    1567              :          call doesn't want the return value.  */
    1568      4607572 :       if (retval
    1569      4607572 :           && (TREE_CODE (retval) != RESULT_DECL
    1570      2411398 :               && (!id->call_stmt
    1571      2411398 :                   || gimple_call_lhs (id->call_stmt) != NULL_TREE)
    1572      2276425 :               && (TREE_CODE (retval) != SSA_NAME
    1573      1793125 :                   || ! SSA_NAME_VAR (retval)
    1574       425314 :                   || TREE_CODE (SSA_NAME_VAR (retval)) != RESULT_DECL)))
    1575              :         {
    1576      4386682 :           copy = gimple_build_assign (id->do_not_unshare
    1577      2193341 :                                       ? id->retvar : unshare_expr (id->retvar),
    1578              :                                       retval);
    1579              :           /* id->retvar is already substituted.  Skip it on later remapping.  */
    1580      2193341 :           skip_first = true;
    1581              :         }
    1582              :       else
    1583              :         return NULL;
    1584              :     }
    1585     93951938 :   else if (gimple_has_substatements (stmt))
    1586              :     {
    1587           67 :       gimple_seq s1, s2;
    1588              : 
    1589              :       /* When cloning bodies from the C++ front end, we will be handed bodies
    1590              :          in High GIMPLE form.  Handle here all the High GIMPLE statements that
    1591              :          have embedded statements.  */
    1592           67 :       switch (gimple_code (stmt))
    1593              :         {
    1594            0 :         case GIMPLE_BIND:
    1595            0 :           copy = copy_gimple_bind (as_a <gbind *> (stmt), id);
    1596            0 :           break;
    1597              : 
    1598            0 :         case GIMPLE_CATCH:
    1599            0 :           {
    1600            0 :             gcatch *catch_stmt = as_a <gcatch *> (stmt);
    1601            0 :             s1 = remap_gimple_seq (gimple_catch_handler (catch_stmt), id);
    1602            0 :             copy = gimple_build_catch (gimple_catch_types (catch_stmt), s1);
    1603              :           }
    1604            0 :           break;
    1605              : 
    1606            0 :         case GIMPLE_EH_FILTER:
    1607            0 :           s1 = remap_gimple_seq (gimple_eh_filter_failure (stmt), id);
    1608            0 :           copy = gimple_build_eh_filter (gimple_eh_filter_types (stmt), s1);
    1609            0 :           break;
    1610              : 
    1611            0 :         case GIMPLE_TRY:
    1612            0 :           s1 = remap_gimple_seq (gimple_try_eval (stmt), id);
    1613            0 :           s2 = remap_gimple_seq (gimple_try_cleanup (stmt), id);
    1614            0 :           copy = gimple_build_try (s1, s2, gimple_try_kind (stmt));
    1615            0 :           break;
    1616              : 
    1617            0 :         case GIMPLE_WITH_CLEANUP_EXPR:
    1618            0 :           s1 = remap_gimple_seq (gimple_wce_cleanup (stmt), id);
    1619            0 :           copy = gimple_build_wce (s1);
    1620            0 :           break;
    1621              : 
    1622            0 :         case GIMPLE_OMP_PARALLEL:
    1623            0 :           {
    1624            0 :             gomp_parallel *omp_par_stmt = as_a <gomp_parallel *> (stmt);
    1625            0 :             s1 = remap_gimple_seq (gimple_omp_body (omp_par_stmt), id);
    1626            0 :             copy = gimple_build_omp_parallel
    1627            0 :                      (s1,
    1628              :                       gimple_omp_parallel_clauses (omp_par_stmt),
    1629              :                       gimple_omp_parallel_child_fn (omp_par_stmt),
    1630              :                       gimple_omp_parallel_data_arg (omp_par_stmt));
    1631              :           }
    1632            0 :           break;
    1633              : 
    1634            0 :         case GIMPLE_OMP_TASK:
    1635            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1636            0 :           copy = gimple_build_omp_task
    1637            0 :                    (s1,
    1638              :                     gimple_omp_task_clauses (stmt),
    1639              :                     gimple_omp_task_child_fn (stmt),
    1640              :                     gimple_omp_task_data_arg (stmt),
    1641              :                     gimple_omp_task_copy_fn (stmt),
    1642              :                     gimple_omp_task_arg_size (stmt),
    1643              :                     gimple_omp_task_arg_align (stmt));
    1644            0 :           break;
    1645              : 
    1646            0 :         case GIMPLE_OMP_FOR:
    1647            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1648            0 :           s2 = remap_gimple_seq (gimple_omp_for_pre_body (stmt), id);
    1649            0 :           copy = gimple_build_omp_for (s1, gimple_omp_for_kind (stmt),
    1650              :                                        gimple_omp_for_clauses (stmt),
    1651              :                                        gimple_omp_for_collapse (stmt), s2);
    1652            0 :           {
    1653            0 :             size_t i;
    1654            0 :             for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
    1655              :               {
    1656            0 :                 gimple_omp_for_set_index (copy, i,
    1657              :                                           gimple_omp_for_index (stmt, i));
    1658            0 :                 gimple_omp_for_set_initial (copy, i,
    1659              :                                             gimple_omp_for_initial (stmt, i));
    1660            0 :                 gimple_omp_for_set_final (copy, i,
    1661              :                                           gimple_omp_for_final (stmt, i));
    1662            0 :                 gimple_omp_for_set_incr (copy, i,
    1663              :                                          gimple_omp_for_incr (stmt, i));
    1664            0 :                 gimple_omp_for_set_cond (copy, i,
    1665              :                                          gimple_omp_for_cond (stmt, i));
    1666              :               }
    1667              :           }
    1668              :           break;
    1669              : 
    1670            0 :         case GIMPLE_OMP_MASTER:
    1671            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1672            0 :           copy = gimple_build_omp_master (s1);
    1673            0 :           break;
    1674              : 
    1675            0 :         case GIMPLE_OMP_MASKED:
    1676            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1677            0 :           copy = gimple_build_omp_masked
    1678            0 :                    (s1, gimple_omp_masked_clauses (stmt));
    1679            0 :           break;
    1680              : 
    1681            0 :         case GIMPLE_OMP_SCOPE:
    1682            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1683            0 :           copy = gimple_build_omp_scope
    1684            0 :                    (s1, gimple_omp_scope_clauses (stmt));
    1685            0 :           break;
    1686              : 
    1687            0 :         case GIMPLE_OMP_DISPATCH:
    1688            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1689            0 :           copy = gimple_build_omp_dispatch (s1,
    1690              :                                             gimple_omp_dispatch_clauses (stmt));
    1691            0 :           break;
    1692              : 
    1693            0 :         case GIMPLE_OMP_TASKGROUP:
    1694            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1695            0 :           copy = gimple_build_omp_taskgroup
    1696            0 :                    (s1, gimple_omp_taskgroup_clauses (stmt));
    1697            0 :           break;
    1698              : 
    1699            0 :         case GIMPLE_OMP_ORDERED:
    1700            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1701            0 :           copy = gimple_build_omp_ordered
    1702            0 :                    (s1,
    1703            0 :                     gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt)));
    1704            0 :           break;
    1705              : 
    1706            0 :         case GIMPLE_OMP_SCAN:
    1707            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1708            0 :           copy = gimple_build_omp_scan
    1709            0 :                    (s1, gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)));
    1710            0 :           break;
    1711              : 
    1712            0 :         case GIMPLE_OMP_SECTION:
    1713            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1714            0 :           copy = gimple_build_omp_section (s1);
    1715            0 :           break;
    1716              : 
    1717            0 :         case GIMPLE_OMP_SECTIONS:
    1718            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1719            0 :           copy = gimple_build_omp_sections
    1720            0 :                    (s1, gimple_omp_sections_clauses (stmt));
    1721            0 :           break;
    1722              : 
    1723            0 :         case GIMPLE_OMP_STRUCTURED_BLOCK:
    1724            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1725            0 :           copy = gimple_build_omp_structured_block (s1);
    1726            0 :           break;
    1727              : 
    1728            0 :         case GIMPLE_OMP_SINGLE:
    1729            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1730            0 :           copy = gimple_build_omp_single
    1731            0 :                    (s1, gimple_omp_single_clauses (stmt));
    1732            0 :           break;
    1733              : 
    1734            0 :         case GIMPLE_OMP_TARGET:
    1735            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1736            0 :           copy = gimple_build_omp_target
    1737            0 :                    (s1, gimple_omp_target_kind (stmt),
    1738              :                     gimple_omp_target_clauses (stmt));
    1739            0 :           break;
    1740              : 
    1741            0 :         case GIMPLE_OMP_TEAMS:
    1742            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1743            0 :           copy = gimple_build_omp_teams
    1744            0 :                    (s1, gimple_omp_teams_clauses (stmt));
    1745            0 :           break;
    1746              : 
    1747            0 :         case GIMPLE_OMP_CRITICAL:
    1748            0 :           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
    1749            0 :           copy = gimple_build_omp_critical (s1,
    1750              :                                             gimple_omp_critical_name
    1751            0 :                                               (as_a <gomp_critical *> (stmt)),
    1752              :                                             gimple_omp_critical_clauses
    1753            0 :                                               (as_a <gomp_critical *> (stmt)));
    1754            0 :           break;
    1755              : 
    1756            0 :         case GIMPLE_ASSUME:
    1757            0 :           s1 = remap_gimple_seq (gimple_assume_body (stmt), id);
    1758            0 :           copy = gimple_build_assume (gimple_assume_guard (stmt), s1);
    1759            0 :           break;
    1760              : 
    1761           67 :         case GIMPLE_TRANSACTION:
    1762           67 :           {
    1763           67 :             gtransaction *old_trans_stmt = as_a <gtransaction *> (stmt);
    1764           67 :             gtransaction *new_trans_stmt;
    1765           67 :             s1 = remap_gimple_seq (gimple_transaction_body (old_trans_stmt),
    1766              :                                    id);
    1767           67 :             copy = new_trans_stmt = gimple_build_transaction (s1);
    1768           67 :             gimple_transaction_set_subcode (new_trans_stmt,
    1769              :               gimple_transaction_subcode (old_trans_stmt));
    1770           67 :             gimple_transaction_set_label_norm (new_trans_stmt,
    1771              :               gimple_transaction_label_norm (old_trans_stmt));
    1772           67 :             gimple_transaction_set_label_uninst (new_trans_stmt,
    1773              :               gimple_transaction_label_uninst (old_trans_stmt));
    1774           67 :             gimple_transaction_set_label_over (new_trans_stmt,
    1775              :               gimple_transaction_label_over (old_trans_stmt));
    1776              :           }
    1777           67 :           break;
    1778              : 
    1779            0 :         default:
    1780            0 :           gcc_unreachable ();
    1781              :         }
    1782              :     }
    1783              :   else
    1784              :     {
    1785     93951871 :       if (gimple_assign_single_p (stmt)
    1786     14002086 :           && gimple_assign_lhs (stmt) == gimple_assign_rhs1 (stmt)
    1787     93951871 :           && auto_var_in_fn_p (gimple_assign_lhs (stmt), id->src_fn))
    1788              :         {
    1789              :           /* Here we handle statements that are not completely rewritten.
    1790              :              First we detect some inlining-induced bogosities for
    1791              :              discarding.  */
    1792              : 
    1793              :           /* Some assignments VAR = VAR; don't generate any rtl code
    1794              :              and thus don't count as variable modification.  Avoid
    1795              :              keeping bogosities like 0 = 0.  */
    1796            0 :           tree decl = gimple_assign_lhs (stmt), value;
    1797            0 :           tree *n;
    1798              : 
    1799            0 :           n = id->decl_map->get (decl);
    1800            0 :           if (n)
    1801              :             {
    1802            0 :               value = *n;
    1803            0 :               STRIP_TYPE_NOPS (value);
    1804            0 :               if (TREE_CONSTANT (value) || TREE_READONLY (value))
    1805            0 :                 return NULL;
    1806              :             }
    1807              :         }
    1808              : 
    1809              :       /* For *ptr_N ={v} {CLOBBER}, if ptr_N is SSA_NAME defined
    1810              :          in a block that we aren't copying during tree_function_versioning,
    1811              :          just drop the clobber stmt.  */
    1812     93951871 :       if (id->blocks_to_copy && gimple_clobber_p (stmt))
    1813              :         {
    1814        17701 :           tree lhs = gimple_assign_lhs (stmt);
    1815        17701 :           if (TREE_CODE (lhs) == MEM_REF
    1816        17701 :               && TREE_CODE (TREE_OPERAND (lhs, 0)) == SSA_NAME)
    1817              :             {
    1818         1136 :               gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (lhs, 0));
    1819         1136 :               if (gimple_bb (def_stmt)
    1820         1476 :                   && !bitmap_bit_p (id->blocks_to_copy,
    1821          340 :                                     gimple_bb (def_stmt)->index))
    1822              :                 return NULL;
    1823              :             }
    1824              :         }
    1825              : 
    1826              :       /* We do not allow CLOBBERs of handled components.  In case
    1827              :          returned value is stored via such handled component, remove
    1828              :          the clobber so stmt verifier is happy.  */
    1829     93951868 :       if (gimple_clobber_p (stmt)
    1830     93951868 :           && TREE_CODE (gimple_assign_lhs (stmt)) == RESULT_DECL)
    1831              :         {
    1832            0 :           tree remapped = remap_decl (gimple_assign_lhs (stmt), id);
    1833            0 :           if (!DECL_P (remapped)
    1834            0 :               && TREE_CODE (remapped) != MEM_REF)
    1835              :             return NULL;
    1836              :         }
    1837              : 
    1838     93951868 :       if (gimple_debug_bind_p (stmt))
    1839              :         {
    1840     47429563 :           tree var = gimple_debug_bind_get_var (stmt);
    1841     47429563 :           tree value = gimple_debug_bind_get_value (stmt);
    1842     47429563 :           if (id->param_body_adjs
    1843     47429563 :               && id->param_body_adjs->m_dead_stmts.contains (stmt))
    1844              :             {
    1845        16554 :               value = unshare_expr_without_location (value);
    1846        16554 :               id->param_body_adjs->remap_with_debug_expressions (&value);
    1847              :             }
    1848              : 
    1849     47429563 :           gdebug *copy = gimple_build_debug_bind (var, value, stmt);
    1850     47429563 :           if (id->reset_location)
    1851            4 :             gimple_set_location (copy, input_location);
    1852     47429563 :           id->debug_stmts.safe_push (copy);
    1853     47429563 :           gimple_seq_add_stmt_without_update (&stmts, copy);
    1854     47429563 :           return stmts;
    1855              :         }
    1856     46522305 :       if (gimple_debug_source_bind_p (stmt))
    1857              :         {
    1858       563968 :           gdebug *copy = gimple_build_debug_source_bind
    1859       563968 :                            (gimple_debug_source_bind_get_var (stmt),
    1860              :                             gimple_debug_source_bind_get_value (stmt),
    1861       563968 :                             stmt);
    1862       563968 :           if (id->reset_location)
    1863            0 :             gimple_set_location (copy, input_location);
    1864       563968 :           id->debug_stmts.safe_push (copy);
    1865       563968 :           gimple_seq_add_stmt_without_update (&stmts, copy);
    1866       563968 :           return stmts;
    1867              :         }
    1868     45958337 :       if (gimple_debug_nonbind_marker_p (stmt))
    1869              :         {
    1870              :           /* If the inlined function has too many debug markers,
    1871              :              don't copy them.  */
    1872     14006354 :           if (id->src_cfun->debug_marker_count
    1873     14006354 :               > param_max_debug_marker_count
    1874     14006354 :               || id->reset_location)
    1875            0 :             return stmts;
    1876              : 
    1877     14006354 :           gdebug *copy = as_a <gdebug *> (gimple_copy (stmt));
    1878     14006354 :           id->debug_stmts.safe_push (copy);
    1879     14006354 :           gimple_seq_add_stmt_without_update (&stmts, copy);
    1880     14006354 :           return stmts;
    1881              :         }
    1882              : 
    1883              :       /* Create a new deep copy of the statement.  */
    1884     31951983 :       copy = gimple_copy (stmt);
    1885              : 
    1886              :       /* Clear flags that need revisiting.  */
    1887     31951983 :       if (gcall *call_stmt = dyn_cast <gcall *> (copy))
    1888              :         {
    1889      5203294 :           if (gimple_call_tail_p (call_stmt))
    1890          148 :             gimple_call_set_tail (call_stmt, false);
    1891      5203294 :           if (gimple_call_from_thunk_p (call_stmt))
    1892          187 :             gimple_call_set_from_thunk (call_stmt, false);
    1893              :           /* Silently clear musttail flag when inlining a function
    1894              :              with must tail call from a non-musttail call.  The inlining
    1895              :              removes one frame so acts like musttail's intent, and we
    1896              :              can be inlining a function with musttail calls in the middle
    1897              :              of caller where musttail will always error.  */
    1898      5203294 :           if (gimple_call_must_tail_p (call_stmt)
    1899           49 :               && id->call_stmt
    1900      5203335 :               && !gimple_call_must_tail_p (id->call_stmt))
    1901           14 :             gimple_call_set_must_tail (call_stmt, false);
    1902      5203294 :           if (gimple_call_internal_p (call_stmt))
    1903        60076 :             switch (gimple_call_internal_fn (call_stmt))
    1904              :               {
    1905          166 :               case IFN_GOMP_SIMD_LANE:
    1906          166 :               case IFN_GOMP_SIMD_VF:
    1907          166 :               case IFN_GOMP_SIMD_LAST_LANE:
    1908          166 :               case IFN_GOMP_SIMD_ORDERED_START:
    1909          166 :               case IFN_GOMP_SIMD_ORDERED_END:
    1910          166 :                 DECL_STRUCT_FUNCTION (id->dst_fn)->has_simduid_loops = true;
    1911          166 :                 break;
    1912              :               default:
    1913              :                 break;
    1914              :               }
    1915              :         }
    1916              : 
    1917              :       /* Remap the region numbers for __builtin_eh_{pointer,filter},
    1918              :          RESX and EH_DISPATCH.  */
    1919     31951983 :       if (id->eh_map)
    1920     31951983 :         switch (gimple_code (copy))
    1921              :           {
    1922      5203294 :           case GIMPLE_CALL:
    1923      5203294 :             {
    1924      5203294 :               tree r, fndecl = gimple_call_fndecl (copy);
    1925      5203294 :               if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
    1926      1437402 :                 switch (DECL_FUNCTION_CODE (fndecl))
    1927              :                   {
    1928            0 :                   case BUILT_IN_EH_COPY_VALUES:
    1929            0 :                     r = gimple_call_arg (copy, 1);
    1930            0 :                     r = remap_eh_region_tree_nr (r, id);
    1931            0 :                     gimple_call_set_arg (copy, 1, r);
    1932              :                     /* FALLTHRU */
    1933              : 
    1934         9735 :                   case BUILT_IN_EH_POINTER:
    1935         9735 :                   case BUILT_IN_EH_FILTER:
    1936         9735 :                     r = gimple_call_arg (copy, 0);
    1937         9735 :                     r = remap_eh_region_tree_nr (r, id);
    1938         9735 :                     gimple_call_set_arg (copy, 0, r);
    1939         9735 :                     break;
    1940              : 
    1941              :                   default:
    1942              :                     break;
    1943              :                   }
    1944              : 
    1945              :               /* Reset alias info if we didn't apply measures to
    1946              :                  keep it valid over inlining by setting DECL_PT_UID.  */
    1947      5203294 :               if (!id->src_cfun->gimple_df
    1948      5203294 :                   || !id->src_cfun->gimple_df->ipa_pta)
    1949      5195941 :                 gimple_call_reset_alias_info (as_a <gcall *> (copy));
    1950              :             }
    1951              :             break;
    1952              : 
    1953        81634 :           case GIMPLE_RESX:
    1954        81634 :             {
    1955        81634 :               gresx *resx_stmt = as_a <gresx *> (copy);
    1956        81634 :               int r = gimple_resx_region (resx_stmt);
    1957        81634 :               r = remap_eh_region_nr (r, id);
    1958        81634 :               gimple_resx_set_region (resx_stmt, r);
    1959              :             }
    1960        81634 :             break;
    1961              : 
    1962        10826 :           case GIMPLE_EH_DISPATCH:
    1963        10826 :             {
    1964        10826 :               geh_dispatch *eh_dispatch = as_a <geh_dispatch *> (copy);
    1965        10826 :               int r = gimple_eh_dispatch_region (eh_dispatch);
    1966        10826 :               r = remap_eh_region_nr (r, id);
    1967        10826 :               gimple_eh_dispatch_set_region (eh_dispatch, r);
    1968              :             }
    1969        10826 :             break;
    1970              : 
    1971              :           default:
    1972              :             break;
    1973              :           }
    1974              :     }
    1975              : 
    1976              :   /* If STMT has a block defined, map it to the newly constructed block.  */
    1977     34145391 :   if (tree block = gimple_block (copy))
    1978              :     {
    1979     30852165 :       tree *n;
    1980     30852165 :       n = id->decl_map->get (block);
    1981     30852165 :       gcc_assert (n);
    1982     30852165 :       gimple_set_block (copy, *n);
    1983              :     }
    1984     34145391 :   if (id->param_body_adjs)
    1985              :     {
    1986      3711915 :       gimple_seq extra_stmts = NULL;
    1987      3711915 :       id->param_body_adjs->modify_gimple_stmt (&copy, &extra_stmts, stmt);
    1988      3711915 :       if (!gimple_seq_empty_p (extra_stmts))
    1989              :         {
    1990           58 :           memset (&wi, 0, sizeof (wi));
    1991           58 :           wi.info = id;
    1992           58 :           for (gimple_stmt_iterator egsi = gsi_start (extra_stmts);
    1993          172 :                !gsi_end_p (egsi);
    1994          114 :                gsi_next (&egsi))
    1995          114 :             walk_gimple_op (gsi_stmt (egsi), remap_gimple_op_r, &wi);
    1996           58 :           gimple_seq_add_seq_without_update (&stmts, extra_stmts);
    1997              :         }
    1998              :     }
    1999              : 
    2000     34145391 :   if (id->reset_location)
    2001          696 :     gimple_set_location (copy, input_location);
    2002              : 
    2003              :   /* Debug statements ought to be rebuilt and not copied.  */
    2004     34145391 :   gcc_checking_assert (!is_gimple_debug (copy));
    2005              : 
    2006              :   /* Remap all the operands in COPY.  */
    2007     34145391 :   memset (&wi, 0, sizeof (wi));
    2008     34145391 :   wi.info = id;
    2009     34145391 :   if (skip_first)
    2010      2193341 :     walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL);
    2011              :   else
    2012     31952050 :     walk_gimple_op (copy, remap_gimple_op_r, &wi);
    2013              : 
    2014              :   /* Clear the copied virtual operands.  We are not remapping them here
    2015              :      but are going to recreate them from scratch.  */
    2016     34145391 :   if (gimple_has_mem_ops (copy))
    2017              :     {
    2018     29296872 :       gimple_set_vdef (copy, NULL_TREE);
    2019     29296872 :       gimple_set_vuse (copy, NULL_TREE);
    2020              :     }
    2021              : 
    2022     34145391 :   if (cfun->can_throw_non_call_exceptions)
    2023              :     {
    2024              :       /* When inlining a function which does not have non-call exceptions
    2025              :          enabled into a function that has (which only happens with
    2026              :          always-inline) we have to fixup stmts that cannot throw.  */
    2027      1712947 :       if (gcond *cond = dyn_cast <gcond *> (copy))
    2028       207927 :         if (gimple_could_trap_p (cond))
    2029              :           {
    2030            1 :             gassign *cmp
    2031            1 :               = gimple_build_assign (make_ssa_name (boolean_type_node),
    2032              :                                      gimple_cond_code (cond),
    2033              :                                      gimple_cond_lhs (cond),
    2034              :                                      gimple_cond_rhs (cond));
    2035            1 :             gimple_seq_add_stmt_without_update (&stmts, cmp);
    2036            1 :             gimple_cond_set_code (cond, NE_EXPR);
    2037            1 :             gimple_cond_set_lhs (cond, gimple_assign_lhs (cmp));
    2038            1 :             gimple_cond_set_rhs (cond, boolean_false_node);
    2039              :           }
    2040              :     }
    2041              : 
    2042     34145391 :   gimple_seq_add_stmt_without_update (&stmts, copy);
    2043     34145391 :   return stmts;
    2044              : }
    2045              : 
    2046              : 
    2047              : /* Copy basic block, scale profile accordingly.  Edges will be taken care of
    2048              :    later  */
    2049              : 
    2050              : static basic_block
    2051     14815247 : copy_bb (copy_body_data *id, basic_block bb,
    2052              :          profile_count num, profile_count den)
    2053              : {
    2054     14815247 :   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
    2055     14815247 :   basic_block copy_basic_block;
    2056     14815247 :   tree decl;
    2057     14815247 :   basic_block prev;
    2058              : 
    2059     14815247 :   profile_count::adjust_for_ipa_scaling (&num, &den);
    2060              : 
    2061              :   /* Search for previous copied basic block.  */
    2062     14815247 :   prev = bb->prev_bb;
    2063     14977665 :   while (!prev->aux)
    2064       162418 :     prev = prev->prev_bb;
    2065              : 
    2066              :   /* create_basic_block() will append every new block to
    2067              :      basic_block_info automatically.  */
    2068     14815247 :   copy_basic_block = create_basic_block (NULL, (basic_block) prev->aux);
    2069     14815247 :   copy_basic_block->count = bb->count.apply_scale (num, den);
    2070              : 
    2071     14815247 :   copy_gsi = gsi_start_bb (copy_basic_block);
    2072              : 
    2073     14815247 :   unsigned min_cond_uid = 0;
    2074     14815247 :   if (id->src_cfun->cond_uids)
    2075              :     {
    2076           63 :       if (!cfun->cond_uids)
    2077            7 :         cfun->cond_uids = new hash_map <gcond*, unsigned> ();
    2078              : 
    2079          193 :       for (auto itr : *id->src_cfun->cond_uids)
    2080           67 :         if (itr.second >= min_cond_uid)
    2081           67 :           min_cond_uid = itr.second + 1;
    2082              :     }
    2083              : 
    2084    128228710 :   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    2085              :     {
    2086     98598216 :       gimple_seq stmts;
    2087     98598216 :       gimple *stmt = gsi_stmt (gsi);
    2088     98598216 :       gimple *orig_stmt = stmt;
    2089     98598216 :       gimple_stmt_iterator stmts_gsi;
    2090     98598216 :       bool stmt_added = false;
    2091              : 
    2092     98598216 :       id->regimplify = false;
    2093     98598216 :       stmts = remap_gimple_stmt (stmt, id);
    2094              : 
    2095     98598216 :       if (gimple_seq_empty_p (stmts))
    2096      2449783 :         continue;
    2097              : 
    2098     96148616 :       seq_gsi = copy_gsi;
    2099              : 
    2100     96148616 :       for (stmts_gsi = gsi_start (stmts);
    2101    192297347 :            !gsi_end_p (stmts_gsi); )
    2102              :         {
    2103     96148731 :           stmt = gsi_stmt (stmts_gsi);
    2104              : 
    2105              :           /* Advance iterator now before stmt is moved to seq_gsi.  */
    2106     96148731 :           gsi_next (&stmts_gsi);
    2107              : 
    2108     96148731 :           if (gimple_nop_p (stmt))
    2109          183 :               continue;
    2110              : 
    2111              :           /* If -fcondition-coverage is used, register the inlined conditions
    2112              :              in the cond->expression mapping of the caller.  The expression tag
    2113              :              is shifted conditions from the two bodies are not mixed.  */
    2114     96148548 :           if (id->src_cfun->cond_uids && is_a <gcond*> (stmt))
    2115              :             {
    2116           19 :               gcond *orig_cond = as_a <gcond*> (orig_stmt);
    2117           19 :               gcond *cond = as_a <gcond*> (stmt);
    2118           19 :               unsigned *v = id->src_cfun->cond_uids->get (orig_cond);
    2119           19 :               if (v)
    2120           19 :                 cfun->cond_uids->put (cond, *v + min_cond_uid);
    2121              :             }
    2122              : 
    2123     96148548 :           gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun,
    2124              :                                             orig_stmt);
    2125              : 
    2126     96148548 :           gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT);
    2127              : 
    2128     96148548 :           if (id->regimplify)
    2129         4554 :             gimple_regimplify_operands (stmt, &seq_gsi);
    2130              : 
    2131     96148731 :           stmt_added = true;
    2132              :         }
    2133              : 
    2134     96148616 :       if (!stmt_added)
    2135          183 :         continue;
    2136              : 
    2137              :       /* If copy_basic_block has been empty at the start of this iteration,
    2138              :          call gsi_start_bb again to get at the newly added statements.  */
    2139     96148433 :       if (gsi_end_p (copy_gsi))
    2140     26200230 :         copy_gsi = gsi_start_bb (copy_basic_block);
    2141              :       else
    2142     83048318 :         gsi_next (&copy_gsi);
    2143              : 
    2144              :       /* Process the new statement.  The call to gimple_regimplify_operands
    2145              :          possibly turned the statement into multiple statements, we
    2146              :          need to process all of them.  */
    2147     96148552 :       do
    2148              :         {
    2149     96148552 :           tree fn;
    2150     96148552 :           gcall *call_stmt;
    2151              : 
    2152     96148552 :           stmt = gsi_stmt (copy_gsi);
    2153     96148552 :           call_stmt = dyn_cast <gcall *> (stmt);
    2154      5203294 :           if (call_stmt
    2155      5203294 :               && gimple_call_va_arg_pack_p (call_stmt)
    2156          324 :               && id->call_stmt
    2157          322 :               && ! gimple_call_va_arg_pack_p (id->call_stmt))
    2158              :             {
    2159              :               /* __builtin_va_arg_pack () should be replaced by
    2160              :                  all arguments corresponding to ... in the caller.  */
    2161          308 :               tree p;
    2162          308 :               gcall *new_call;
    2163          308 :               vec<tree> argarray;
    2164          308 :               size_t nargs_caller = gimple_call_num_args (id->call_stmt);
    2165          308 :               size_t nargs = nargs_caller;
    2166              : 
    2167          792 :               for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
    2168              :                 {
    2169              :                   /* Avoid crashing on invalid IL that doesn't have a
    2170              :                      varargs function or that passes not enough arguments.  */
    2171          500 :                   if (nargs == 0)
    2172              :                     break;
    2173          484 :                   nargs--;
    2174              :                 }
    2175              : 
    2176              :               /* Create the new array of arguments.  */
    2177          308 :               size_t nargs_callee = gimple_call_num_args (call_stmt);
    2178          308 :               size_t n = nargs + nargs_callee;
    2179          308 :               argarray.create (n);
    2180          308 :               argarray.safe_grow_cleared (n, true);
    2181              : 
    2182              :               /* Copy all the arguments before '...'  */
    2183          308 :               if (nargs_callee)
    2184          612 :                 memcpy (argarray.address (),
    2185          306 :                         gimple_call_arg_ptr (call_stmt, 0),
    2186              :                         nargs_callee * sizeof (tree));
    2187              : 
    2188              :               /* Append the arguments passed in '...'  */
    2189          308 :               if (nargs)
    2190          174 :                 memcpy (argarray.address () + nargs_callee,
    2191          174 :                         gimple_call_arg_ptr (id->call_stmt, 0)
    2192          174 :                         + (nargs_caller - nargs), nargs * sizeof (tree));
    2193              : 
    2194          308 :               new_call = gimple_build_call_vec (gimple_call_fn (call_stmt),
    2195              :                                                 argarray);
    2196              : 
    2197          308 :               argarray.release ();
    2198              : 
    2199              :               /* Copy all GIMPLE_CALL flags, location and block, except
    2200              :                  GF_CALL_VA_ARG_PACK.  */
    2201          308 :               gimple_call_copy_flags (new_call, call_stmt);
    2202          308 :               gimple_call_set_va_arg_pack (new_call, false);
    2203          616 :               gimple_call_set_fntype (new_call, gimple_call_fntype (call_stmt));
    2204              :               /* location includes block.  */
    2205          308 :               gimple_set_location (new_call, gimple_location (stmt));
    2206          308 :               gimple_call_set_lhs (new_call, gimple_call_lhs (call_stmt));
    2207              : 
    2208          308 :               gsi_replace (&copy_gsi, new_call, false);
    2209          308 :               stmt = new_call;
    2210              :             }
    2211     96148244 :           else if (call_stmt
    2212      5202986 :                    && id->call_stmt
    2213      4232495 :                    && (decl = gimple_call_fndecl (stmt))
    2214    100246978 :                    && fndecl_built_in_p (decl, BUILT_IN_VA_ARG_PACK_LEN))
    2215              :             {
    2216              :               /* __builtin_va_arg_pack_len () should be replaced by
    2217              :                  the number of anonymous arguments.  */
    2218          147 :               size_t nargs = gimple_call_num_args (id->call_stmt);
    2219          147 :               tree count, p;
    2220          147 :               gimple *new_stmt;
    2221              : 
    2222          423 :               for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
    2223          276 :                 nargs--;
    2224              : 
    2225          147 :               if (!gimple_call_lhs (stmt))
    2226              :                 {
    2227              :                   /* Drop unused calls.  */
    2228            1 :                   gsi_remove (&copy_gsi, false);
    2229            1 :                   continue;
    2230              :                 }
    2231          146 :               else if (!gimple_call_va_arg_pack_p (id->call_stmt))
    2232              :                 {
    2233          124 :                   count = build_int_cst (integer_type_node, nargs);
    2234          124 :                   new_stmt = gimple_build_assign (gimple_call_lhs (stmt), count);
    2235          124 :                   gsi_replace (&copy_gsi, new_stmt, false);
    2236          124 :                   stmt = new_stmt;
    2237              :                 }
    2238           22 :               else if (nargs != 0)
    2239              :                 {
    2240            7 :                   tree newlhs = make_ssa_name (integer_type_node);
    2241            7 :                   count = build_int_cst (integer_type_node, nargs);
    2242            7 :                   new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
    2243              :                                                   PLUS_EXPR, newlhs, count);
    2244            7 :                   gimple_call_set_lhs (stmt, newlhs);
    2245            7 :                   gsi_insert_after (&copy_gsi, new_stmt, GSI_NEW_STMT);
    2246              :                 }
    2247              :             }
    2248     96148097 :           else if (call_stmt
    2249      5202839 :                    && id->call_stmt
    2250    100380445 :                    && gimple_call_internal_p (stmt))
    2251        45970 :             switch (gimple_call_internal_fn (stmt))
    2252              :               {
    2253          154 :               case IFN_TSAN_FUNC_EXIT:
    2254              :                 /* Drop .TSAN_FUNC_EXIT () internal calls during inlining.  */
    2255          154 :                 gsi_remove (&copy_gsi, false);
    2256          154 :                 continue;
    2257         1396 :               case IFN_ASAN_MARK:
    2258              :                 /* Drop .ASAN_MARK internal calls during inlining into
    2259              :                    no_sanitize functions.  */
    2260         1396 :                 if (!sanitize_flags_p (SANITIZE_ADDRESS, id->dst_fn)
    2261         1396 :                     && !sanitize_flags_p (SANITIZE_HWADDRESS, id->dst_fn))
    2262              :                   {
    2263           14 :                     gsi_remove (&copy_gsi, false);
    2264           14 :                     continue;
    2265              :                   }
    2266              :                 break;
    2267              :               default:
    2268              :                 break;
    2269              :               }
    2270              : 
    2271              :           /* Statements produced by inlining can be unfolded, especially
    2272              :              when we constant propagated some operands.  We can't fold
    2273              :              them right now for two reasons:
    2274              :              1) folding require SSA_NAME_DEF_STMTs to be correct
    2275              :              2) we can't change function calls to builtins.
    2276              :              So we just mark statement for later folding.  We mark
    2277              :              all new statements, instead just statements that has changed
    2278              :              by some nontrivial substitution so even statements made
    2279              :              foldable indirectly are updated.  If this turns out to be
    2280              :              expensive, copy_body can be told to watch for nontrivial
    2281              :              changes.  */
    2282     96148383 :           if (id->statements_to_fold)
    2283     96148383 :             id->statements_to_fold->add (stmt);
    2284              : 
    2285              :           /* We're duplicating a CALL_EXPR.  Find any corresponding
    2286              :              callgraph edges and update or duplicate them.  */
    2287     96148383 :           if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
    2288              :             {
    2289      5203001 :               struct cgraph_edge *edge;
    2290              : 
    2291      5203001 :               switch (id->transform_call_graph_edges)
    2292              :                 {
    2293      4232510 :                 case CB_CGE_DUPLICATE:
    2294      4232510 :                   edge = id->src_node->get_edge (orig_stmt);
    2295      4232510 :                   if (edge)
    2296              :                     {
    2297      4186711 :                       struct cgraph_edge *old_edge = edge;
    2298              : 
    2299              :                       /* A speculative call is consist of multiple
    2300              :                          edges - indirect edge and one or more direct edges
    2301              :                          Duplicate the whole thing and distribute frequencies
    2302              :                          accordingly.  */
    2303      4186711 :                       if (edge->speculative)
    2304              :                         {
    2305        20423 :                           int n = 0;
    2306        20423 :                           profile_count direct_cnt
    2307        20423 :                                  = profile_count::zero ();
    2308              : 
    2309              :                           /* First figure out the distribution of counts
    2310              :                              so we can re-scale BB profile accordingly.  */
    2311        47735 :                           for (cgraph_edge *e = old_edge; e;
    2312        27312 :                                e = e->next_speculative_call_target ())
    2313        27312 :                             direct_cnt = direct_cnt + e->count;
    2314              : 
    2315        20423 :                           cgraph_edge *indirect
    2316        20423 :                                  = old_edge->speculative_call_indirect_edge ();
    2317        20423 :                           profile_count indir_cnt = indirect->count;
    2318              : 
    2319              :                           /* Next iterate all direct edges, clone it and its
    2320              :                              corresponding reference and update profile.  */
    2321        20423 :                           for (cgraph_edge *e = old_edge;
    2322        47735 :                                e;
    2323        27312 :                                e = e->next_speculative_call_target ())
    2324              :                             {
    2325        27312 :                               profile_count cnt = e->count;
    2326              : 
    2327        27312 :                               id->dst_node->clone_reference
    2328        27312 :                                  (e->speculative_call_target_ref (), stmt);
    2329        27312 :                               edge = e->clone (id->dst_node, call_stmt,
    2330              :                                                gimple_uid (stmt), num, den,
    2331              :                                                true);
    2332        27312 :                               profile_probability prob
    2333        27312 :                                  = cnt.probability_in (direct_cnt
    2334              :                                                        + indir_cnt);
    2335        27312 :                               edge->count
    2336              :                                  = copy_basic_block->count.apply_probability
    2337        27312 :                                          (prob);
    2338        27312 :                               n++;
    2339              :                             }
    2340        20423 :                           gcc_checking_assert
    2341              :                                  (indirect->num_speculative_call_targets_p ()
    2342              :                                   == n);
    2343              : 
    2344              :                           /* Duplicate the indirect edge after all direct edges
    2345              :                              cloned.  */
    2346        20423 :                           indirect = indirect->clone (id->dst_node, call_stmt,
    2347              :                                                       gimple_uid (stmt),
    2348              :                                                       num, den,
    2349              :                                                       true);
    2350              : 
    2351        20423 :                           profile_probability prob
    2352        20423 :                              = indir_cnt.probability_in (direct_cnt
    2353              :                                                          + indir_cnt);
    2354        20423 :                           indirect->count
    2355        20423 :                              = copy_basic_block->count.apply_probability (prob);
    2356              :                         }
    2357              :                       /* If edge is a callback-carrying edge, copy all its
    2358              :                          attached edges as well.  */
    2359      4166288 :                       else if (edge->has_callback)
    2360              :                         {
    2361         2394 :                           edge
    2362         2394 :                             = edge->clone (id->dst_node, call_stmt,
    2363              :                                            gimple_uid (stmt), num, den, true);
    2364         2394 :                           cgraph_edge *e;
    2365         2846 :                           for (e = old_edge->first_callback_edge (); e;
    2366          452 :                                e = e->next_callback_edge ())
    2367          452 :                             edge = e->clone (id->dst_node, call_stmt,
    2368              :                                              gimple_uid (stmt), num, den, true);
    2369              :                         }
    2370              :                       else
    2371              :                         {
    2372      4163894 :                           edge = edge->clone (id->dst_node, call_stmt,
    2373              :                                               gimple_uid (stmt),
    2374              :                                               num, den,
    2375              :                                               true);
    2376      4163894 :                           edge->count = copy_basic_block->count;
    2377              :                         }
    2378              :                     }
    2379              :                   break;
    2380              : 
    2381       773661 :                 case CB_CGE_MOVE_CLONES:
    2382       773661 :                   id->dst_node->set_call_stmt_including_clones (orig_stmt,
    2383              :                                                                 call_stmt);
    2384       773661 :                   edge = id->dst_node->get_edge (stmt);
    2385       773661 :                   break;
    2386              : 
    2387       196830 :                 case CB_CGE_MOVE:
    2388       196830 :                   edge = id->dst_node->get_edge (orig_stmt);
    2389       196830 :                   if (edge)
    2390       196464 :                     edge = cgraph_edge::set_call_stmt (edge, call_stmt);
    2391              :                   break;
    2392              : 
    2393            0 :                 default:
    2394            0 :                   gcc_unreachable ();
    2395              :                 }
    2396              : 
    2397              :               /* Constant propagation on argument done during inlining
    2398              :                  may create new direct call.  Produce an edge for it.  */
    2399      4163894 :               if ((!edge
    2400      5044777 :                    || (edge->indirect_inlining_edge
    2401         4048 :                        && id->transform_call_graph_edges == CB_CGE_MOVE_CLONES))
    2402       158720 :                   && id->dst_node->definition
    2403      1052282 :                   && (fn = gimple_call_fndecl (stmt)) != NULL)
    2404              :                 {
    2405            0 :                   struct cgraph_node *dest = cgraph_node::get_create (fn);
    2406              : 
    2407              :                   /* We have missing edge in the callgraph.  This can happen
    2408              :                      when previous inlining turned an indirect call into a
    2409              :                      direct call by constant propagating arguments or we are
    2410              :                      producing dead clone (for further cloning).  In all
    2411              :                      other cases we hit a bug (incorrect node sharing is the
    2412              :                      most common reason for missing edges).  */
    2413            0 :                   gcc_assert (!dest->definition
    2414              :                               || dest->address_taken
    2415              :                               || !id->src_node->definition
    2416              :                               || !id->dst_node->definition);
    2417            0 :                   if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
    2418            0 :                     id->dst_node->create_edge_including_clones
    2419            0 :                       (dest, orig_stmt, call_stmt, bb->count,
    2420              :                        CIF_ORIGINALLY_INDIRECT_CALL);
    2421              :                   else
    2422            0 :                     id->dst_node->create_edge (dest, call_stmt,
    2423              :                                         bb->count)->inline_failed
    2424            0 :                       = CIF_ORIGINALLY_INDIRECT_CALL;
    2425            0 :                   if (dump_file)
    2426              :                     {
    2427            0 :                       fprintf (dump_file, "Created new direct edge to %s\n",
    2428              :                                dest->dump_name ());
    2429              :                     }
    2430              :                 }
    2431              : 
    2432      5203001 :               notice_special_calls (as_a <gcall *> (stmt));
    2433              :             }
    2434              : 
    2435     96148383 :           maybe_duplicate_eh_stmt_fn (cfun, stmt, id->src_cfun, orig_stmt,
    2436              :                                       id->eh_map, id->eh_lp_nr);
    2437              : 
    2438     96148383 :           gsi_next (&copy_gsi);
    2439              :         }
    2440     96148552 :       while (!gsi_end_p (copy_gsi));
    2441              : 
    2442    192296866 :       copy_gsi = gsi_last_bb (copy_basic_block);
    2443              :     }
    2444              : 
    2445     14815247 :   return copy_basic_block;
    2446              : }
    2447              : 
    2448              : /* Inserting Single Entry Multiple Exit region in SSA form into code in SSA
    2449              :    form is quite easy, since dominator relationship for old basic blocks does
    2450              :    not change.
    2451              : 
    2452              :    There is however exception where inlining might change dominator relation
    2453              :    across EH edges from basic block within inlined functions destinating
    2454              :    to landing pads in function we inline into.
    2455              : 
    2456              :    The function fills in PHI_RESULTs of such PHI nodes if they refer
    2457              :    to gimple regs.  Otherwise, the function mark PHI_RESULT of such
    2458              :    PHI nodes for renaming.  For non-gimple regs, renaming is safe: the
    2459              :    EH edges are abnormal and SSA_NAME_OCCURS_IN_ABNORMAL_PHI must be
    2460              :    set, and this means that there will be no overlapping live ranges
    2461              :    for the underlying symbol.
    2462              : 
    2463              :    This might change in future if we allow redirecting of EH edges and
    2464              :    we might want to change way build CFG pre-inlining to include
    2465              :    all the possible edges then.  */
    2466              : static void
    2467       785592 : update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
    2468              :                                   bool can_throw, bool nonlocal_goto)
    2469              : {
    2470       785592 :   edge e;
    2471       785592 :   edge_iterator ei;
    2472              : 
    2473      2166739 :   FOR_EACH_EDGE (e, ei, bb->succs)
    2474      1381147 :     if (!e->dest->aux
    2475       755196 :         || ((basic_block)e->dest->aux)->index == ENTRY_BLOCK)
    2476              :       {
    2477       625951 :         gphi *phi;
    2478       625951 :         gphi_iterator si;
    2479              : 
    2480       625951 :         if (!nonlocal_goto)
    2481       625513 :           gcc_assert (e->flags & EDGE_EH);
    2482              : 
    2483       625951 :         if (!can_throw)
    2484          229 :           gcc_assert (!(e->flags & EDGE_EH));
    2485              : 
    2486      1041053 :         for (si = gsi_start_phis (e->dest); !gsi_end_p (si); gsi_next (&si))
    2487              :           {
    2488       415102 :             edge re;
    2489              : 
    2490       415102 :             phi = si.phi ();
    2491              : 
    2492              :             /* For abnormal goto/call edges the receiver can be the
    2493              :                ENTRY_BLOCK.  Do not assert this cannot happen.  */
    2494              : 
    2495       415102 :             gcc_assert ((e->flags & EDGE_EH)
    2496              :                         || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)));
    2497              : 
    2498       415102 :             re = find_edge (ret_bb, e->dest);
    2499       415102 :             gcc_checking_assert (re);
    2500       415102 :             gcc_assert ((re->flags & (EDGE_EH | EDGE_ABNORMAL))
    2501              :                         == (e->flags & (EDGE_EH | EDGE_ABNORMAL)));
    2502              : 
    2503       415102 :             SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
    2504              :                      USE_FROM_PTR (PHI_ARG_DEF_PTR_FROM_EDGE (phi, re)));
    2505              :           }
    2506              :       }
    2507       785592 : }
    2508              : 
    2509              : /* Insert clobbers for automatic variables of inlined ID->src_fn
    2510              :    function at the start of basic block ID->eh_landing_pad_dest.  */
    2511              : 
    2512              : static void
    2513       453637 : add_clobbers_to_eh_landing_pad (copy_body_data *id)
    2514              : {
    2515       453637 :   tree var;
    2516       453637 :   basic_block bb = id->eh_landing_pad_dest;
    2517       453637 :   live_vars_map *vars = NULL;
    2518       453637 :   unsigned int cnt = 0;
    2519       453637 :   unsigned int i;
    2520      1563814 :   FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
    2521       700337 :     if (VAR_P (var)
    2522       700337 :         && !DECL_HARD_REGISTER (var)
    2523       700337 :         && !TREE_THIS_VOLATILE (var)
    2524       700317 :         && !DECL_HAS_VALUE_EXPR_P (var)
    2525       691348 :         && !is_gimple_reg (var)
    2526       297527 :         && auto_var_in_fn_p (var, id->src_fn)
    2527       995526 :         && !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var)))
    2528              :       {
    2529       295189 :         tree *t = id->decl_map->get (var);
    2530       295189 :         if (!t)
    2531            0 :           continue;
    2532       295189 :         tree new_var = *t;
    2533       295189 :         if (VAR_P (new_var)
    2534       295189 :             && !DECL_HARD_REGISTER (new_var)
    2535       295189 :             && !TREE_THIS_VOLATILE (new_var)
    2536       295189 :             && !DECL_HAS_VALUE_EXPR_P (new_var)
    2537       295189 :             && !is_gimple_reg (new_var)
    2538       590378 :             && auto_var_in_fn_p (new_var, id->dst_fn))
    2539              :           {
    2540       295189 :             if (vars == NULL)
    2541       184243 :               vars = new live_vars_map;
    2542       295189 :             vars->put (DECL_UID (var), cnt++);
    2543              :           }
    2544              :       }
    2545       453637 :   if (vars == NULL)
    2546       269394 :     return;
    2547              : 
    2548       184243 :   vec<bitmap_head> live = compute_live_vars (id->src_cfun, vars);
    2549      1069544 :   FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
    2550       516815 :     if (VAR_P (var))
    2551              :       {
    2552       516815 :         edge e;
    2553       516815 :         edge_iterator ei;
    2554       516815 :         bool needed = false;
    2555       516815 :         unsigned int *v = vars->get (DECL_UID (var));
    2556       516815 :         if (v == NULL)
    2557       221626 :           continue;
    2558      5191331 :         FOR_EACH_EDGE (e, ei, bb->preds)
    2559      5058838 :           if ((e->flags & EDGE_EH) != 0
    2560      5058798 :               && e->src->index >= id->add_clobbers_to_eh_landing_pads)
    2561              :             {
    2562       485630 :               basic_block src_bb = (basic_block) e->src->aux;
    2563              : 
    2564       485630 :               if (bitmap_bit_p (&live[src_bb->index], *v))
    2565              :                 {
    2566              :                   needed = true;
    2567              :                   break;
    2568              :                 }
    2569              :             }
    2570       295189 :         if (needed)
    2571              :           {
    2572       162696 :             tree new_var = *id->decl_map->get (var);
    2573       162696 :             gimple_stmt_iterator gsi = gsi_after_labels (bb);
    2574       162696 :             tree clobber = build_clobber (TREE_TYPE (new_var));
    2575       162696 :             gimple *clobber_stmt = gimple_build_assign (new_var, clobber);
    2576       162696 :             gsi_insert_before (&gsi, clobber_stmt, GSI_NEW_STMT);
    2577              :           }
    2578              :       }
    2579       184243 :   destroy_live_vars (live);
    2580       184243 :   delete vars;
    2581              : }
    2582              : 
    2583              : /* Copy edges from BB into its copy constructed earlier, scale profile
    2584              :    accordingly.  Edges will be taken care of later.  Assume aux
    2585              :    pointers to point to the copies of each BB.  Return true if any
    2586              :    debug stmts are left after a statement that must end the basic block.  */
    2587              : 
    2588              : static bool
    2589     24509317 : copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
    2590              :                    basic_block ret_bb, basic_block abnormal_goto_dest,
    2591              :                    copy_body_data *id)
    2592              : {
    2593     24509317 :   basic_block new_bb = (basic_block) bb->aux;
    2594     24509317 :   edge_iterator ei;
    2595     24509317 :   edge old_edge;
    2596     24509317 :   gimple_stmt_iterator si;
    2597     24509317 :   bool need_debug_cleanup = false;
    2598              : 
    2599              :   /* Use the indices from the original blocks to create edges for the
    2600              :      new ones.  */
    2601     47443122 :   FOR_EACH_EDGE (old_edge, ei, bb->succs)
    2602     22933805 :     if (!(old_edge->flags & EDGE_EH))
    2603              :       {
    2604     22773854 :         edge new_edge;
    2605     22773854 :         int flags = old_edge->flags;
    2606     22773854 :         location_t locus = old_edge->goto_locus;
    2607              : 
    2608              :         /* Return edges do get a FALLTHRU flag when they get inlined.  */
    2609     22773854 :         if (old_edge->dest->index == EXIT_BLOCK
    2610      4845071 :             && !(flags & (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE|EDGE_FAKE))
    2611      4845071 :             && old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
    2612      4607572 :           flags |= EDGE_FALLTHRU;
    2613              : 
    2614     22773854 :         new_edge
    2615     22773854 :           = make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
    2616     22773854 :         new_edge->probability = old_edge->probability;
    2617     22773854 :         if (!id->reset_location)
    2618     22773466 :           new_edge->goto_locus = remap_location (locus, id);
    2619              :       }
    2620              : 
    2621     24509317 :   if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK)
    2622              :     return false;
    2623              : 
    2624              :   /* When doing function splitting, we must decrease count of the return block
    2625              :      which was previously reachable by block we did not copy.  */
    2626     14815247 :   if (single_succ_p (bb) && single_succ_edge (bb)->dest->index == EXIT_BLOCK)
    2627     11520273 :     FOR_EACH_EDGE (old_edge, ei, bb->preds)
    2628      6675202 :       if (old_edge->src->index != ENTRY_BLOCK
    2629      3829449 :           && !old_edge->src->aux)
    2630        47093 :         new_bb->count -= old_edge->count ().apply_scale (num, den);
    2631              : 
    2632              :   /* Walk stmts from end to start so that splitting will adjust the BB
    2633              :      pointer for each stmt at most once, even when we split the block
    2634              :      multiple times.  */
    2635     14815247 :   bool seen_nondebug = false;
    2636    125778884 :   for (si = gsi_last_bb (new_bb); !gsi_end_p (si);)
    2637              :     {
    2638     96148390 :       bool can_throw, nonlocal_goto;
    2639     96148390 :       gimple *copy_stmt = gsi_stmt (si);
    2640              : 
    2641              :       /* Do this before the possible split_block.  */
    2642     96148390 :       gsi_prev (&si);
    2643              : 
    2644              :       /* If this tree could throw an exception, there are two
    2645              :          cases where we need to add abnormal edge(s): the
    2646              :          tree wasn't in a region and there is a "current
    2647              :          region" in the caller; or the original tree had
    2648              :          EH edges.  In both cases split the block after the tree,
    2649              :          and add abnormal edge(s) as needed; we need both
    2650              :          those from the callee and the caller.
    2651              :          We check whether the copy can throw, because the const
    2652              :          propagation can change an INDIRECT_REF which throws
    2653              :          into a COMPONENT_REF which doesn't.  If the copy
    2654              :          can throw, the original could also throw.  */
    2655     96148390 :       can_throw = stmt_can_throw_internal (cfun, copy_stmt);
    2656     96148390 :       nonlocal_goto
    2657     96148390 :         = (stmt_can_make_abnormal_goto (copy_stmt)
    2658     96148390 :            && !computed_goto_p (copy_stmt));
    2659              : 
    2660     96147894 :       if (can_throw || nonlocal_goto)
    2661              :         {
    2662              :           /* If there's only debug insns after copy_stmt don't split
    2663              :              the block but instead mark the block for cleanup.  */
    2664       785730 :           if (!seen_nondebug)
    2665              :             need_debug_cleanup = true;
    2666              :           else
    2667              :             {
    2668              :               /* Note that bb's predecessor edges aren't necessarily
    2669              :                  right at this point; split_block doesn't care.  */
    2670       289559 :               edge e = split_block (new_bb, copy_stmt);
    2671       289559 :               e->dest->aux = new_bb->aux;
    2672       289559 :               seen_nondebug = false;
    2673              :             }
    2674              :         }
    2675              : 
    2676     96148390 :       if (!is_gimple_debug (copy_stmt))
    2677     34145165 :         seen_nondebug = true;
    2678              : 
    2679     96148390 :       bool update_probs = false;
    2680              : 
    2681     96148390 :       if (gimple_code (copy_stmt) == GIMPLE_EH_DISPATCH)
    2682              :         {
    2683        10826 :           make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt));
    2684        10826 :           update_probs = true;
    2685              :         }
    2686     96137564 :       else if (can_throw)
    2687              :         {
    2688       785363 :           make_eh_edge (copy_stmt);
    2689       785363 :           update_probs = true;
    2690              :         }
    2691              : 
    2692              :       /* EH edges may not match old edges.  Copy as much as possible.  */
    2693       796189 :       if (update_probs)
    2694              :         {
    2695       796189 :           edge e;
    2696       796189 :           edge_iterator ei;
    2697       796189 :           basic_block copy_stmt_bb = gimple_bb (copy_stmt);
    2698              : 
    2699      1602956 :           FOR_EACH_EDGE (old_edge, ei, bb->succs)
    2700       806767 :             if ((old_edge->flags & EDGE_EH)
    2701       169017 :                 && (e = find_edge (copy_stmt_bb,
    2702       169017 :                                    (basic_block) old_edge->dest->aux))
    2703       966517 :                 && (e->flags & EDGE_EH))
    2704       159750 :               e->probability = old_edge->probability;
    2705              : 
    2706      2189293 :           FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
    2707      1393104 :             if (e->flags & EDGE_EH)
    2708              :               {
    2709       785363 :                 if (!e->probability.initialized_p ())
    2710       625999 :                   e->probability = profile_probability::never ();
    2711       785363 :                 if (e->dest->index < id->add_clobbers_to_eh_landing_pads)
    2712              :                   {
    2713       624578 :                     if (id->eh_landing_pad_dest == NULL)
    2714       453637 :                       id->eh_landing_pad_dest = e->dest;
    2715              :                     else
    2716       170941 :                       gcc_assert (id->eh_landing_pad_dest == e->dest);
    2717              :                   }
    2718              :               }
    2719              :         }
    2720              : 
    2721              : 
    2722              :       /* If the call we inline cannot make abnormal goto do not add
    2723              :          additional abnormal edges but only retain those already present
    2724              :          in the original function body.  */
    2725     96148390 :       if (abnormal_goto_dest == NULL)
    2726              :         nonlocal_goto = false;
    2727         1101 :       if (nonlocal_goto)
    2728              :         {
    2729          338 :           basic_block copy_stmt_bb = gimple_bb (copy_stmt);
    2730              : 
    2731          338 :           if (get_abnormal_succ_dispatcher (copy_stmt_bb))
    2732              :             nonlocal_goto = false;
    2733              :           /* ABNORMAL_DISPATCHER (1) is for longjmp/setjmp or nonlocal gotos
    2734              :              in OpenMP regions which aren't allowed to be left abnormally.
    2735              :              So, no need to add abnormal edge in that case.  */
    2736          338 :           else if (is_gimple_call (copy_stmt)
    2737          338 :                    && gimple_call_internal_p (copy_stmt)
    2738            0 :                    && (gimple_call_internal_fn (copy_stmt)
    2739              :                        == IFN_ABNORMAL_DISPATCHER)
    2740          338 :                    && gimple_call_arg (copy_stmt, 0) == boolean_true_node)
    2741              :             nonlocal_goto = false;
    2742              :           else
    2743              :             {
    2744          338 :               make_single_succ_edge (copy_stmt_bb, abnormal_goto_dest,
    2745              :                                      EDGE_ABNORMAL);
    2746          338 :               gimple_call_set_ctrl_altering (copy_stmt, true);
    2747          338 :               if (is_a <gcall *> (copy_stmt)
    2748          338 :                   && (gimple_call_flags (copy_stmt) & ECF_NORETURN))
    2749           78 :                 fixup_noreturn_call (copy_stmt);
    2750              :             }
    2751              :         }
    2752              : 
    2753     96148390 :       if ((can_throw || nonlocal_goto)
    2754     96933982 :           && gimple_in_ssa_p (cfun))
    2755       785592 :         update_ssa_across_abnormal_edges (gimple_bb (copy_stmt), ret_bb,
    2756              :                                           can_throw, nonlocal_goto);
    2757              :     }
    2758              :   return need_debug_cleanup;
    2759              : }
    2760              : 
    2761              : /* Copy the PHIs.  All blocks and edges are copied, some blocks
    2762              :    was possibly split and new outgoing EH edges inserted.
    2763              :    BB points to the block of original function and AUX pointers links
    2764              :    the original and newly copied blocks.  */
    2765              : 
    2766              : static void
    2767     24509317 : copy_phis_for_bb (basic_block bb, copy_body_data *id)
    2768              : {
    2769     24509317 :   basic_block const new_bb = (basic_block) bb->aux;
    2770     24509317 :   edge_iterator ei;
    2771     24509317 :   gphi *phi;
    2772     24509317 :   gphi_iterator si;
    2773     24509317 :   edge new_edge;
    2774     24509317 :   bool inserted = false;
    2775              : 
    2776     27578151 :   for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
    2777              :     {
    2778      3068834 :       tree res, new_res;
    2779      3068834 :       gphi *new_phi;
    2780              : 
    2781      3068834 :       phi = si.phi ();
    2782      3068834 :       res = PHI_RESULT (phi);
    2783      3068834 :       new_res = res;
    2784      3068834 :       if (!virtual_operand_p (res)
    2785      3068834 :           && (!id->param_body_adjs
    2786      3068834 :               || !id->param_body_adjs->m_dead_stmts.contains (phi)))
    2787              :         {
    2788      1706419 :           walk_tree (&new_res, copy_tree_body_r, id, NULL);
    2789      1706419 :           if (EDGE_COUNT (new_bb->preds) == 0)
    2790              :             {
    2791              :               /* Technically we'd want a SSA_DEFAULT_DEF here... */
    2792            0 :               SSA_NAME_DEF_STMT (new_res) = gimple_build_nop ();
    2793              :             }
    2794              :           else
    2795              :             {
    2796      1706419 :               new_phi = create_phi_node (new_res, new_bb);
    2797      5599264 :               FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
    2798              :                 {
    2799      3892845 :                   edge old_edge = find_edge ((basic_block) new_edge->src->aux,
    2800      3892845 :                                              bb);
    2801      3892845 :                   tree arg;
    2802      3892845 :                   tree new_arg;
    2803      3892845 :                   edge_iterator ei2;
    2804      3892845 :                   location_t locus;
    2805              : 
    2806              :                   /* When doing partial cloning, we allow PHIs on the entry
    2807              :                      block as long as all the arguments are the same.
    2808              :                      Find any input edge to see argument to copy.  */
    2809      3892845 :                   if (!old_edge)
    2810         2057 :                     FOR_EACH_EDGE (old_edge, ei2, bb->preds)
    2811         2057 :                       if (!old_edge->src->aux)
    2812              :                         break;
    2813              : 
    2814      3892845 :                   arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge);
    2815      3892845 :                   new_arg = arg;
    2816      3892845 :                   walk_tree (&new_arg, copy_tree_body_r, id, NULL);
    2817      3892845 :                   gcc_assert (new_arg);
    2818              :                   /* With return slot optimization we can end up with
    2819              :                      non-gimple (foo *)&this->m, fix that here.  */
    2820      3892845 :                   if (TREE_CODE (new_arg) != SSA_NAME
    2821      1582803 :                       && TREE_CODE (new_arg) != FUNCTION_DECL
    2822      5475648 :                       && !is_gimple_val (new_arg))
    2823              :                     {
    2824           12 :                       gimple_seq stmts = NULL;
    2825           12 :                       new_arg = force_gimple_operand (new_arg, &stmts, true,
    2826              :                                                       NULL);
    2827           12 :                       gsi_insert_seq_on_edge (new_edge, stmts);
    2828           12 :                       inserted = true;
    2829              :                     }
    2830      3892845 :                   locus = gimple_phi_arg_location_from_edge (phi, old_edge);
    2831      3892845 :                   if (id->reset_location)
    2832            0 :                     locus = input_location;
    2833              :                   else
    2834      3892845 :                     locus = remap_location (locus, id);
    2835      3892845 :                   add_phi_arg (new_phi, new_arg, new_edge, locus);
    2836              :                 }
    2837              :             }
    2838              :         }
    2839              :     }
    2840              : 
    2841              :   /* Commit the delayed edge insertions.  */
    2842     24509317 :   if (inserted)
    2843           36 :     FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
    2844           24 :       gsi_commit_one_edge_insert (new_edge, NULL);
    2845     24509317 : }
    2846              : 
    2847              : 
    2848              : /* Wrapper for remap_decl so it can be used as a callback.  */
    2849              : 
    2850              : static tree
    2851       104529 : remap_decl_1 (tree decl, void *data)
    2852              : {
    2853       104529 :   return remap_decl (decl, (copy_body_data *) data);
    2854              : }
    2855              : 
    2856              : /* Build struct function and associated datastructures for the new clone
    2857              :    NEW_FNDECL to be build.  CALLEE_FNDECL is the original.  Function changes
    2858              :    the cfun to the function of new_fndecl (and current_function_decl too).  */
    2859              : 
    2860              : static void
    2861       253174 : initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
    2862              : {
    2863       253174 :   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
    2864              : 
    2865              :   /* Register specific tree functions.  */
    2866       253174 :   gimple_register_cfg_hooks ();
    2867              : 
    2868              :   /* Get clean struct function.  */
    2869       253174 :   push_struct_function (new_fndecl, true);
    2870       253174 :   targetm.target_option.relayout_function (new_fndecl);
    2871              : 
    2872              :   /* We will rebuild these, so just sanity check that they are empty.  */
    2873       253174 :   gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL);
    2874       253174 :   gcc_assert (cfun->local_decls == NULL);
    2875       253174 :   gcc_assert (cfun->cfg == NULL);
    2876       253174 :   gcc_assert (cfun->decl == new_fndecl);
    2877              : 
    2878              :   /* Copy items we preserve during cloning.  */
    2879       253174 :   cfun->static_chain_decl = src_cfun->static_chain_decl;
    2880       253174 :   cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area;
    2881       253174 :   cfun->function_end_locus = src_cfun->function_end_locus;
    2882       253174 :   cfun->curr_properties = src_cfun->curr_properties;
    2883       253174 :   cfun->va_list_gpr_size = src_cfun->va_list_gpr_size;
    2884       253174 :   cfun->va_list_fpr_size = src_cfun->va_list_fpr_size;
    2885       253174 :   cfun->has_nonlocal_label = src_cfun->has_nonlocal_label;
    2886       253174 :   cfun->calls_eh_return = src_cfun->calls_eh_return;
    2887       253174 :   cfun->stdarg = src_cfun->stdarg;
    2888       253174 :   cfun->after_inlining = src_cfun->after_inlining;
    2889       253174 :   cfun->can_throw_non_call_exceptions
    2890       253174 :     = src_cfun->can_throw_non_call_exceptions;
    2891       253174 :   cfun->can_delete_dead_exceptions = src_cfun->can_delete_dead_exceptions;
    2892       253174 :   cfun->returns_struct = src_cfun->returns_struct;
    2893       253174 :   cfun->returns_pcc_struct = src_cfun->returns_pcc_struct;
    2894       253174 :   cfun->assume_function = src_cfun->assume_function;
    2895              : 
    2896       253174 :   init_empty_tree_cfg ();
    2897              : 
    2898       253174 :   profile_status_for_fn (cfun) = profile_status_for_fn (src_cfun);
    2899       253174 :   cfun->cfg->full_profile = src_cfun->cfg->full_profile;
    2900              : 
    2901       253174 :   profile_count num = count;
    2902       253174 :   profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
    2903       253174 :   profile_count::adjust_for_ipa_scaling (&num, &den);
    2904              : 
    2905       253174 :   ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
    2906       253174 :     ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
    2907       253174 :   EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
    2908       253174 :     EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
    2909       253174 :   if (src_cfun->eh)
    2910       253174 :     init_eh_for_function ();
    2911              : 
    2912       253174 :   if (src_cfun->gimple_df)
    2913              :     {
    2914       253174 :       init_tree_ssa (cfun);
    2915       253174 :       cfun->gimple_df->in_ssa_p = src_cfun->gimple_df->in_ssa_p;
    2916       253174 :       if (cfun->gimple_df->in_ssa_p)
    2917       253174 :         init_ssa_operands (cfun);
    2918              :     }
    2919       253174 : }
    2920              : 
    2921              : /* Helper function for copy_cfg_body.  Move debug stmts from the end
    2922              :    of NEW_BB to the beginning of successor basic blocks when needed.  If the
    2923              :    successor has multiple predecessors, reset them, otherwise keep
    2924              :    their value.  */
    2925              : 
    2926              : static void
    2927      1649537 : maybe_move_debug_stmts_to_successors (copy_body_data *id, basic_block new_bb)
    2928              : {
    2929      1649537 :   edge e;
    2930      1649537 :   edge_iterator ei;
    2931      1649537 :   gimple_stmt_iterator si = gsi_last_nondebug_bb (new_bb);
    2932              : 
    2933      1649537 :   if (gsi_end_p (si)
    2934      1720417 :       || gsi_one_before_end_p (si)
    2935      1819392 :       || !(stmt_can_throw_internal (cfun, gsi_stmt (si))
    2936        70881 :            || stmt_can_make_abnormal_goto (gsi_stmt (si))))
    2937      1621443 :     return;
    2938              : 
    2939        84284 :   FOR_EACH_EDGE (e, ei, new_bb->succs)
    2940              :     {
    2941        56190 :       gimple_stmt_iterator ssi = gsi_last_bb (new_bb);
    2942        56190 :       gimple_stmt_iterator dsi = gsi_after_labels (e->dest);
    2943       177061 :       while (is_gimple_debug (gsi_stmt (ssi)))
    2944              :         {
    2945       120871 :           gimple *stmt = gsi_stmt (ssi);
    2946       120871 :           gdebug *new_stmt;
    2947       120871 :           tree var;
    2948       120871 :           tree value;
    2949              : 
    2950              :           /* For the last edge move the debug stmts instead of copying
    2951              :              them.  */
    2952       120871 :           if (ei_one_before_end_p (ei))
    2953              :             {
    2954        60434 :               si = ssi;
    2955        60434 :               gsi_prev (&ssi);
    2956        60434 :               if (!single_pred_p (e->dest) && gimple_debug_bind_p (stmt))
    2957              :                 {
    2958        58912 :                   gimple_debug_bind_reset_value (stmt);
    2959       117185 :                   gimple_set_location (stmt, UNKNOWN_LOCATION);
    2960              :                 }
    2961        60434 :               gsi_remove (&si, false);
    2962        60434 :               gsi_insert_before (&dsi, stmt, GSI_NEW_STMT);
    2963        60434 :               continue;
    2964              :             }
    2965              : 
    2966        60437 :           if (gimple_debug_bind_p (stmt))
    2967              :             {
    2968        58915 :               var = gimple_debug_bind_get_var (stmt);
    2969        58915 :               if (single_pred_p (e->dest))
    2970              :                 {
    2971        49230 :                   value = gimple_debug_bind_get_value (stmt);
    2972        49230 :                   value = unshare_expr (value);
    2973        49230 :                   new_stmt = gimple_build_debug_bind (var, value, stmt);
    2974              :                 }
    2975              :               else
    2976         9685 :                 new_stmt = gimple_build_debug_bind (var, NULL_TREE, NULL);
    2977              :             }
    2978         1522 :           else if (gimple_debug_source_bind_p (stmt))
    2979              :             {
    2980            0 :               var = gimple_debug_source_bind_get_var (stmt);
    2981            0 :               value = gimple_debug_source_bind_get_value (stmt);
    2982            0 :               new_stmt = gimple_build_debug_source_bind (var, value, stmt);
    2983              :             }
    2984         1522 :           else if (gimple_debug_nonbind_marker_p (stmt))
    2985         1522 :             new_stmt = as_a <gdebug *> (gimple_copy (stmt));
    2986              :           else
    2987            0 :             gcc_unreachable ();
    2988        60437 :           gsi_insert_before (&dsi, new_stmt, GSI_NEW_STMT);
    2989        60437 :           id->debug_stmts.safe_push (new_stmt);
    2990        60437 :           gsi_prev (&ssi);
    2991              :         }
    2992              :     }
    2993              : }
    2994              : 
    2995              : /* Make a copy of the sub-loops of SRC_PARENT and place them
    2996              :    as siblings of DEST_PARENT.  */
    2997              : 
    2998              : static void
    2999      5519654 : copy_loops (copy_body_data *id,
    3000              :             class loop *dest_parent, class loop *src_parent)
    3001              : {
    3002      5519654 :   class loop *src_loop = src_parent->inner;
    3003      6143841 :   while (src_loop)
    3004              :     {
    3005       624187 :       if (!id->blocks_to_copy
    3006       624187 :           || bitmap_bit_p (id->blocks_to_copy, src_loop->header->index))
    3007              :         {
    3008       620014 :           class loop *dest_loop = alloc_loop ();
    3009              : 
    3010              :           /* Assign the new loop its header and latch and associate
    3011              :              those with the new loop.  */
    3012       620014 :           dest_loop->header = (basic_block)src_loop->header->aux;
    3013       620014 :           dest_loop->header->loop_father = dest_loop;
    3014       620014 :           if (src_loop->latch != NULL)
    3015              :             {
    3016       619998 :               dest_loop->latch = (basic_block)src_loop->latch->aux;
    3017       619998 :               dest_loop->latch->loop_father = dest_loop;
    3018              :             }
    3019              : 
    3020              :           /* Copy loop meta-data.  */
    3021       620014 :           copy_loop_info (src_loop, dest_loop);
    3022       620014 :           if (dest_loop->unroll)
    3023         2192 :             cfun->has_unroll = true;
    3024       620014 :           if (dest_loop->force_vectorize)
    3025          154 :             cfun->has_force_vectorize_loops = true;
    3026       620014 :           if (id->src_cfun->last_clique != 0)
    3027        80423 :             dest_loop->owned_clique
    3028        80423 :               = remap_dependence_clique (id,
    3029        80423 :                                          src_loop->owned_clique
    3030              :                                          ? src_loop->owned_clique : 1);
    3031              : 
    3032              :           /* Finally place it into the loop array and the loop tree.  */
    3033       620014 :           place_new_loop (cfun, dest_loop);
    3034       620014 :           flow_loop_tree_node_add (dest_parent, dest_loop);
    3035              : 
    3036       620014 :           if (src_loop->simduid)
    3037              :             {
    3038           68 :               dest_loop->simduid = remap_decl (src_loop->simduid, id);
    3039           68 :               cfun->has_simduid_loops = true;
    3040              :             }
    3041              : 
    3042              :           /* Recurse.  */
    3043       620014 :           copy_loops (id, dest_loop, src_loop);
    3044              :         }
    3045       624187 :       src_loop = src_loop->next;
    3046              :     }
    3047      5519654 : }
    3048              : 
    3049              : /* Call redirect_call_stmt_to_callee on all calls in BB.  */
    3050              : 
    3051              : void
    3052     13249894 : redirect_all_calls (copy_body_data * id, basic_block bb)
    3053              : {
    3054     13249894 :   gimple_stmt_iterator si;
    3055     13249894 :   gimple *last = last_nondebug_stmt (bb);
    3056    109176106 :   for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
    3057              :     {
    3058     82676318 :       gimple *stmt = gsi_stmt (si);
    3059     82676318 :       if (is_gimple_call (stmt))
    3060              :         {
    3061      4287134 :           struct cgraph_edge *edge = id->dst_node->get_edge (stmt);
    3062      4287134 :           if (edge)
    3063              :             {
    3064      4241335 :               if (!id->killed_new_ssa_names)
    3065      2260969 :                 id->killed_new_ssa_names = new hash_set<tree> (16);
    3066      4241335 :               cgraph_edge::redirect_call_stmt_to_callee (
    3067              :                 edge, id->killed_new_ssa_names);
    3068      4241335 :               if (edge->has_callback)
    3069              :                 {
    3070              :                   /* When redirecting a carrying edge, we need to redirect its
    3071              :                      attached edges as well.  */
    3072         2394 :                   cgraph_edge *cbe;
    3073         2846 :                   for (cbe = edge->first_callback_edge (); cbe;
    3074          452 :                        cbe = cbe->next_callback_edge ())
    3075          452 :                     cgraph_edge::redirect_call_stmt_to_callee (
    3076              :                       cbe, id->killed_new_ssa_names);
    3077              :                 }
    3078              : 
    3079      4241335 :               if (stmt == last && id->call_stmt && maybe_clean_eh_stmt (stmt))
    3080        59996 :                 gimple_purge_dead_eh_edges (bb);
    3081              :             }
    3082              :         }
    3083              :     }
    3084     13249894 : }
    3085              : 
    3086              : /* Make a copy of the body of FN so that it can be inserted inline in
    3087              :    another function.  Walks FN via CFG, returns new fndecl.  */
    3088              : 
    3089              : static tree
    3090      4899640 : copy_cfg_body (copy_body_data * id,
    3091              :                basic_block entry_block_map, basic_block exit_block_map,
    3092              :                basic_block new_entry)
    3093              : {
    3094      4899640 :   tree callee_fndecl = id->src_fn;
    3095              :   /* Original cfun for the callee, doesn't change.  */
    3096      4899640 :   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
    3097      4899640 :   struct function *cfun_to_copy;
    3098      4899640 :   basic_block bb;
    3099      4899640 :   tree new_fndecl = NULL;
    3100      4899640 :   bool need_debug_cleanup = false;
    3101      4899640 :   int last;
    3102      4899640 :   profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
    3103      4899640 :   profile_count num = entry_block_map->count;
    3104              : 
    3105      4899640 :   cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
    3106              : 
    3107              :   /* Register specific tree functions.  */
    3108      4899640 :   gimple_register_cfg_hooks ();
    3109              : 
    3110              :   /* If we are offlining region of the function, make sure to connect
    3111              :      new entry to ENTRY_BLOCK_PTR_FOR_FN (cfun).  Since new entry can be
    3112              :      part of loop, we must compute frequency and probability of
    3113              :      ENTRY_BLOCK_PTR_FOR_FN (cfun) based on the frequencies and
    3114              :      probabilities of edges incoming from nonduplicated region.  */
    3115      4899640 :   if (new_entry)
    3116              :     {
    3117        52605 :       edge e;
    3118        52605 :       edge_iterator ei;
    3119        52605 :       ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ();
    3120              : 
    3121       113283 :       FOR_EACH_EDGE (e, ei, new_entry->preds)
    3122        60678 :         if (!e->src->aux)
    3123        60678 :           ENTRY_BLOCK_PTR_FOR_FN (cfun)->count += e->count ();
    3124              :       /* Do not scale - the profile of offlined region should
    3125              :          remain unchanged.  */
    3126        52605 :       num = den = profile_count::one ();
    3127              :     }
    3128              : 
    3129      4899640 :   profile_count::adjust_for_ipa_scaling (&num, &den);
    3130              : 
    3131              :   /* Must have a CFG here at this point.  */
    3132      4899640 :   gcc_assert (ENTRY_BLOCK_PTR_FOR_FN
    3133              :               (DECL_STRUCT_FUNCTION (callee_fndecl)));
    3134              : 
    3135              : 
    3136      4899640 :   ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = entry_block_map;
    3137      4899640 :   EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = exit_block_map;
    3138      4899640 :   entry_block_map->aux = ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy);
    3139      4899640 :   exit_block_map->aux = EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy);
    3140              : 
    3141              :   /* Duplicate any exception-handling regions.  */
    3142      4899640 :   if (cfun->eh)
    3143      4899640 :     id->eh_map = duplicate_eh_regions (cfun_to_copy, NULL, id->eh_lp_nr,
    3144              :                                        remap_decl_1, id);
    3145              : 
    3146              :   /* Use aux pointers to map the original blocks to copy.  */
    3147     19898457 :   FOR_EACH_BB_FN (bb, cfun_to_copy)
    3148     14998817 :     if (!id->blocks_to_copy || bitmap_bit_p (id->blocks_to_copy, bb->index))
    3149              :       {
    3150     14815247 :         basic_block new_bb = copy_bb (id, bb, num, den);
    3151     14815247 :         bb->aux = new_bb;
    3152     14815247 :         new_bb->aux = bb;
    3153     14815247 :         new_bb->loop_father = entry_block_map->loop_father;
    3154              :       }
    3155              : 
    3156      4899640 :   last = last_basic_block_for_fn (cfun);
    3157              : 
    3158              :   /* Now that we've duplicated the blocks, duplicate their edges.  */
    3159      4899640 :   basic_block abnormal_goto_dest = NULL;
    3160      4899640 :   if (id->call_stmt
    3161      4899640 :       && stmt_can_make_abnormal_goto (id->call_stmt))
    3162              :     {
    3163          327 :       gimple_stmt_iterator gsi = gsi_for_stmt (id->call_stmt);
    3164              : 
    3165          327 :       bb = gimple_bb (id->call_stmt);
    3166          327 :       gsi_next (&gsi);
    3167          327 :       if (gsi_end_p (gsi))
    3168          327 :         abnormal_goto_dest = get_abnormal_succ_dispatcher (bb);
    3169              :     }
    3170     29697737 :   FOR_ALL_BB_FN (bb, cfun_to_copy)
    3171     24798097 :     if (!id->blocks_to_copy
    3172     24798097 :         || (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
    3173     24509317 :       need_debug_cleanup |= copy_edges_for_bb (bb, num, den, exit_block_map,
    3174              :                                                abnormal_goto_dest, id);
    3175              : 
    3176      4899640 :   if (id->eh_landing_pad_dest)
    3177              :     {
    3178       453637 :       add_clobbers_to_eh_landing_pad (id);
    3179       453637 :       id->eh_landing_pad_dest = NULL;
    3180              :     }
    3181              : 
    3182      4899640 :   if (new_entry)
    3183              :     {
    3184        52605 :       edge e = make_edge (entry_block_map, (basic_block)new_entry->aux,
    3185              :                           EDGE_FALLTHRU);
    3186        52605 :       e->probability = profile_probability::always ();
    3187              :     }
    3188              : 
    3189              :   /* Duplicate the loop tree, if available and wanted.  */
    3190      4899640 :   if (loops_for_fn (src_cfun) != NULL
    3191      4899640 :       && current_loops != NULL)
    3192              :     {
    3193      4899640 :       copy_loops (id, entry_block_map->loop_father,
    3194              :                   get_loop (src_cfun, 0));
    3195              :       /* Defer to cfgcleanup to update loop-father fields of basic-blocks.  */
    3196      4899640 :       loops_state_set (LOOPS_NEED_FIXUP);
    3197              :     }
    3198              : 
    3199              :   /* If the loop tree in the source function needed fixup, mark the
    3200              :      destination loop tree for fixup, too.  */
    3201      4899640 :   if (loops_for_fn (src_cfun)->state & LOOPS_NEED_FIXUP)
    3202            0 :     loops_state_set (LOOPS_NEED_FIXUP);
    3203              : 
    3204      4899640 :   if (gimple_in_ssa_p (cfun))
    3205     29697737 :     FOR_ALL_BB_FN (bb, cfun_to_copy)
    3206     24798097 :       if (!id->blocks_to_copy
    3207     24798097 :           || (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
    3208     24509317 :         copy_phis_for_bb (bb, id);
    3209              : 
    3210     29697737 :   FOR_ALL_BB_FN (bb, cfun_to_copy)
    3211     24798097 :     if (bb->aux)
    3212              :       {
    3213     24614527 :         if (need_debug_cleanup
    3214      2169495 :             && bb->index != ENTRY_BLOCK
    3215      1848474 :             && bb->index != EXIT_BLOCK)
    3216      1527453 :           maybe_move_debug_stmts_to_successors (id, (basic_block) bb->aux);
    3217              :         /* Update call edge destinations.  This cannot be done before loop
    3218              :            info is updated, because we may split basic blocks.  */
    3219     24614527 :         if (id->transform_call_graph_edges == CB_CGE_DUPLICATE
    3220     22171331 :             && bb->index != ENTRY_BLOCK
    3221     17524865 :             && bb->index != EXIT_BLOCK)
    3222     12878399 :           redirect_all_calls (id, (basic_block)bb->aux);
    3223     24614527 :         ((basic_block)bb->aux)->aux = NULL;
    3224     24614527 :         bb->aux = NULL;
    3225              :       }
    3226              : 
    3227              :   /* Zero out AUX fields of newly created block during EH edge
    3228              :      insertion. */
    3229      5271135 :   for (; last < last_basic_block_for_fn (cfun); last++)
    3230              :     {
    3231       371495 :       if (need_debug_cleanup)
    3232       122084 :         maybe_move_debug_stmts_to_successors (id,
    3233       122084 :                                               BASIC_BLOCK_FOR_FN (cfun, last));
    3234       371495 :       BASIC_BLOCK_FOR_FN (cfun, last)->aux = NULL;
    3235              :       /* Update call edge destinations.  This cannot be done before loop
    3236              :          info is updated, because we may split basic blocks.  */
    3237       371495 :       if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
    3238       371495 :         redirect_all_calls (id, BASIC_BLOCK_FOR_FN (cfun, last));
    3239              :     }
    3240      4899640 :   entry_block_map->aux = NULL;
    3241      4899640 :   exit_block_map->aux = NULL;
    3242              : 
    3243      4899640 :   if (id->eh_map)
    3244              :     {
    3245      4899640 :       delete id->eh_map;
    3246      4899640 :       id->eh_map = NULL;
    3247              :     }
    3248      4899640 :   if (id->dependence_map)
    3249              :     {
    3250       733457 :       delete id->dependence_map;
    3251       733457 :       id->dependence_map = NULL;
    3252              :     }
    3253              : 
    3254      4899640 :   return new_fndecl;
    3255              : }
    3256              : 
    3257              : /* Copy the debug STMT using ID.  We deal with these statements in a
    3258              :    special way: if any variable in their VALUE expression wasn't
    3259              :    remapped yet, we won't remap it, because that would get decl uids
    3260              :    out of sync, causing codegen differences between -g and -g0.  If
    3261              :    this arises, we drop the VALUE expression altogether.  */
    3262              : 
    3263              : static void
    3264     62063662 : copy_debug_stmt (gdebug *stmt, copy_body_data *id)
    3265              : {
    3266     62063662 :   tree t, *n;
    3267     62063662 :   struct walk_stmt_info wi;
    3268              : 
    3269     62063662 :   if (tree block = gimple_block (stmt))
    3270              :     {
    3271     40601259 :       n = id->decl_map->get (block);
    3272     40601259 :       gimple_set_block (stmt, n ? *n : id->block);
    3273              :     }
    3274              : 
    3275     62063662 :   if (gimple_debug_nonbind_marker_p (stmt))
    3276              :     {
    3277     14007876 :       if (id->call_stmt && !gimple_block (stmt))
    3278              :         {
    3279        37451 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    3280        37451 :           gsi_remove (&gsi, true);
    3281              :         }
    3282     14007876 :       return;
    3283              :     }
    3284              : 
    3285              :   /* Remap all the operands in COPY.  */
    3286     48055786 :   memset (&wi, 0, sizeof (wi));
    3287     48055786 :   wi.info = id;
    3288              : 
    3289     48055786 :   processing_debug_stmt = 1;
    3290              : 
    3291     48055786 :   if (gimple_debug_source_bind_p (stmt))
    3292       563968 :     t = gimple_debug_source_bind_get_var (stmt);
    3293     47491818 :   else if (gimple_debug_bind_p (stmt))
    3294     47491818 :     t = gimple_debug_bind_get_var (stmt);
    3295              :   else
    3296            0 :     gcc_unreachable ();
    3297              : 
    3298     48055786 :   if (TREE_CODE (t) == PARM_DECL
    3299       266888 :       && id->debug_map
    3300     48057622 :       && (n = id->debug_map->get (t)))
    3301              :     {
    3302            0 :       gcc_assert (VAR_P (*n));
    3303            0 :       t = *n;
    3304              :     }
    3305     48055786 :   else if (VAR_P (t) && !is_global_var (t) && !id->decl_map->get (t))
    3306              :     /* T is a non-localized variable.  */;
    3307              :   else
    3308     41531059 :     walk_tree (&t, remap_gimple_op_r, &wi, NULL);
    3309              : 
    3310     48055786 :   if (gimple_debug_bind_p (stmt))
    3311              :     {
    3312     47491818 :       gimple_debug_bind_set_var (stmt, t);
    3313              : 
    3314     47491818 :       if (gimple_debug_bind_has_value_p (stmt))
    3315     27033318 :         walk_tree (gimple_debug_bind_get_value_ptr (stmt),
    3316              :                    remap_gimple_op_r, &wi, NULL);
    3317              : 
    3318              :       /* Punt if any decl couldn't be remapped.  */
    3319     47491818 :       if (processing_debug_stmt < 0)
    3320       678185 :         gimple_debug_bind_reset_value (stmt);
    3321              :     }
    3322       563968 :   else if (gimple_debug_source_bind_p (stmt))
    3323              :     {
    3324       563968 :       gimple_debug_source_bind_set_var (stmt, t);
    3325              :       /* When inlining and source bind refers to one of the optimized
    3326              :          away parameters, change the source bind into normal debug bind
    3327              :          referring to the corresponding DEBUG_EXPR_DECL that should have
    3328              :          been bound before the call stmt.  */
    3329       563968 :       t = gimple_debug_source_bind_get_value (stmt);
    3330       563968 :       if (t != NULL_TREE
    3331       563968 :           && TREE_CODE (t) == PARM_DECL
    3332       429389 :           && id->call_stmt)
    3333              :         {
    3334       424322 :           vec<tree, va_gc> **debug_args = decl_debug_args_lookup (id->src_fn);
    3335       424322 :           unsigned int i;
    3336       424322 :           if (debug_args != NULL)
    3337              :             {
    3338       535901 :               for (i = 0; i < vec_safe_length (*debug_args); i += 2)
    3339       535900 :                 if ((**debug_args)[i] == DECL_ORIGIN (t)
    3340       535900 :                     && TREE_CODE ((**debug_args)[i + 1]) == DEBUG_EXPR_DECL)
    3341              :                   {
    3342       418474 :                     t = (**debug_args)[i + 1];
    3343       418474 :                     stmt->subcode = GIMPLE_DEBUG_BIND;
    3344       418474 :                     gimple_debug_bind_set_value (stmt, t);
    3345       418474 :                     break;
    3346              :                   }
    3347              :             }
    3348              :         }
    3349       563968 :       if (gimple_debug_source_bind_p (stmt))
    3350       145494 :         walk_tree (gimple_debug_source_bind_get_value_ptr (stmt),
    3351              :                    remap_gimple_op_r, &wi, NULL);
    3352              :     }
    3353              : 
    3354     48055786 :   processing_debug_stmt = 0;
    3355              : 
    3356     48055786 :   update_stmt (stmt);
    3357              : }
    3358              : 
    3359              : /* Process deferred debug stmts.  In order to give values better odds
    3360              :    of being successfully remapped, we delay the processing of debug
    3361              :    stmts until all other stmts that might require remapping are
    3362              :    processed.  */
    3363              : 
    3364              : static void
    3365      4899640 : copy_debug_stmts (copy_body_data *id)
    3366              : {
    3367      4899640 :   if (!id->debug_stmts.exists ())
    3368              :     return;
    3369              : 
    3370     65163648 :   for (gdebug *stmt : id->debug_stmts)
    3371              :     /* But avoid re-processing debug stmts that have been elided.  */
    3372     62063662 :     if (gimple_bb (stmt))
    3373     62063662 :       copy_debug_stmt (stmt, id);
    3374              : 
    3375      3099986 :   id->debug_stmts.release ();
    3376              : }
    3377              : 
    3378              : /* Make a copy of the body of SRC_FN so that it can be inserted inline in
    3379              :    another function.  */
    3380              : 
    3381              : static tree
    3382     31020400 : copy_tree_body (copy_body_data *id)
    3383              : {
    3384     31020400 :   tree fndecl = id->src_fn;
    3385     31020400 :   tree body = DECL_SAVED_TREE (fndecl);
    3386              : 
    3387     31020400 :   walk_tree (&body, copy_tree_body_r, id, NULL);
    3388              : 
    3389     31020400 :   return body;
    3390              : }
    3391              : 
    3392              : /* Make a copy of the body of FN so that it can be inserted inline in
    3393              :    another function.  */
    3394              : 
    3395              : static tree
    3396      4899640 : copy_body (copy_body_data *id,
    3397              :            basic_block entry_block_map, basic_block exit_block_map,
    3398              :            basic_block new_entry)
    3399              : {
    3400      4899640 :   tree fndecl = id->src_fn;
    3401      4899640 :   tree body;
    3402              : 
    3403              :   /* If this body has a CFG, walk CFG and copy.  */
    3404      4899640 :   gcc_assert (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (fndecl)));
    3405      4899640 :   body = copy_cfg_body (id, entry_block_map, exit_block_map,
    3406              :                         new_entry);
    3407      4899640 :   copy_debug_stmts (id);
    3408      4899640 :   if (id->killed_new_ssa_names)
    3409              :     {
    3410      2260969 :       ipa_release_ssas_in_hash (id->killed_new_ssa_names);
    3411      4521938 :       delete id->killed_new_ssa_names;
    3412      2260969 :       id->killed_new_ssa_names = NULL;
    3413              :     }
    3414              : 
    3415      4899640 :   return body;
    3416              : }
    3417              : 
    3418              : /* Return true if VALUE is an ADDR_EXPR of an automatic variable
    3419              :    defined in function FN, or of a data member thereof.  */
    3420              : 
    3421              : static bool
    3422       168330 : self_inlining_addr_expr (tree value, tree fn)
    3423              : {
    3424       168330 :   tree var;
    3425              : 
    3426       168330 :   if (TREE_CODE (value) != ADDR_EXPR)
    3427              :     return false;
    3428              : 
    3429       161254 :   var = get_base_address (TREE_OPERAND (value, 0));
    3430              : 
    3431       161254 :   return var && auto_var_in_fn_p (var, fn);
    3432              : }
    3433              : 
    3434              : /* Append to BB a debug annotation that binds VAR to VALUE, inheriting
    3435              :    lexical block and line number information from base_stmt, if given,
    3436              :    or from the last stmt of the block otherwise.  */
    3437              : 
    3438              : static gimple *
    3439      7970348 : insert_init_debug_bind (copy_body_data *id,
    3440              :                         basic_block bb, tree var, tree value,
    3441              :                         gimple *base_stmt)
    3442              : {
    3443      7970348 :   gimple *note;
    3444      7970348 :   gimple_stmt_iterator gsi;
    3445      7970348 :   tree tracked_var;
    3446              : 
    3447      7970348 :   if (!gimple_in_ssa_p (id->src_cfun))
    3448              :     return NULL;
    3449              : 
    3450      7970348 :   if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
    3451              :     return NULL;
    3452              : 
    3453      7051751 :   tracked_var = target_for_debug_bind (var);
    3454      7051751 :   if (!tracked_var)
    3455              :     return NULL;
    3456              : 
    3457      6382752 :   if (bb)
    3458              :     {
    3459      6375741 :       gsi = gsi_last_bb (bb);
    3460      6375741 :       if (!base_stmt && !gsi_end_p (gsi))
    3461              :         base_stmt = gsi_stmt (gsi);
    3462              :     }
    3463              : 
    3464      6382752 :   note = gimple_build_debug_bind (tracked_var,
    3465      6382752 :                                   value == error_mark_node
    3466      6382752 :                                   ? NULL_TREE : unshare_expr (value),
    3467              :                                   base_stmt);
    3468              : 
    3469      6382752 :   if (bb)
    3470              :     {
    3471      6375741 :       if (!gsi_end_p (gsi))
    3472      5533234 :         gsi_insert_after (&gsi, note, GSI_SAME_STMT);
    3473              :       else
    3474       842507 :         gsi_insert_before (&gsi, note, GSI_SAME_STMT);
    3475              :     }
    3476              : 
    3477              :   return note;
    3478              : }
    3479              : 
    3480              : static void
    3481       483876 : insert_init_stmt (copy_body_data *id, basic_block bb, gimple *init_stmt)
    3482              : {
    3483              :   /* If VAR represents a zero-sized variable, it's possible that the
    3484              :      assignment statement may result in no gimple statements.  */
    3485       483876 :   if (init_stmt)
    3486              :     {
    3487       483876 :       gimple_stmt_iterator si = gsi_last_bb (bb);
    3488              : 
    3489              :       /* We can end up with init statements that store to a non-register
    3490              :          from a rhs with a conversion.  Handle that here by forcing the
    3491              :          rhs into a temporary.  gimple_regimplify_operands is not
    3492              :          prepared to do this for us.  */
    3493       483876 :       if (!is_gimple_debug (init_stmt)
    3494       476865 :           && !is_gimple_reg (gimple_assign_lhs (init_stmt))
    3495       368178 :           && is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt)))
    3496       510089 :           && gimple_assign_rhs_class (init_stmt) == GIMPLE_UNARY_RHS)
    3497              :         {
    3498            2 :           tree rhs = build1 (gimple_assign_rhs_code (init_stmt),
    3499            1 :                              TREE_TYPE (gimple_assign_lhs (init_stmt)),
    3500              :                              gimple_assign_rhs1 (init_stmt));
    3501            1 :           rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false,
    3502              :                                           GSI_NEW_STMT);
    3503            1 :           gimple_assign_set_rhs_code (init_stmt, TREE_CODE (rhs));
    3504            1 :           gimple_assign_set_rhs1 (init_stmt, rhs);
    3505              :         }
    3506       483876 :       gsi_insert_after (&si, init_stmt, GSI_NEW_STMT);
    3507       483876 :       if (!is_gimple_debug (init_stmt))
    3508              :         {
    3509       476865 :           gimple_regimplify_operands (init_stmt, &si);
    3510              : 
    3511       476865 :           tree def = gimple_assign_lhs (init_stmt);
    3512       476865 :           insert_init_debug_bind (id, bb, def, def, init_stmt);
    3513              :         }
    3514              :     }
    3515       483876 : }
    3516              : 
    3517              : /* Deal with mismatched formal/actual parameters, in a rather brute-force way
    3518              :    if need be (which should only be necessary for invalid programs).  Attempt
    3519              :    to convert VAL to TYPE and return the result if it is possible, just return
    3520              :    a zero constant of the given type if it fails.  */
    3521              : 
    3522              : tree
    3523          973 : force_value_to_type (tree type, tree value)
    3524              : {
    3525              :   /* If we can match up types by promotion/demotion do so.  */
    3526          973 :   if (fold_convertible_p (type, value))
    3527          549 :     return fold_convert (type, value);
    3528              : 
    3529              :   /* ???  For valid programs we should not end up here.
    3530              :      Still if we end up with truly mismatched types here, fall back
    3531              :      to using a VIEW_CONVERT_EXPR or a literal zero to not leak invalid
    3532              :      GIMPLE to the following passes.  */
    3533          424 :   if (TREE_CODE (value) == WITH_SIZE_EXPR)
    3534            0 :     return error_mark_node;
    3535          424 :   else if (!is_gimple_reg_type (TREE_TYPE (value))
    3536          424 :            || TYPE_SIZE (type) == TYPE_SIZE (TREE_TYPE (value)))
    3537          133 :     return fold_build1 (VIEW_CONVERT_EXPR, type, value);
    3538              :   else
    3539          291 :     return build_zero_cst (type);
    3540              : }
    3541              : 
    3542              : /* Initialize parameter P with VALUE.  If needed, produce init statement
    3543              :    at the end of BB.  When BB is NULL, we return init statement to be
    3544              :    output later.  */
    3545              : static gimple *
    3546      7960136 : setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn,
    3547              :                      basic_block bb, tree *vars)
    3548              : {
    3549      7960136 :   gimple *init_stmt = NULL;
    3550      7960136 :   tree var;
    3551      7960136 :   tree def = (gimple_in_ssa_p (cfun)
    3552      7960136 :               ? ssa_default_def (id->src_cfun, p) : NULL);
    3553              : 
    3554              :   /* Make an equivalent VAR_DECL.  Note that we must NOT remap the type
    3555              :      here since the type of this decl must be visible to the calling
    3556              :      function.  */
    3557      7960136 :   var = copy_decl_to_var (p, id);
    3558              : 
    3559              :   /* Declare this new variable.  */
    3560      7960136 :   DECL_CHAIN (var) = *vars;
    3561      7960136 :   *vars = var;
    3562              : 
    3563              :   /* Make gimplifier happy about this variable.  */
    3564      7960136 :   DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
    3565              : 
    3566              :   /* If the parameter is never assigned to, has no SSA_NAMEs created,
    3567              :      we would not need to create a new variable here at all, if it
    3568              :      weren't for debug info.  Still, we can just use the argument
    3569              :      value.  */
    3570      7960136 :   if (TREE_READONLY (p)
    3571      4212762 :       && !TREE_ADDRESSABLE (p)
    3572      4211764 :       && value
    3573      4211750 :       && !TREE_SIDE_EFFECTS (value)
    3574     12171880 :       && !def)
    3575              :     {
    3576              :       /* We may produce non-gimple trees by adding NOPs or introduce invalid
    3577              :          sharing when the value is not constant or DECL.  And we need to make
    3578              :          sure that it cannot be modified from another path in the callee.  */
    3579       381590 :       if (((is_gimple_min_invariant (value)
    3580              :             /* When the parameter is used in a context that forces it to
    3581              :                not be a GIMPLE register avoid substituting something that
    3582              :                is not a decl there.  */
    3583       167880 :             && ! DECL_NOT_GIMPLE_REG_P (p))
    3584       213721 :            || (DECL_P (value) && TREE_READONLY (value))
    3585       213637 :            || (auto_var_in_fn_p (value, id->dst_fn)
    3586          385 :                && !TREE_ADDRESSABLE (value)))
    3587       168335 :           && useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value))
    3588              :           /* We have to be very careful about ADDR_EXPR.  Make sure
    3589              :              the base variable isn't a local variable of the inlined
    3590              :              function, e.g., when doing recursive inlining, direct or
    3591              :              mutually-recursive or whatever, which is why we don't
    3592              :              just test whether fn == current_function_decl.  */
    3593       549920 :           && ! self_inlining_addr_expr (value, fn))
    3594              :         {
    3595       168330 :           insert_decl_map (id, p, value);
    3596       168330 :           if (!id->debug_map)
    3597       163107 :             id->debug_map = new hash_map<tree, tree>;
    3598       168330 :           id->debug_map->put (p, var);
    3599       168330 :           return insert_init_debug_bind (id, bb, var, value, NULL);
    3600              :         }
    3601              :     }
    3602              : 
    3603              :   /* Register the VAR_DECL as the equivalent for the PARM_DECL;
    3604              :      that way, when the PARM_DECL is encountered, it will be
    3605              :      automatically replaced by the VAR_DECL.  */
    3606      7791806 :   insert_decl_map (id, p, var);
    3607              : 
    3608              :   /* Even if P was TREE_READONLY, the new VAR should not be.  In the original
    3609              :      code, we would have constructed a temporary, and then the function body
    3610              :      would have never changed the value of P.  However, now, we will be
    3611              :      constructing VAR directly.  Therefore, it must not be TREE_READONLY.  */
    3612      7791806 :   TREE_READONLY (var) = 0;
    3613              : 
    3614      7791806 :   tree rhs = value;
    3615      7791806 :   if (value
    3616      7790865 :       && value != error_mark_node
    3617     15582671 :       && !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value)))
    3618          962 :     rhs = force_value_to_type (TREE_TYPE (p), value);
    3619              : 
    3620              :   /* If there is no setup required and we are in SSA, take the easy route
    3621              :      replacing all SSA names representing the function parameter by the
    3622              :      SSA name passed to function.
    3623              : 
    3624              :      We need to construct map for the variable anyway as it might be used
    3625              :      in different SSA names when parameter is set in function.
    3626              : 
    3627              :      Do replacement at -O0 for const arguments replaced by constant.
    3628              :      This is important for builtin_constant_p and other construct requiring
    3629              :      constant argument to be visible in inlined function body.  */
    3630     15583612 :   if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p)
    3631      6992316 :       && (optimize
    3632        26275 :           || (TREE_READONLY (p)
    3633        12584 :               && is_gimple_min_invariant (rhs)))
    3634      6969677 :       && (TREE_CODE (rhs) == SSA_NAME
    3635      2610558 :           || is_gimple_min_invariant (rhs))
    3636      6969395 :       && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
    3637              :     {
    3638      6969395 :       insert_decl_map (id, def, rhs);
    3639      6969395 :       return insert_init_debug_bind (id, bb, var, rhs, NULL);
    3640              :     }
    3641              : 
    3642              :   /* If the value of argument is never used, don't care about initializing
    3643              :      it.  */
    3644      1618702 :   if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p))
    3645              :     {
    3646              :       /* When there's a gross type mismatch between the passed value
    3647              :          and the declared argument type drop it on the floor and do
    3648              :          not bother to insert a debug bind.  */
    3649       355772 :       if (value && !is_gimple_reg_type (TREE_TYPE (value)))
    3650              :         return NULL;
    3651       355758 :       return insert_init_debug_bind (id, bb, var, rhs, NULL);
    3652              :     }
    3653              : 
    3654              :   /* Initialize this VAR_DECL from the equivalent argument.  Convert
    3655              :      the argument to the proper type in case it was promoted.  */
    3656       466639 :   if (value)
    3657              :     {
    3658       466544 :       if (rhs == error_mark_node)
    3659              :         {
    3660            0 :           insert_decl_map (id, p, var);
    3661            0 :           return insert_init_debug_bind (id, bb, var, rhs, NULL);
    3662              :         }
    3663              : 
    3664       466544 :       STRIP_USELESS_TYPE_CONVERSION (rhs);
    3665              : 
    3666              :       /* If we are in SSA form properly remap the default definition.  */
    3667       933088 :       if (gimple_in_ssa_p (cfun) && is_gimple_reg (p))
    3668              :         {
    3669        25547 :           if (def)
    3670              :             {
    3671        22921 :               def = remap_ssa_name (def, id);
    3672        22921 :               init_stmt = gimple_build_assign (def, rhs);
    3673        22921 :               SSA_NAME_IS_DEFAULT_DEF (def) = 0;
    3674        22921 :               set_ssa_default_def (cfun, var, NULL);
    3675              :             }
    3676              :         }
    3677       440997 :       else if (!is_empty_type (TREE_TYPE (var)))
    3678       368178 :         init_stmt = gimple_build_assign (var, rhs);
    3679              : 
    3680       466544 :       if (bb && init_stmt)
    3681       390800 :         insert_init_stmt (id, bb, init_stmt);
    3682              :     }
    3683              :   return init_stmt;
    3684              : }
    3685              : 
    3686              : /* Generate code to initialize the parameters of the function at the
    3687              :    top of the stack in ID from the GIMPLE_CALL STMT.  */
    3688              : 
    3689              : static void
    3690      4646466 : initialize_inlined_parameters (copy_body_data *id, gimple *stmt,
    3691              :                                tree fn, basic_block bb)
    3692              : {
    3693      4646466 :   tree parms;
    3694      4646466 :   size_t i;
    3695      4646466 :   tree p;
    3696      4646466 :   tree vars = NULL_TREE;
    3697      4646466 :   tree static_chain = gimple_call_chain (stmt);
    3698              : 
    3699              :   /* Figure out what the parameters are.  */
    3700      4646466 :   parms = DECL_ARGUMENTS (fn);
    3701              : 
    3702              :   /* Loop through the parameter declarations, replacing each with an
    3703              :      equivalent VAR_DECL, appropriately initialized.  */
    3704     12579349 :   for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
    3705              :     {
    3706      7932883 :       tree val;
    3707      7932883 :       val = i < gimple_call_num_args (stmt) ? gimple_call_arg (stmt, i) : NULL;
    3708      7932883 :       setup_one_parameter (id, p, val, fn, bb, &vars);
    3709              :     }
    3710              :   /* After remapping parameters remap their types.  This has to be done
    3711              :      in a second loop over all parameters to appropriately remap
    3712              :      variable sized arrays when the size is specified in a
    3713              :      parameter following the array.  */
    3714     12579349 :   for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
    3715              :     {
    3716      7932883 :       tree *varp = id->decl_map->get (p);
    3717      7932883 :       if (varp && VAR_P (*varp))
    3718              :         {
    3719     15532998 :           tree def = (gimple_in_ssa_p (cfun) && is_gimple_reg (p)
    3720      7325385 :                       ? ssa_default_def (id->src_cfun, p) : NULL);
    3721      7766499 :           tree var = *varp;
    3722      7766499 :           TREE_TYPE (var) = remap_type (TREE_TYPE (var), id);
    3723              :           /* Also remap the default definition if it was remapped
    3724              :              to the default definition of the parameter replacement
    3725              :              by the parameter setup.  */
    3726      7766499 :           if (def)
    3727              :             {
    3728      6968643 :               tree *defp = id->decl_map->get (def);
    3729      6968643 :               if (defp
    3730      6968549 :                   && TREE_CODE (*defp) == SSA_NAME
    3731     13677590 :                   && SSA_NAME_VAR (*defp) == var)
    3732        22921 :                 TREE_TYPE (*defp) = TREE_TYPE (var);
    3733              :             }
    3734              :           /* When not optimizing and the parameter is unused, assign to
    3735              :              a dummy SSA name.  Do this after remapping the type above.  */
    3736       797856 :           else if (!optimize
    3737         3481 :                    && is_gimple_reg (p)
    3738       800482 :                    && i < gimple_call_num_args (stmt))
    3739              :             {
    3740         2626 :               tree val = gimple_call_arg (stmt, i);
    3741         2626 :               if (val != error_mark_node)
    3742              :                 {
    3743         2626 :                   if (!useless_type_conversion_p (TREE_TYPE (p),
    3744         2626 :                                                   TREE_TYPE (val)))
    3745            0 :                     val = force_value_to_type (TREE_TYPE (p), val);
    3746         2626 :                   def = make_ssa_name (var);
    3747         2626 :                   gimple *init_stmt = gimple_build_assign (def, val);
    3748         2626 :                   insert_init_stmt (id, bb, init_stmt);
    3749              :                 }
    3750              :             }
    3751              :         }
    3752              :     }
    3753              : 
    3754              :   /* Initialize the static chain.  */
    3755      4646466 :   p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl;
    3756      4646466 :   gcc_assert (fn != current_function_decl);
    3757      4646466 :   if (p)
    3758              :     {
    3759              :       /* Calling a function that requires a static chain directly
    3760              :          without setting up the static chain has undefined behavior
    3761              :          at run-time.  */
    3762         2340 :       if (!static_chain)
    3763            0 :         static_chain = null_pointer_node;
    3764         2340 :       setup_one_parameter (id, p, static_chain, fn, bb, &vars);
    3765              :     }
    3766              : 
    3767              :   /* Reverse so the variables appear in the correct order in DWARF
    3768              :      debug info.  */
    3769      4646466 :   vars = nreverse (vars);
    3770              : 
    3771      4646466 :   declare_inline_vars (id->block, vars);
    3772      4646466 : }
    3773              : 
    3774              : 
    3775              : /* Declare a return variable to replace the RESULT_DECL for the
    3776              :    function we are calling.  An appropriate DECL_STMT is returned.
    3777              :    The USE_STMT is filled to contain a use of the declaration to
    3778              :    indicate the return value of the function.
    3779              : 
    3780              :    RETURN_SLOT, if non-null is place where to store the result.  It
    3781              :    is set only for CALL_EXPR_RETURN_SLOT_OPT.  MODIFY_DEST, if non-null,
    3782              :    was the LHS of the MODIFY_EXPR to which this call is the RHS.
    3783              : 
    3784              :    The return value is a (possibly null) value that holds the result
    3785              :    as seen by the caller.  */
    3786              : 
    3787              : static tree
    3788      4646466 : declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest,
    3789              :                          basic_block entry_bb)
    3790              : {
    3791      4646466 :   tree callee = id->src_fn;
    3792      4646466 :   tree result = DECL_RESULT (callee);
    3793      4646466 :   tree callee_type = TREE_TYPE (result);
    3794      4646466 :   tree caller_type;
    3795      4646466 :   tree var, use;
    3796              : 
    3797              :   /* Handle type-mismatches in the function declaration return type
    3798              :      vs. the call expression.  */
    3799      4646466 :   if (modify_dest)
    3800      2176285 :     caller_type = TREE_TYPE (modify_dest);
    3801      2470181 :   else if (return_slot)
    3802       143027 :     caller_type = TREE_TYPE (return_slot);
    3803              :   else /* No LHS on the call.  */
    3804      2327154 :     caller_type = TREE_TYPE (TREE_TYPE (callee));
    3805              : 
    3806              :   /* We don't need to do anything for functions that don't return anything.  */
    3807      4646466 :   if (VOID_TYPE_P (callee_type))
    3808              :     return NULL_TREE;
    3809              : 
    3810              :   /* If there was a return slot, then the return value is the
    3811              :      dereferenced address of that object.  */
    3812      2489501 :   if (return_slot)
    3813              :     {
    3814              :       /* The front end shouldn't have used both return_slot and
    3815              :          a modify expression.  */
    3816       143027 :       gcc_assert (!modify_dest);
    3817       143027 :       if (DECL_BY_REFERENCE (result))
    3818              :         {
    3819        83140 :           tree return_slot_addr = build_fold_addr_expr (return_slot);
    3820        83140 :           STRIP_USELESS_TYPE_CONVERSION (return_slot_addr);
    3821              : 
    3822              :           /* We are going to construct *&return_slot and we can't do that
    3823              :              for variables believed to be not addressable.
    3824              : 
    3825              :              FIXME: This check possibly can match, because values returned
    3826              :              via return slot optimization are not believed to have address
    3827              :              taken by alias analysis.  */
    3828        83140 :           gcc_assert (TREE_CODE (return_slot) != SSA_NAME);
    3829        83140 :           var = return_slot_addr;
    3830        83140 :           mark_addressable (return_slot);
    3831              :         }
    3832              :       else
    3833              :         {
    3834        59887 :           var = return_slot;
    3835        59887 :           gcc_assert (TREE_CODE (var) != SSA_NAME);
    3836        59887 :           if (TREE_ADDRESSABLE (result))
    3837        23338 :             mark_addressable (var);
    3838              :         }
    3839       143027 :       if (DECL_NOT_GIMPLE_REG_P (result)
    3840       143027 :           && DECL_P (var))
    3841            0 :         DECL_NOT_GIMPLE_REG_P (var) = 1;
    3842              : 
    3843       143027 :       if (!useless_type_conversion_p (callee_type, caller_type))
    3844        83141 :         var = build1 (VIEW_CONVERT_EXPR, callee_type, var);
    3845              : 
    3846       143027 :       use = NULL;
    3847       143027 :       goto done;
    3848              :     }
    3849              : 
    3850              :   /* All types requiring non-trivial constructors should have been handled.  */
    3851      2346474 :   gcc_assert (!TREE_ADDRESSABLE (callee_type));
    3852              : 
    3853              :   /* Attempt to avoid creating a new temporary variable.  */
    3854      2346474 :   if (modify_dest
    3855      2176264 :       && TREE_CODE (modify_dest) != SSA_NAME)
    3856              :     {
    3857       387060 :       bool use_it = false;
    3858              : 
    3859              :       /* We can't use MODIFY_DEST if there's type promotion involved.  */
    3860       387060 :       if (!useless_type_conversion_p (callee_type, caller_type))
    3861              :         use_it = false;
    3862              : 
    3863              :       /* ??? If we're assigning to a variable sized type, then we must
    3864              :          reuse the destination variable, because we've no good way to
    3865              :          create variable sized temporaries at this point.  */
    3866       387058 :       else if (!poly_int_tree_p (TYPE_SIZE_UNIT (caller_type)))
    3867              :         use_it = true;
    3868              : 
    3869              :       /* If the callee cannot possibly modify MODIFY_DEST, then we can
    3870              :          reuse it as the result of the call directly.  Don't do this if
    3871              :          it would promote MODIFY_DEST to addressable.  */
    3872       387058 :       else if (TREE_ADDRESSABLE (result))
    3873              :         use_it = false;
    3874              :       else
    3875              :         {
    3876       385732 :           tree base_m = get_base_address (modify_dest);
    3877              : 
    3878              :           /* If the base isn't a decl, then it's a pointer, and we don't
    3879              :              know where that's going to go.  */
    3880       385732 :           if (!DECL_P (base_m))
    3881              :             use_it = false;
    3882       381710 :           else if (is_global_var (base_m))
    3883              :             use_it = false;
    3884       381255 :           else if (DECL_NOT_GIMPLE_REG_P (result)
    3885       381255 :                    && !DECL_NOT_GIMPLE_REG_P (base_m))
    3886              :             use_it = false;
    3887       381255 :           else if (!TREE_ADDRESSABLE (base_m))
    3888              :             use_it = true;
    3889              :         }
    3890              : 
    3891              :       if (use_it)
    3892              :         {
    3893       279730 :           var = modify_dest;
    3894       279730 :           use = NULL;
    3895       279730 :           goto done;
    3896              :         }
    3897              :     }
    3898              : 
    3899      2066744 :   gcc_assert (poly_int_tree_p (TYPE_SIZE_UNIT (callee_type)));
    3900              : 
    3901      2066744 :   var = copy_result_decl_to_var (result, id);
    3902      2066744 :   DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
    3903              : 
    3904              :   /* Do not have the rest of GCC warn about this variable as it should
    3905              :      not be visible to the user.  */
    3906      2066744 :   suppress_warning (var /* OPT_Wuninitialized? */);
    3907              : 
    3908      2066744 :   declare_inline_vars (id->block, var);
    3909              : 
    3910              :   /* Build the use expr.  If the return type of the function was
    3911              :      promoted, convert it back to the expected type.  */
    3912      2066744 :   use = var;
    3913      2066744 :   if (!useless_type_conversion_p (caller_type, TREE_TYPE (var)))
    3914              :     {
    3915              :       /* If we can match up types by promotion/demotion do so.  */
    3916            7 :       if (fold_convertible_p (caller_type, var))
    3917            0 :         use = fold_convert (caller_type, var);
    3918              :       else
    3919              :         {
    3920              :           /* ???  For valid programs we should not end up here.
    3921              :              Still if we end up with truly mismatched types here, fall back
    3922              :              to using a MEM_REF to not leak invalid GIMPLE to the following
    3923              :              passes.  */
    3924              :           /* Prevent var from being written into SSA form.  */
    3925            7 :           if (is_gimple_reg_type (TREE_TYPE (var)))
    3926            7 :             DECL_NOT_GIMPLE_REG_P (var) = true;
    3927            7 :           use = fold_build2 (MEM_REF, caller_type,
    3928              :                              build_fold_addr_expr (var),
    3929              :                              build_int_cst (ptr_type_node, 0));
    3930              :         }
    3931              :     }
    3932              : 
    3933      2066744 :   STRIP_USELESS_TYPE_CONVERSION (use);
    3934              : 
    3935      2066744 :   if (DECL_BY_REFERENCE (result))
    3936              :     {
    3937            0 :       TREE_ADDRESSABLE (var) = 1;
    3938            0 :       var = build_fold_addr_expr (var);
    3939              :     }
    3940              : 
    3941      2066744 :  done:
    3942              :   /* Register the VAR_DECL as the equivalent for the RESULT_DECL; that
    3943              :      way, when the RESULT_DECL is encountered, it will be
    3944              :      automatically replaced by the VAR_DECL.
    3945              : 
    3946              :      When returning by reference, ensure that RESULT_DECL remaps to
    3947              :      gimple_val.  */
    3948      2489501 :   if (DECL_BY_REFERENCE (result)
    3949      2489501 :       && !is_gimple_val (var))
    3950              :     {
    3951        83140 :       tree temp = create_tmp_var (TREE_TYPE (result), "retvalptr");
    3952        83140 :       insert_decl_map (id, result, temp);
    3953              :       /* When RESULT_DECL is in SSA form, we need to remap and initialize
    3954              :          it's default_def SSA_NAME.  */
    3955        83140 :       if (gimple_in_ssa_p (id->src_cfun)
    3956        83140 :           && is_gimple_reg (result))
    3957        83140 :         if (tree default_def = ssa_default_def (id->src_cfun, result))
    3958              :           {
    3959        83103 :             temp = make_ssa_name (temp);
    3960        83103 :             insert_decl_map (id, default_def, temp);
    3961              :           }
    3962        83140 :       insert_init_stmt (id, entry_bb, gimple_build_assign (temp, var));
    3963              :     }
    3964              :   else
    3965      2406361 :     insert_decl_map (id, result, var);
    3966              : 
    3967              :   /* Remember this so we can ignore it in remap_decls.  */
    3968      2489501 :   id->retvar = var;
    3969      2489501 :   return use;
    3970              : }
    3971              : 
    3972              : /* Determine if the function can be copied.  If so return NULL.  If
    3973              :    not return a string describing the reason for failure.  */
    3974              : 
    3975              : const char *
    3976     18567568 : copy_forbidden (struct function *fun)
    3977              : {
    3978     18567568 :   const char *reason = fun->cannot_be_copied_reason;
    3979              : 
    3980              :   /* Only examine the function once.  */
    3981     18567568 :   if (fun->cannot_be_copied_set)
    3982              :     return reason;
    3983              : 
    3984              :   /* We cannot copy a function that receives a non-local goto
    3985              :      because we cannot remap the destination label used in the
    3986              :      function that is performing the non-local goto.  */
    3987              :   /* ??? Actually, this should be possible, if we work at it.
    3988              :      No doubt there's just a handful of places that simply
    3989              :      assume it doesn't happen and don't substitute properly.  */
    3990     11079116 :   if (fun->has_nonlocal_label)
    3991              :     {
    3992          782 :       reason = G_("function %q+F can never be copied "
    3993              :                   "because it receives a non-local goto");
    3994          782 :       goto fail;
    3995              :     }
    3996              : 
    3997     11078334 :   if (fun->has_forced_label_in_static)
    3998              :     {
    3999          230 :       reason = G_("function %q+F can never be copied because it saves "
    4000              :                   "address of local label in a static variable");
    4001          230 :       goto fail;
    4002              :     }
    4003              : 
    4004     11078104 :  fail:
    4005     11079116 :   fun->cannot_be_copied_reason = reason;
    4006     11079116 :   fun->cannot_be_copied_set = true;
    4007     11079116 :   return reason;
    4008              : }
    4009              : 
    4010              : 
    4011              : static const char *inline_forbidden_reason;
    4012              : 
    4013              : /* A callback for walk_gimple_seq to handle statements.  Returns non-null
    4014              :    iff a function cannot be inlined.  Also sets the reason why. */
    4015              : 
    4016              : static tree
    4017    177993119 : inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
    4018              :                          struct walk_stmt_info *wip)
    4019              : {
    4020    177993119 :   tree fn = (tree) wip->info;
    4021    177993119 :   tree t;
    4022    177993119 :   gimple *stmt = gsi_stmt (*gsi);
    4023              : 
    4024    177993119 :   switch (gimple_code (stmt))
    4025              :     {
    4026     19446322 :     case GIMPLE_CALL:
    4027              :       /* Refuse to inline alloca call unless user explicitly forced so as
    4028              :          this may change program's memory overhead drastically when the
    4029              :          function using alloca is called in loop.  In GCC present in
    4030              :          SPEC2000 inlining into schedule_block cause it to require 2GB of
    4031              :          RAM instead of 256MB.  Don't do so for alloca calls emitted for
    4032              :          VLA objects as those can't cause unbounded growth (they're always
    4033              :          wrapped inside stack_save/stack_restore regions.  */
    4034     19446322 :       if (gimple_maybe_alloca_call_p (stmt)
    4035         9558 :           && !gimple_call_alloca_for_var_p (as_a <gcall *> (stmt))
    4036     19451095 :           && !lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn)))
    4037              :         {
    4038         4645 :           inline_forbidden_reason
    4039         4645 :             = G_("function %q+F can never be inlined because it uses "
    4040              :                  "alloca (override using the always_inline attribute)");
    4041         4645 :           *handled_ops_p = true;
    4042         4645 :           return fn;
    4043              :         }
    4044              : 
    4045     19441677 :       t = gimple_call_fndecl (stmt);
    4046     19441677 :       if (t == NULL_TREE)
    4047              :         break;
    4048              : 
    4049              :       /* We cannot inline functions that call setjmp.  */
    4050     18713495 :       if (setjmp_call_p (t))
    4051              :         {
    4052         1045 :           inline_forbidden_reason
    4053         1045 :             = G_("function %q+F can never be inlined because it uses setjmp");
    4054         1045 :           *handled_ops_p = true;
    4055         1045 :           return t;
    4056              :         }
    4057              : 
    4058     18712450 :       if (DECL_BUILT_IN_CLASS (t) == BUILT_IN_NORMAL)
    4059      3845638 :         switch (DECL_FUNCTION_CODE (t))
    4060              :           {
    4061              :             /* We cannot inline functions that take a variable number of
    4062              :                arguments.  */
    4063         2814 :           case BUILT_IN_VA_START:
    4064         2814 :           case BUILT_IN_NEXT_ARG:
    4065         2814 :           case BUILT_IN_VA_END:
    4066         2814 :             inline_forbidden_reason
    4067         2814 :               = G_("function %q+F can never be inlined because it "
    4068              :                    "uses variable argument lists");
    4069         2814 :             *handled_ops_p = true;
    4070         2814 :             return t;
    4071              : 
    4072          226 :           case BUILT_IN_LONGJMP:
    4073              :             /* We can't inline functions that call __builtin_longjmp at
    4074              :                all.  The non-local goto machinery really requires the
    4075              :                destination be in a different function.  If we allow the
    4076              :                function calling __builtin_longjmp to be inlined into the
    4077              :                function calling __builtin_setjmp, Things will Go Awry.  */
    4078          226 :             inline_forbidden_reason
    4079          226 :               = G_("function %q+F can never be inlined because "
    4080              :                    "it uses setjmp-longjmp exception handling");
    4081          226 :             *handled_ops_p = true;
    4082          226 :             return t;
    4083              : 
    4084          276 :           case BUILT_IN_NONLOCAL_GOTO:
    4085              :             /* Similarly.  */
    4086          276 :             inline_forbidden_reason
    4087          276 :               = G_("function %q+F can never be inlined because "
    4088              :                    "it uses non-local goto");
    4089          276 :             *handled_ops_p = true;
    4090          276 :             return t;
    4091              : 
    4092          384 :           case BUILT_IN_RETURN:
    4093          384 :           case BUILT_IN_APPLY_ARGS:
    4094              :             /* If a __builtin_apply_args caller would be inlined,
    4095              :                it would be saving arguments of the function it has
    4096              :                been inlined into.  Similarly __builtin_return would
    4097              :                return from the function the inline has been inlined into.  */
    4098          384 :             inline_forbidden_reason
    4099          384 :               = G_("function %q+F can never be inlined because "
    4100              :                    "it uses %<__builtin_return%> or %<__builtin_apply_args%>");
    4101          384 :             *handled_ops_p = true;
    4102          384 :             return t;
    4103              : 
    4104              :           default:
    4105              :             break;
    4106              :           }
    4107              :       break;
    4108              : 
    4109          313 :     case GIMPLE_GOTO:
    4110          313 :       t = gimple_goto_dest (stmt);
    4111              : 
    4112              :       /* We will not inline a function which uses computed goto.  The
    4113              :          addresses of its local labels, which may be tucked into
    4114              :          global storage, are of course not constant across
    4115              :          instantiations, which causes unexpected behavior.  */
    4116          313 :       if (TREE_CODE (t) != LABEL_DECL)
    4117              :         {
    4118          313 :           inline_forbidden_reason
    4119          313 :             = G_("function %q+F can never be inlined "
    4120              :                  "because it contains a computed goto");
    4121          313 :           *handled_ops_p = true;
    4122          313 :           return t;
    4123              :         }
    4124              :       break;
    4125              : 
    4126              :     default:
    4127              :       break;
    4128              :     }
    4129              : 
    4130    177983416 :   *handled_ops_p = false;
    4131    177983416 :   return NULL_TREE;
    4132              : }
    4133              : 
    4134              : /* Return true if FNDECL is a function that cannot be inlined into
    4135              :    another one.  */
    4136              : 
    4137              : static bool
    4138      5959263 : inline_forbidden_p (tree fndecl)
    4139              : {
    4140      5959263 :   struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
    4141      5959263 :   struct walk_stmt_info wi;
    4142      5959263 :   basic_block bb;
    4143      5959263 :   bool forbidden_p = false;
    4144              : 
    4145              :   /* First check for shared reasons not to copy the code.  */
    4146      5959263 :   inline_forbidden_reason = copy_forbidden (fun);
    4147      5959263 :   if (inline_forbidden_reason != NULL)
    4148              :     return true;
    4149              : 
    4150              :   /* Next, walk the statements of the function looking for
    4151              :      constructs we can't handle, or are non-optimal for inlining.  */
    4152      5958558 :   hash_set<tree> visited_nodes;
    4153      5958558 :   memset (&wi, 0, sizeof (wi));
    4154      5958558 :   wi.info = (void *) fndecl;
    4155      5958558 :   wi.pset = &visited_nodes;
    4156              : 
    4157              :   /* We cannot inline a function with a variable-sized parameter because we
    4158              :      cannot materialize a temporary of such a type in the caller if need be.
    4159              :      Note that the return case is not symmetrical because we can guarantee
    4160              :      that a temporary is not needed by means of CALL_EXPR_RETURN_SLOT_OPT.  */
    4161     18926757 :   for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
    4162     12968201 :     if (!poly_int_tree_p (DECL_SIZE (parm)))
    4163              :       {
    4164            2 :         inline_forbidden_reason
    4165            2 :           = G_("function %q+F can never be inlined because "
    4166              :                "it has a VLA argument");
    4167            2 :         return true;
    4168              :       }
    4169              : 
    4170     38668944 :   FOR_EACH_BB_FN (bb, fun)
    4171              :     {
    4172     32720091 :       gimple *ret;
    4173     32720091 :       gimple_seq seq = bb_seq (bb);
    4174     32720091 :       ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, NULL, &wi);
    4175     32720091 :       forbidden_p = (ret != NULL);
    4176     32720091 :       if (forbidden_p)
    4177              :         break;
    4178              :     }
    4179              : 
    4180              :   return forbidden_p;
    4181      5958558 : }
    4182              : 
    4183              : /* Return false if the function FNDECL cannot be inlined on account of its
    4184              :    attributes, true otherwise.  */
    4185              : static bool
    4186      5959263 : function_attribute_inlinable_p (const_tree fndecl)
    4187              : {
    4188     11893499 :   for (auto scoped_attributes : targetm.attribute_table)
    4189              :     {
    4190      5959263 :       const_tree a;
    4191              : 
    4192      7851390 :       for (a = DECL_ATTRIBUTES (fndecl); a; a = TREE_CHAIN (a))
    4193              :         {
    4194      1917154 :           const_tree name = get_attribute_name (a);
    4195              : 
    4196     54973484 :           for (const attribute_spec &attribute : scoped_attributes->attributes)
    4197     53081357 :             if (is_attribute_p (attribute.name, name))
    4198        25027 :               return targetm.function_attribute_inlinable_p (fndecl);
    4199              :         }
    4200              :     }
    4201              : 
    4202              :   return true;
    4203              : }
    4204              : 
    4205              : /* Returns nonzero if FN is a function that does not have any
    4206              :    fundamental inline blocking properties.  */
    4207              : 
    4208              : bool
    4209      6588884 : tree_inlinable_function_p (tree fn)
    4210              : {
    4211      6588884 :   bool inlinable = true;
    4212      6588884 :   bool do_warning;
    4213      6588884 :   tree always_inline;
    4214              : 
    4215              :   /* If we've already decided this function shouldn't be inlined,
    4216              :      there's no need to check again.  */
    4217      6588884 :   if (DECL_UNINLINABLE (fn))
    4218              :     return false;
    4219              : 
    4220              :   /* We only warn for functions declared `inline' by the user.  */
    4221      5979781 :   do_warning = (opt_for_fn (fn, warn_inline)
    4222          637 :                 && DECL_DECLARED_INLINE_P (fn)
    4223          508 :                 && !DECL_NO_INLINE_WARNING_P (fn)
    4224      5980159 :                 && !DECL_IN_SYSTEM_HEADER (fn));
    4225              : 
    4226      5979781 :   always_inline = lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn));
    4227              : 
    4228      5979781 :   if (flag_no_inline
    4229        97128 :       && always_inline == NULL)
    4230              :     {
    4231        20518 :       if (do_warning)
    4232            0 :         warning (OPT_Winline, "function %q+F can never be inlined because it "
    4233              :                  "is suppressed using %<-fno-inline%>", fn);
    4234              :       inlinable = false;
    4235              :     }
    4236              : 
    4237      5959263 :   else if (!function_attribute_inlinable_p (fn))
    4238              :     {
    4239            0 :       if (do_warning)
    4240            0 :         warning (OPT_Winline, "function %q+F can never be inlined because it "
    4241              :                  "uses attributes conflicting with inlining", fn);
    4242              :       inlinable = false;
    4243              :     }
    4244              : 
    4245      5959263 :   else if (inline_forbidden_p (fn))
    4246              :     {
    4247              :       /* See if we should warn about uninlinable functions.  Previously,
    4248              :          some of these warnings would be issued while trying to expand
    4249              :          the function inline, but that would cause multiple warnings
    4250              :          about functions that would for example call alloca.  But since
    4251              :          this a property of the function, just one warning is enough.
    4252              :          As a bonus we can now give more details about the reason why a
    4253              :          function is not inlinable.  */
    4254        10410 :       if (always_inline)
    4255            2 :         error (inline_forbidden_reason, fn);
    4256        10408 :       else if (do_warning)
    4257            2 :         warning (OPT_Winline, inline_forbidden_reason, fn);
    4258              : 
    4259              :       inlinable = false;
    4260              :     }
    4261              : 
    4262              :   /* Squirrel away the result so that we don't have to check again.  */
    4263      5979781 :   DECL_UNINLINABLE (fn) = !inlinable;
    4264              : 
    4265      5979781 :   return inlinable;
    4266              : }
    4267              : 
    4268              : /* Estimate the cost of a memory move of type TYPE.  Use machine dependent
    4269              :    word size and take possible memcpy call into account and return
    4270              :    cost based on whether optimizing for size or speed according to SPEED_P.  */
    4271              : 
    4272              : int
    4273    337293683 : estimate_move_cost (tree type, bool ARG_UNUSED (speed_p))
    4274              : {
    4275    337293683 :   HOST_WIDE_INT size;
    4276              : 
    4277    337293683 :   gcc_assert (!VOID_TYPE_P (type));
    4278              : 
    4279    337293683 :   if (VECTOR_TYPE_P (type))
    4280              :     {
    4281      4343862 :       scalar_mode inner = SCALAR_TYPE_MODE (TREE_TYPE (type));
    4282      4343862 :       machine_mode simd = targetm.vectorize.preferred_simd_mode (inner);
    4283      4343862 :       int orig_mode_size
    4284      8687724 :         = estimated_poly_value (GET_MODE_SIZE (TYPE_MODE (type)));
    4285      8687724 :       int simd_mode_size = estimated_poly_value (GET_MODE_SIZE (simd));
    4286      4343862 :       return ((orig_mode_size + simd_mode_size - 1)
    4287      4343862 :               / simd_mode_size);
    4288              :     }
    4289              : 
    4290    332949821 :   size = int_size_in_bytes (type);
    4291              : 
    4292    332952286 :   if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (speed_p))
    4293              :     /* Cost of a memcpy call, 3 arguments and the call.  */
    4294              :     return 4;
    4295              :   else
    4296    331996988 :     return ((size + MOVE_MAX_PIECES - 1) / MOVE_MAX_PIECES);
    4297              : }
    4298              : 
    4299              : /* Returns cost of operation CODE, according to WEIGHTS  */
    4300              : 
    4301              : static int
    4302    405112190 : estimate_operator_cost (enum tree_code code, eni_weights *weights,
    4303              :                         tree op1 ATTRIBUTE_UNUSED, tree op2)
    4304              : {
    4305    405112190 :   switch (code)
    4306              :     {
    4307              :     /* These are "free" conversions, or their presumed cost
    4308              :        is folded into other operations.  */
    4309              :     case RANGE_EXPR:
    4310              :     CASE_CONVERT:
    4311              :     case COMPLEX_EXPR:
    4312              :     case PAREN_EXPR:
    4313              :     case VIEW_CONVERT_EXPR:
    4314              :       return 0;
    4315              : 
    4316              :     /* Assign cost of 1 to usual operations.
    4317              :        ??? We may consider mapping RTL costs to this.  */
    4318              :     case COND_EXPR:
    4319              :     case VEC_COND_EXPR:
    4320              :     case VEC_PERM_EXPR:
    4321              : 
    4322              :     case PLUS_EXPR:
    4323              :     case POINTER_PLUS_EXPR:
    4324              :     case POINTER_DIFF_EXPR:
    4325              :     case MINUS_EXPR:
    4326              :     case MULT_EXPR:
    4327              :     case MULT_HIGHPART_EXPR:
    4328              : 
    4329              :     case ADDR_SPACE_CONVERT_EXPR:
    4330              :     case FIXED_CONVERT_EXPR:
    4331              :     case FIX_TRUNC_EXPR:
    4332              : 
    4333              :     case NEGATE_EXPR:
    4334              :     case FLOAT_EXPR:
    4335              :     case MIN_EXPR:
    4336              :     case MAX_EXPR:
    4337              :     case ABS_EXPR:
    4338              :     case ABSU_EXPR:
    4339              : 
    4340              :     case LSHIFT_EXPR:
    4341              :     case RSHIFT_EXPR:
    4342              :     case LROTATE_EXPR:
    4343              :     case RROTATE_EXPR:
    4344              : 
    4345              :     case BIT_IOR_EXPR:
    4346              :     case BIT_XOR_EXPR:
    4347              :     case BIT_AND_EXPR:
    4348              :     case BIT_NOT_EXPR:
    4349              : 
    4350              :     case TRUTH_ANDIF_EXPR:
    4351              :     case TRUTH_ORIF_EXPR:
    4352              :     case TRUTH_AND_EXPR:
    4353              :     case TRUTH_OR_EXPR:
    4354              :     case TRUTH_XOR_EXPR:
    4355              :     case TRUTH_NOT_EXPR:
    4356              : 
    4357              :     case LT_EXPR:
    4358              :     case LE_EXPR:
    4359              :     case GT_EXPR:
    4360              :     case GE_EXPR:
    4361              :     case EQ_EXPR:
    4362              :     case NE_EXPR:
    4363              :     case ORDERED_EXPR:
    4364              :     case UNORDERED_EXPR:
    4365              : 
    4366              :     case UNLT_EXPR:
    4367              :     case UNLE_EXPR:
    4368              :     case UNGT_EXPR:
    4369              :     case UNGE_EXPR:
    4370              :     case UNEQ_EXPR:
    4371              :     case LTGT_EXPR:
    4372              : 
    4373              :     case CONJ_EXPR:
    4374              : 
    4375              :     case PREDECREMENT_EXPR:
    4376              :     case PREINCREMENT_EXPR:
    4377              :     case POSTDECREMENT_EXPR:
    4378              :     case POSTINCREMENT_EXPR:
    4379              : 
    4380              :     case REALIGN_LOAD_EXPR:
    4381              : 
    4382              :     case WIDEN_SUM_EXPR:
    4383              :     case WIDEN_MULT_EXPR:
    4384              :     case DOT_PROD_EXPR:
    4385              :     case SAD_EXPR:
    4386              :     case WIDEN_MULT_PLUS_EXPR:
    4387              :     case WIDEN_MULT_MINUS_EXPR:
    4388              :     case WIDEN_LSHIFT_EXPR:
    4389              : 
    4390              :     case VEC_WIDEN_MULT_HI_EXPR:
    4391              :     case VEC_WIDEN_MULT_LO_EXPR:
    4392              :     case VEC_WIDEN_MULT_EVEN_EXPR:
    4393              :     case VEC_WIDEN_MULT_ODD_EXPR:
    4394              :     case VEC_UNPACK_HI_EXPR:
    4395              :     case VEC_UNPACK_LO_EXPR:
    4396              :     case VEC_UNPACK_FLOAT_HI_EXPR:
    4397              :     case VEC_UNPACK_FLOAT_LO_EXPR:
    4398              :     case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
    4399              :     case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
    4400              :     case VEC_PACK_TRUNC_EXPR:
    4401              :     case VEC_PACK_SAT_EXPR:
    4402              :     case VEC_PACK_FIX_TRUNC_EXPR:
    4403              :     case VEC_PACK_FLOAT_EXPR:
    4404              :     case VEC_WIDEN_LSHIFT_HI_EXPR:
    4405              :     case VEC_WIDEN_LSHIFT_LO_EXPR:
    4406              :     case VEC_DUPLICATE_EXPR:
    4407              :     case VEC_SERIES_EXPR:
    4408              : 
    4409              :       return 1;
    4410              : 
    4411              :     /* Few special cases of expensive operations.  This is useful
    4412              :        to avoid inlining on functions having too many of these.  */
    4413      2687246 :     case TRUNC_DIV_EXPR:
    4414      2687246 :     case CEIL_DIV_EXPR:
    4415      2687246 :     case FLOOR_DIV_EXPR:
    4416      2687246 :     case ROUND_DIV_EXPR:
    4417      2687246 :     case EXACT_DIV_EXPR:
    4418      2687246 :     case TRUNC_MOD_EXPR:
    4419      2687246 :     case CEIL_MOD_EXPR:
    4420      2687246 :     case FLOOR_MOD_EXPR:
    4421      2687246 :     case ROUND_MOD_EXPR:
    4422      2687246 :     case RDIV_EXPR:
    4423      2687246 :       if (TREE_CODE (op2) != INTEGER_CST)
    4424      1344213 :         return weights->div_mod_cost;
    4425              :       return 1;
    4426              : 
    4427              :     /* Bit-field insertion needs several shift and mask operations.  */
    4428              :     case BIT_INSERT_EXPR:
    4429              :       return 3;
    4430              : 
    4431    182827348 :     default:
    4432              :       /* We expect a copy assignment with no operator.  */
    4433    182827348 :       gcc_assert (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS);
    4434              :       return 0;
    4435              :     }
    4436              : }
    4437              : 
    4438              : 
    4439              : /* Estimate number of instructions that will be created by expanding
    4440              :    the statements in the statement sequence STMTS.
    4441              :    WEIGHTS contains weights attributed to various constructs.  */
    4442              : 
    4443              : int
    4444       232102 : estimate_num_insns_seq (gimple_seq stmts, eni_weights *weights)
    4445              : {
    4446       232102 :   int cost;
    4447       232102 :   gimple_stmt_iterator gsi;
    4448              : 
    4449       232102 :   cost = 0;
    4450       616425 :   for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
    4451       384323 :     cost += estimate_num_insns (gsi_stmt (gsi), weights);
    4452              : 
    4453       232102 :   return cost;
    4454              : }
    4455              : 
    4456              : 
    4457              : /* Estimate number of instructions that will be created by expanding STMT.
    4458              :    WEIGHTS contains weights attributed to various constructs.  */
    4459              : 
    4460              : int
    4461    576322197 : estimate_num_insns (gimple *stmt, eni_weights *weights)
    4462              : {
    4463    576322197 :   unsigned cost, i;
    4464    576322197 :   enum gimple_code code = gimple_code (stmt);
    4465    576322197 :   tree lhs;
    4466    576322197 :   tree rhs;
    4467              : 
    4468    576322197 :   switch (code)
    4469              :     {
    4470    328838360 :     case GIMPLE_ASSIGN:
    4471              :       /* Try to estimate the cost of assignments.  We have three cases to
    4472              :          deal with:
    4473              :          1) Simple assignments to registers;
    4474              :          2) Stores to things that must live in memory.  This includes
    4475              :             "normal" stores to scalars, but also assignments of large
    4476              :             structures, or constructors of big arrays;
    4477              : 
    4478              :          Let us look at the first two cases, assuming we have "a = b + C":
    4479              :          <GIMPLE_ASSIGN <var_decl "a">
    4480              :                 <plus_expr <var_decl "b"> <constant C>>
    4481              :          If "a" is a GIMPLE register, the assignment to it is free on almost
    4482              :          any target, because "a" usually ends up in a real register.  Hence
    4483              :          the only cost of this expression comes from the PLUS_EXPR, and we
    4484              :          can ignore the GIMPLE_ASSIGN.
    4485              :          If "a" is not a GIMPLE register, the assignment to "a" will most
    4486              :          likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost
    4487              :          of moving something into "a", which we compute using the function
    4488              :          estimate_move_cost.  */
    4489    328838360 :       if (gimple_clobber_p (stmt))
    4490              :         return 0;       /* ={v} {CLOBBER} stmt expands to nothing.  */
    4491              : 
    4492    312128463 :       lhs = gimple_assign_lhs (stmt);
    4493    312128463 :       rhs = gimple_assign_rhs1 (stmt);
    4494              : 
    4495    312128463 :       cost = 0;
    4496              : 
    4497              :       /* Account for the cost of moving to / from memory.  */
    4498    312128463 :       if (gimple_store_p (stmt))
    4499     76864340 :         cost += estimate_move_cost (TREE_TYPE (lhs), weights->time_based);
    4500    312128463 :       if (gimple_assign_load_p (stmt))
    4501     85610398 :         cost += estimate_move_cost (TREE_TYPE (rhs), weights->time_based);
    4502              : 
    4503    404601232 :       cost += estimate_operator_cost (gimple_assign_rhs_code (stmt), weights,
    4504              :                                       gimple_assign_rhs1 (stmt),
    4505    312128463 :                                       get_gimple_rhs_class (gimple_assign_rhs_code (stmt))
    4506              :                                       == GIMPLE_BINARY_RHS
    4507     92472769 :                                       ? gimple_assign_rhs2 (stmt) : NULL);
    4508    312128463 :       break;
    4509              : 
    4510     92983727 :     case GIMPLE_COND:
    4511     92983727 :       cost = 1 + estimate_operator_cost (gimple_cond_code (stmt), weights,
    4512              :                                          gimple_op (stmt, 0),
    4513              :                                          gimple_op (stmt, 1));
    4514     92983727 :       break;
    4515              : 
    4516       695540 :     case GIMPLE_SWITCH:
    4517       695540 :       {
    4518       695540 :         gswitch *switch_stmt = as_a <gswitch *> (stmt);
    4519              :         /* Take into account cost of the switch + guess 2 conditional jumps for
    4520              :            each case label.
    4521              : 
    4522              :            TODO: once the switch expansion logic is sufficiently separated, we can
    4523              :            do better job on estimating cost of the switch.  */
    4524       695540 :         if (weights->time_based)
    4525       127672 :           cost = floor_log2 (gimple_switch_num_labels (switch_stmt)) * 2;
    4526              :         else
    4527       567868 :           cost = gimple_switch_num_labels (switch_stmt) * 2;
    4528              :       }
    4529              :       break;
    4530              : 
    4531     73906421 :     case GIMPLE_CALL:
    4532     73906421 :       {
    4533     73906421 :         tree decl;
    4534              : 
    4535     73906421 :         if (gimple_call_internal_p (stmt))
    4536              :           return 0;
    4537     71028648 :         else if ((decl = gimple_call_fndecl (stmt))
    4538     71028648 :                  && fndecl_built_in_p (decl))
    4539              :           {
    4540              :             /* Do not special case builtins where we see the body.
    4541              :                This just confuse inliner.  */
    4542     18129407 :             struct cgraph_node *node;
    4543     18129407 :             if ((node = cgraph_node::get (decl))
    4544     18129407 :                 && node->definition)
    4545              :               ;
    4546              :             /* For builtins that are likely expanded to nothing or
    4547              :                inlined do not account operand costs.  */
    4548     18099359 :             else if (is_simple_builtin (decl))
    4549              :               return 0;
    4550     15215993 :             else if (is_inexpensive_builtin (decl))
    4551      1937593 :               return weights->target_builtin_call_cost;
    4552     13278400 :             else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
    4553              :               {
    4554              :                 /* We canonicalize x * x to pow (x, 2.0) with -ffast-math, so
    4555              :                    specialize the cheap expansion we do here.
    4556              :                    ???  This asks for a more general solution.  */
    4557     13033408 :                 switch (DECL_FUNCTION_CODE (decl))
    4558              :                   {
    4559         9729 :                     case BUILT_IN_POW:
    4560         9729 :                     case BUILT_IN_POWF:
    4561         9729 :                     case BUILT_IN_POWL:
    4562         9729 :                       if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
    4563        13683 :                           && (real_equal
    4564         3954 :                               (&TREE_REAL_CST (gimple_call_arg (stmt, 1)),
    4565              :                                &dconst2)))
    4566          541 :                         return estimate_operator_cost
    4567          541 :                             (MULT_EXPR, weights, gimple_call_arg (stmt, 0),
    4568          541 :                              gimple_call_arg (stmt, 0));
    4569              :                       break;
    4570              : 
    4571              :                     default:
    4572              :                       break;
    4573              :                   }
    4574              :               }
    4575              :           }
    4576              : 
    4577     66207148 :         cost = decl ? weights->call_cost : weights->indirect_call_cost;
    4578     66207148 :         if (gimple_call_lhs (stmt))
    4579     25883411 :           cost += estimate_move_cost (TREE_TYPE (gimple_call_lhs (stmt)),
    4580              :                                       weights->time_based);
    4581    200449995 :         for (i = 0; i < gimple_call_num_args (stmt); i++)
    4582              :           {
    4583    134242847 :             tree arg = gimple_call_arg (stmt, i);
    4584    134242847 :             cost += estimate_move_cost (TREE_TYPE (arg),
    4585              :                                         weights->time_based);
    4586              :           }
    4587              :         break;
    4588              :       }
    4589              : 
    4590     18341305 :     case GIMPLE_RETURN:
    4591     18341305 :       return weights->return_cost;
    4592              : 
    4593              :     case GIMPLE_GOTO:
    4594              :     case GIMPLE_LABEL:
    4595              :     case GIMPLE_NOP:
    4596              :     case GIMPLE_PHI:
    4597              :     case GIMPLE_PREDICT:
    4598              :     case GIMPLE_DEBUG:
    4599              :       return 0;
    4600              : 
    4601       664928 :     case GIMPLE_ASM:
    4602       664928 :       {
    4603       664928 :         int count = asm_str_count (gimple_asm_string (as_a <gasm *> (stmt)));
    4604              :         /* 1000 means infinity. This avoids overflows later
    4605              :            with very long asm statements.  */
    4606       664928 :         if (count > 1000)
    4607              :           count = 1000;
    4608              :         /* If this asm is asm inline, count anything as minimum size.  */
    4609       664928 :         if (gimple_asm_inline_p (as_a <gasm *> (stmt)))
    4610          709 :           count = MIN (1, count);
    4611       664928 :         return MAX (1, count);
    4612              :       }
    4613              : 
    4614              :     case GIMPLE_RESX:
    4615              :       /* This is either going to be an external function call with one
    4616              :          argument, or two register copy statements plus a goto.  */
    4617              :       return 2;
    4618              : 
    4619        18758 :     case GIMPLE_EH_DISPATCH:
    4620              :       /* ??? This is going to turn into a switch statement.  Ideally
    4621              :          we'd have a look at the eh region and estimate the number of
    4622              :          edges involved.  */
    4623        18758 :       return 10;
    4624              : 
    4625            0 :     case GIMPLE_BIND:
    4626            0 :       return estimate_num_insns_seq (
    4627            0 :                gimple_bind_body (as_a <gbind *> (stmt)),
    4628            0 :                weights);
    4629              : 
    4630            0 :     case GIMPLE_EH_FILTER:
    4631            0 :       return estimate_num_insns_seq (gimple_eh_filter_failure (stmt), weights);
    4632              : 
    4633         8292 :     case GIMPLE_CATCH:
    4634         8292 :       return estimate_num_insns_seq (gimple_catch_handler (
    4635         8292 :                                        as_a <gcatch *> (stmt)),
    4636         8292 :                                      weights);
    4637              : 
    4638         8297 :     case GIMPLE_TRY:
    4639         8297 :       return (estimate_num_insns_seq (gimple_try_eval (stmt), weights)
    4640         8297 :               + estimate_num_insns_seq (gimple_try_cleanup (stmt), weights));
    4641              : 
    4642              :     /* OMP directives are generally very expensive.  */
    4643              : 
    4644              :     case GIMPLE_OMP_RETURN:
    4645              :     case GIMPLE_OMP_SECTIONS_SWITCH:
    4646              :     case GIMPLE_OMP_ATOMIC_STORE:
    4647              :     case GIMPLE_OMP_CONTINUE:
    4648              :       /* ...except these, which are cheap.  */
    4649              :       return 0;
    4650              : 
    4651            0 :     case GIMPLE_OMP_ATOMIC_LOAD:
    4652            0 :       return weights->omp_cost;
    4653              : 
    4654            0 :     case GIMPLE_OMP_FOR:
    4655            0 :       return (weights->omp_cost
    4656            0 :               + estimate_num_insns_seq (gimple_omp_body (stmt), weights)
    4657            0 :               + estimate_num_insns_seq (gimple_omp_for_pre_body (stmt), weights));
    4658              : 
    4659            0 :     case GIMPLE_OMP_PARALLEL:
    4660            0 :     case GIMPLE_OMP_TASK:
    4661            0 :     case GIMPLE_OMP_CRITICAL:
    4662            0 :     case GIMPLE_OMP_MASTER:
    4663            0 :     case GIMPLE_OMP_MASKED:
    4664            0 :     case GIMPLE_OMP_SCOPE:
    4665            0 :     case GIMPLE_OMP_DISPATCH:
    4666            0 :     case GIMPLE_OMP_TASKGROUP:
    4667            0 :     case GIMPLE_OMP_ORDERED:
    4668            0 :     case GIMPLE_OMP_SCAN:
    4669            0 :     case GIMPLE_OMP_SECTION:
    4670            0 :     case GIMPLE_OMP_SECTIONS:
    4671            0 :     case GIMPLE_OMP_STRUCTURED_BLOCK:
    4672            0 :     case GIMPLE_OMP_SINGLE:
    4673            0 :     case GIMPLE_OMP_TARGET:
    4674            0 :     case GIMPLE_OMP_TEAMS:
    4675            0 :       return (weights->omp_cost
    4676            0 :               + estimate_num_insns_seq (gimple_omp_body (stmt), weights));
    4677              : 
    4678           75 :     case GIMPLE_TRANSACTION:
    4679           75 :       return (weights->tm_cost
    4680           75 :               + estimate_num_insns_seq (gimple_transaction_body (
    4681           75 :                                           as_a <gtransaction *> (stmt)),
    4682           75 :                                         weights));
    4683              : 
    4684            0 :     default:
    4685            0 :       gcc_unreachable ();
    4686              :     }
    4687              : 
    4688    472014878 :   return cost;
    4689              : }
    4690              : 
    4691              : /* Estimate number of instructions that will be created by expanding
    4692              :    function FNDECL.  WEIGHTS contains weights attributed to various
    4693              :    constructs.  */
    4694              : 
    4695              : int
    4696            0 : estimate_num_insns_fn (tree fndecl, eni_weights *weights)
    4697              : {
    4698            0 :   struct function *my_function = DECL_STRUCT_FUNCTION (fndecl);
    4699            0 :   gimple_stmt_iterator bsi;
    4700            0 :   basic_block bb;
    4701            0 :   int n = 0;
    4702              : 
    4703            0 :   gcc_assert (my_function && my_function->cfg);
    4704            0 :   FOR_EACH_BB_FN (bb, my_function)
    4705              :     {
    4706            0 :       for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
    4707            0 :         n += estimate_num_insns (gsi_stmt (bsi), weights);
    4708              :     }
    4709              : 
    4710            0 :   return n;
    4711              : }
    4712              : 
    4713              : 
    4714              : /* Initializes weights used by estimate_num_insns.  */
    4715              : 
    4716              : void
    4717       289641 : init_inline_once (void)
    4718              : {
    4719       289641 :   eni_size_weights.call_cost = 1;
    4720       289641 :   eni_size_weights.indirect_call_cost = 3;
    4721       289641 :   eni_size_weights.target_builtin_call_cost = 1;
    4722       289641 :   eni_size_weights.div_mod_cost = 1;
    4723       289641 :   eni_size_weights.omp_cost = 40;
    4724       289641 :   eni_size_weights.tm_cost = 10;
    4725       289641 :   eni_size_weights.time_based = false;
    4726       289641 :   eni_size_weights.return_cost = 1;
    4727              : 
    4728              :   /* Estimating time for call is difficult, since we have no idea what the
    4729              :      called function does.  In the current uses of eni_time_weights,
    4730              :      underestimating the cost does less harm than overestimating it, so
    4731              :      we choose a rather small value here.  */
    4732       289641 :   eni_time_weights.call_cost = 10;
    4733       289641 :   eni_time_weights.indirect_call_cost = 15;
    4734       289641 :   eni_time_weights.target_builtin_call_cost = 1;
    4735       289641 :   eni_time_weights.div_mod_cost = 10;
    4736       289641 :   eni_time_weights.omp_cost = 40;
    4737       289641 :   eni_time_weights.tm_cost = 40;
    4738       289641 :   eni_time_weights.time_based = true;
    4739       289641 :   eni_time_weights.return_cost = 2;
    4740       289641 : }
    4741              : 
    4742              : 
    4743              : /* Install new lexical TREE_BLOCK underneath 'current_block'.  */
    4744              : 
    4745              : static void
    4746     31050921 : prepend_lexical_block (tree current_block, tree new_block)
    4747              : {
    4748     31050921 :   BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block);
    4749     31050921 :   BLOCK_SUBBLOCKS (current_block) = new_block;
    4750     31050921 :   BLOCK_SUPERCONTEXT (new_block) = current_block;
    4751     31050921 : }
    4752              : 
    4753              : /* Add local variables from CALLEE to CALLER.  */
    4754              : 
    4755              : static inline void
    4756      4798334 : add_local_variables (struct function *callee, struct function *caller,
    4757              :                      copy_body_data *id)
    4758              : {
    4759      4798334 :   tree var;
    4760      4798334 :   unsigned ix;
    4761              : 
    4762     13299696 :   FOR_EACH_LOCAL_DECL (callee, ix, var)
    4763      4473890 :     if (!can_be_nonlocal (var, id))
    4764              :       {
    4765      4386596 :         tree new_var = remap_decl (var, id);
    4766              : 
    4767              :         /* Remap debug-expressions.  */
    4768      4386596 :         if (VAR_P (new_var)
    4769      4386596 :             && DECL_HAS_DEBUG_EXPR_P (var)
    4770      4706088 :             && new_var != var)
    4771              :           {
    4772       319492 :             tree tem = DECL_DEBUG_EXPR (var);
    4773       319492 :             bool old_regimplify = id->regimplify;
    4774       319492 :             id->remapping_type_depth++;
    4775       319492 :             walk_tree (&tem, copy_tree_body_r, id, NULL);
    4776       319492 :             id->remapping_type_depth--;
    4777       319492 :             id->regimplify = old_regimplify;
    4778       319492 :             SET_DECL_DEBUG_EXPR (new_var, tem);
    4779       319492 :             DECL_HAS_DEBUG_EXPR_P (new_var) = 1;
    4780              :           }
    4781      4386596 :         add_local_decl (caller, new_var);
    4782              :       }
    4783      4798334 : }
    4784              : 
    4785              : /* Add to BINDINGS a debug stmt resetting SRCVAR if inlining might
    4786              :    have brought in or introduced any debug stmts for SRCVAR.  */
    4787              : 
    4788              : static inline void
    4789      9952718 : reset_debug_binding (copy_body_data *id, tree srcvar, gimple_seq *bindings)
    4790              : {
    4791      9952718 :   tree *remappedvarp = id->decl_map->get (srcvar);
    4792              : 
    4793      9952718 :   if (!remappedvarp)
    4794              :     return;
    4795              : 
    4796      9896856 :   if (!VAR_P (*remappedvarp))
    4797              :     return;
    4798              : 
    4799      9745789 :   if (*remappedvarp == id->retvar)
    4800              :     return;
    4801              : 
    4802      9745789 :   tree tvar = target_for_debug_bind (*remappedvarp);
    4803      9745789 :   if (!tvar)
    4804              :     return;
    4805              : 
    4806     15668200 :   gdebug *stmt = gimple_build_debug_bind (tvar, NULL_TREE,
    4807      7834100 :                                           id->call_stmt);
    4808      7834100 :   gimple_seq_add_stmt (bindings, stmt);
    4809              : }
    4810              : 
    4811              : /* For each inlined variable for which we may have debug bind stmts,
    4812              :    add before GSI a final debug stmt resetting it, marking the end of
    4813              :    its life, so that var-tracking knows it doesn't have to compute
    4814              :    further locations for it.  */
    4815              : 
    4816              : static inline void
    4817      4646466 : reset_debug_bindings (copy_body_data *id, gimple_stmt_iterator gsi)
    4818              : {
    4819      4646466 :   tree var;
    4820      4646466 :   unsigned ix;
    4821      4646466 :   gimple_seq bindings = NULL;
    4822              : 
    4823      4646466 :   if (!gimple_in_ssa_p (id->src_cfun))
    4824       556339 :     return;
    4825              : 
    4826      4646466 :   if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
    4827              :     return;
    4828              : 
    4829      4090127 :   for (var = DECL_ARGUMENTS (id->src_fn);
    4830     11124914 :        var; var = DECL_CHAIN (var))
    4831      7034787 :     reset_debug_binding (id, var, &bindings);
    4832              : 
    4833     10440983 :   FOR_EACH_LOCAL_DECL (id->src_cfun, ix, var)
    4834      2917931 :     reset_debug_binding (id, var, &bindings);
    4835              : 
    4836      4090127 :   gsi_insert_seq_before_without_update (&gsi, bindings, GSI_SAME_STMT);
    4837              : }
    4838              : 
    4839              : /* If STMT is a GIMPLE_CALL, replace it with its inline expansion.  */
    4840              : 
    4841              : static bool
    4842     15948101 : expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id,
    4843              :                     bitmap to_purge)
    4844              : {
    4845     15948101 :   tree use_retvar;
    4846     15948101 :   tree fn;
    4847     15948101 :   hash_map<tree, tree> *dst;
    4848     15948101 :   hash_map<tree, tree> *st = NULL;
    4849     15948101 :   tree return_slot;
    4850     15948101 :   tree modify_dest;
    4851     15948101 :   struct cgraph_edge *cg_edge;
    4852     15948101 :   cgraph_inline_failed_t reason;
    4853     15948101 :   basic_block return_block;
    4854     15948101 :   edge e;
    4855     15948101 :   gimple_stmt_iterator gsi, stmt_gsi;
    4856     15948101 :   bool successfully_inlined = false;
    4857     15948101 :   bool purge_dead_abnormal_edges;
    4858     15948101 :   gcall *call_stmt;
    4859     15948101 :   unsigned int prop_mask, src_properties;
    4860     15948101 :   struct function *dst_cfun;
    4861     15948101 :   tree simduid;
    4862     15948101 :   use_operand_p use;
    4863     15948101 :   gimple *simtenter_stmt = NULL;
    4864     15948101 :   vec<tree> *simtvars_save;
    4865     15948101 :   tree save_stack = NULL_TREE;
    4866              : 
    4867              :   /* The gimplifier uses input_location in too many places, such as
    4868              :      internal_get_tmp_var ().  */
    4869     15948101 :   location_t saved_location = input_location;
    4870     15948101 :   input_location = gimple_location (stmt);
    4871              : 
    4872              :   /* From here on, we're only interested in CALL_EXPRs.  */
    4873     15948101 :   call_stmt = dyn_cast <gcall *> (stmt);
    4874     15948101 :   if (!call_stmt)
    4875            0 :     goto egress;
    4876              : 
    4877     15948101 :   cg_edge = id->dst_node->get_edge (stmt);
    4878              :   /* Edge should exist and speculations should be resolved at this
    4879              :      stage.  */
    4880     15948101 :   gcc_checking_assert (cg_edge && !cg_edge->speculative);
    4881              :   /* First, see if we can figure out what function is being called.
    4882              :      If we cannot, then there is no hope of inlining the function.  */
    4883     15948101 :   if (cg_edge->indirect_unknown_callee)
    4884       240570 :     goto egress;
    4885     15707531 :   fn = cg_edge->callee->decl;
    4886     15707531 :   gcc_checking_assert (fn);
    4887              : 
    4888              :   /* If FN is a declaration of a function in a nested scope that was
    4889              :      globally declared inline, we don't set its DECL_INITIAL.
    4890              :      However, we can't blindly follow DECL_ABSTRACT_ORIGIN because the
    4891              :      C++ front-end uses it for cdtors to refer to their internal
    4892              :      declarations, that are not real functions.  Fortunately those
    4893              :      don't have trees to be saved, so we can tell by checking their
    4894              :      gimple_body.  */
    4895     15707531 :   if (!DECL_INITIAL (fn)
    4896      6945034 :       && DECL_ABSTRACT_ORIGIN (fn)
    4897     15843064 :       && gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn)))
    4898          288 :     fn = DECL_ABSTRACT_ORIGIN (fn);
    4899              : 
    4900              :   /* Don't try to inline functions that are not well-suited to inlining.  */
    4901     15707531 :   if (cg_edge->inline_failed)
    4902              :     {
    4903     11060867 :       reason = cg_edge->inline_failed;
    4904              :       /* If this call was originally indirect, we do not want to emit any
    4905              :          inlining related warnings or sorry messages because there are no
    4906              :          guarantees regarding those.  */
    4907     11060867 :       if (cg_edge->indirect_inlining_edge)
    4908         1965 :         goto egress;
    4909              : 
    4910     11058902 :       if (lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn))
    4911              :           /* For extern inline functions that get redefined we always
    4912              :              silently ignored always_inline flag. Better behavior would
    4913              :              be to be able to keep both bodies and use extern inline body
    4914              :              for inlining, but we can't do that because frontends overwrite
    4915              :              the body.  */
    4916           63 :           && !cg_edge->callee->redefined_extern_inline
    4917              :           /* During early inline pass, report only when optimization is
    4918              :              not turned on.  */
    4919           63 :           && (symtab->global_info_ready
    4920           61 :               || !optimize
    4921           51 :               || cgraph_inline_failed_type (reason) == CIF_FINAL_ERROR)
    4922              :           /* PR 20090218-1_0.c. Body can be provided by another module. */
    4923     11058936 :           && (reason != CIF_BODY_NOT_AVAILABLE || !flag_generate_lto))
    4924              :         {
    4925           27 :           error ("inlining failed in call to %<always_inline%> %q+F: %s", fn,
    4926              :                  cgraph_inline_failed_string (reason));
    4927           27 :           if (gimple_location (stmt) != UNKNOWN_LOCATION)
    4928           27 :             inform (gimple_location (stmt), "called from here");
    4929            0 :           else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
    4930            0 :             inform (DECL_SOURCE_LOCATION (cfun->decl),
    4931              :                    "called from this function");
    4932              :         }
    4933     11058875 :       else if (opt_for_fn (fn, warn_inline)
    4934          339 :                && DECL_DECLARED_INLINE_P (fn)
    4935          110 :                && !DECL_NO_INLINE_WARNING_P (fn)
    4936           77 :                && !DECL_IN_SYSTEM_HEADER (fn)
    4937           60 :                && reason != CIF_UNSPECIFIED
    4938           60 :                && !lookup_attribute ("noinline", DECL_ATTRIBUTES (fn))
    4939              :                /* Do not warn about not inlined recursive calls.  */
    4940           60 :                && !cg_edge->recursive_p ()
    4941              :                /* Avoid warnings during early inline pass. */
    4942     11058935 :                && symtab->global_info_ready)
    4943              :         {
    4944            8 :           auto_diagnostic_group d;
    4945            8 :           if (warning (OPT_Winline, "inlining failed in call to %q+F: %s",
    4946              :                        fn, _(cgraph_inline_failed_string (reason))))
    4947              :             {
    4948            8 :               if (gimple_location (stmt) != UNKNOWN_LOCATION)
    4949            8 :                 inform (gimple_location (stmt), "called from here");
    4950            0 :               else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
    4951            0 :                 inform (DECL_SOURCE_LOCATION (cfun->decl),
    4952              :                        "called from this function");
    4953              :             }
    4954            8 :         }
    4955     11058902 :       goto egress;
    4956              :     }
    4957      4646664 :   id->src_node = cg_edge->callee;
    4958              : 
    4959              :   /* If callee is thunk, all we need is to adjust the THIS pointer
    4960              :      and redirect to function being thunked.  */
    4961      4646664 :   if (id->src_node->thunk)
    4962              :     {
    4963          198 :       cgraph_edge *edge;
    4964          198 :       tree virtual_offset = NULL;
    4965          198 :       profile_count count = cg_edge->count;
    4966          198 :       tree op;
    4967          198 :       gimple_stmt_iterator iter = gsi_for_stmt (stmt);
    4968          198 :       thunk_info *info = thunk_info::get (id->src_node);
    4969              : 
    4970          198 :       cgraph_edge::remove (cg_edge);
    4971          396 :       edge = id->src_node->callees->clone (id->dst_node, call_stmt,
    4972              :                                            gimple_uid (stmt),
    4973              :                                            profile_count::one (),
    4974              :                                            profile_count::one (),
    4975              :                                            true);
    4976          198 :       edge->count = count;
    4977          198 :       if (info->virtual_offset_p)
    4978            7 :         virtual_offset = size_int (info->virtual_value);
    4979          198 :       op = create_tmp_reg_fn (cfun, TREE_TYPE (gimple_call_arg (stmt, 0)),
    4980              :                               NULL);
    4981          198 :       gsi_insert_before (&iter, gimple_build_assign (op,
    4982              :                                                     gimple_call_arg (stmt, 0)),
    4983              :                          GSI_NEW_STMT);
    4984          198 :       gcc_assert (info->this_adjusting);
    4985          198 :       op = thunk_adjust (&iter, op, 1, info->fixed_offset,
    4986              :                          virtual_offset, info->indirect_offset);
    4987              : 
    4988          198 :       gimple_call_set_arg (stmt, 0, op);
    4989          198 :       gimple_call_set_fndecl (stmt, edge->callee->decl);
    4990          198 :       update_stmt (stmt);
    4991          198 :       id->src_node->remove ();
    4992          198 :       successfully_inlined = expand_call_inline (bb, stmt, id, to_purge);
    4993          198 :       maybe_remove_unused_call_args (cfun, stmt);
    4994              :       /* This used to return true even though we do fail to inline in
    4995              :          some cases.  See PR98525.  */
    4996          198 :       goto egress;
    4997              :     }
    4998      4646466 :   fn = cg_edge->callee->decl;
    4999      4646466 :   cg_edge->callee->get_untransformed_body ();
    5000              : 
    5001      4646466 :   if (flag_checking && cg_edge->callee->decl != id->dst_node->decl)
    5002      4646457 :     cg_edge->callee->verify ();
    5003              : 
    5004              :   /* We will be inlining this callee.  */
    5005      4646466 :   id->eh_lp_nr = lookup_stmt_eh_lp (stmt);
    5006              : 
    5007              :   /* Update the callers EH personality.  */
    5008      4646466 :   if (DECL_FUNCTION_PERSONALITY (fn))
    5009       301900 :     DECL_FUNCTION_PERSONALITY (cg_edge->caller->decl)
    5010       150950 :       = DECL_FUNCTION_PERSONALITY (fn);
    5011              : 
    5012              :   /* Split the block before the GIMPLE_CALL.  */
    5013      4646466 :   stmt_gsi = gsi_for_stmt (stmt);
    5014      4646466 :   gsi_prev (&stmt_gsi);
    5015      4646466 :   e = split_block (bb, gsi_end_p (stmt_gsi) ? NULL : gsi_stmt (stmt_gsi));
    5016      4646466 :   bb = e->src;
    5017      4646466 :   return_block = e->dest;
    5018      4646466 :   remove_edge (e);
    5019              : 
    5020              :   /* If the GIMPLE_CALL was in the last statement of BB, it may have
    5021              :      been the source of abnormal edges.  In this case, schedule
    5022              :      the removal of dead abnormal edges.  */
    5023      4646466 :   gsi = gsi_start_bb (return_block);
    5024      4646466 :   gsi_next (&gsi);
    5025      4646466 :   purge_dead_abnormal_edges = gsi_end_p (gsi);
    5026              : 
    5027      4646466 :   stmt_gsi = gsi_start_bb (return_block);
    5028              : 
    5029              :   /* Build a block containing code to initialize the arguments, the
    5030              :      actual inline expansion of the body, and a label for the return
    5031              :      statements within the function to jump to.  The type of the
    5032              :      statement expression is the return type of the function call.
    5033              :      ???  If the call does not have an associated block then we will
    5034              :      remap all callee blocks to NULL, effectively dropping most of
    5035              :      its debug information.  This should only happen for calls to
    5036              :      artificial decls inserted by the compiler itself.  We need to
    5037              :      either link the inlined blocks into the caller block tree or
    5038              :      not refer to them in any way to not break GC for locations.  */
    5039      4646466 :   if (tree block = gimple_block (stmt))
    5040              :     {
    5041              :       /* We do want to assign a not UNKNOWN_LOCATION BLOCK_SOURCE_LOCATION
    5042              :          to make inlined_function_outer_scope_p return true on this BLOCK.  */
    5043      4619558 :       location_t loc = LOCATION_LOCUS (gimple_location (stmt));
    5044      4619558 :       if (loc == UNKNOWN_LOCATION)
    5045       262152 :         loc = LOCATION_LOCUS (DECL_SOURCE_LOCATION (fn));
    5046       262152 :       if (loc == UNKNOWN_LOCATION)
    5047              :         loc = BUILTINS_LOCATION;
    5048      4619558 :       if (has_discriminator (gimple_location (stmt)))
    5049      1615829 :         loc = location_with_discriminator
    5050      1615829 :                 (loc, get_discriminator_from_loc (gimple_location (stmt)));
    5051      4619558 :       id->block = make_node (BLOCK);
    5052      7954062 :       BLOCK_ABSTRACT_ORIGIN (id->block) = DECL_ORIGIN (fn);
    5053      4619558 :       BLOCK_SOURCE_LOCATION (id->block) = loc;
    5054      4619558 :       prepend_lexical_block (block, id->block);
    5055              :     }
    5056              : 
    5057              :   /* Local declarations will be replaced by their equivalents in this map.  */
    5058      4646466 :   st = id->decl_map;
    5059      4646466 :   id->decl_map = new hash_map<tree, tree>;
    5060      4646466 :   dst = id->debug_map;
    5061      4646466 :   id->debug_map = NULL;
    5062      4646466 :   if (flag_stack_reuse != SR_NONE)
    5063      4639528 :     id->add_clobbers_to_eh_landing_pads = last_basic_block_for_fn (cfun);
    5064              : 
    5065              :   /* Record the function we are about to inline.  */
    5066      4646466 :   id->src_fn = fn;
    5067      4646466 :   id->src_cfun = DECL_STRUCT_FUNCTION (fn);
    5068      4646466 :   id->reset_location = DECL_IGNORED_P (fn);
    5069      4646466 :   id->call_stmt = call_stmt;
    5070      4646466 :   cfun->cfg->full_profile &= id->src_cfun->cfg->full_profile;
    5071              : 
    5072              :   /* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
    5073              :      variables to be added to IFN_GOMP_SIMT_ENTER argument list.  */
    5074      4646466 :   dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
    5075      4646466 :   simtvars_save = id->dst_simt_vars;
    5076      4646466 :   if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
    5077        21469 :       && (simduid = bb->loop_father->simduid) != NULL_TREE
    5078            0 :       && (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
    5079            0 :       && single_imm_use (simduid, &use, &simtenter_stmt)
    5080            0 :       && is_gimple_call (simtenter_stmt)
    5081      4646466 :       && gimple_call_internal_p (simtenter_stmt, IFN_GOMP_SIMT_ENTER))
    5082            0 :     vec_alloc (id->dst_simt_vars, 0);
    5083              :   else
    5084      4646466 :     id->dst_simt_vars = NULL;
    5085              : 
    5086      4646466 :   if (profile_status_for_fn (id->src_cfun) == PROFILE_ABSENT)
    5087        31291 :     profile_status_for_fn (dst_cfun) = PROFILE_ABSENT;
    5088              : 
    5089              :   /* If the src function contains an IFN_VA_ARG, then so will the dst
    5090              :      function after inlining.  Likewise for IFN_GOMP_USE_SIMT.  */
    5091      4646466 :   prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
    5092      4646466 :   src_properties = id->src_cfun->curr_properties & prop_mask;
    5093      4646466 :   if (src_properties != prop_mask)
    5094         1063 :     dst_cfun->curr_properties &= src_properties | ~prop_mask;
    5095      4646466 :   dst_cfun->calls_eh_return |= id->src_cfun->calls_eh_return;
    5096      4646466 :   id->dst_node->has_omp_variant_constructs
    5097      4646466 :     |= id->src_node->has_omp_variant_constructs;
    5098              : 
    5099      4646466 :   gcc_assert (!id->src_cfun->after_inlining);
    5100              : 
    5101      4646466 :   id->entry_bb = bb;
    5102      4646466 :   if (lookup_attribute ("cold", DECL_ATTRIBUTES (fn)))
    5103              :     {
    5104         1462 :       gimple_stmt_iterator si = gsi_last_bb (bb);
    5105         1462 :       gsi_insert_after (&si, gimple_build_predict (PRED_COLD_FUNCTION,
    5106              :                                                    NOT_TAKEN),
    5107              :                         GSI_NEW_STMT);
    5108              :     }
    5109      4646466 :   initialize_inlined_parameters (id, stmt, fn, bb);
    5110      4090145 :   if (debug_nonbind_markers_p && debug_inline_points && id->block
    5111      8724530 :       && inlined_function_outer_scope_p (id->block))
    5112              :     {
    5113      4078064 :       gimple_stmt_iterator si = gsi_last_bb (bb);
    5114      4078064 :       gsi_insert_after (&si, gimple_build_debug_inline_entry
    5115      4078064 :                         (id->block, DECL_SOURCE_LOCATION (id->src_fn)),
    5116              :                         GSI_NEW_STMT);
    5117              :     }
    5118              : 
    5119              :   /* If function to be inlined calls alloca, wrap the inlined function
    5120              :      in between save_stack = __builtin_stack_save (); and
    5121              :      __builtin_stack_restore (save_stack); calls.  */
    5122      4646466 :   if (id->src_cfun->calls_alloca && !gimple_call_noreturn_p (stmt))
    5123              :     /* Don't do this for VLA allocations though, just for user alloca
    5124              :        calls.  */
    5125         4703 :     for (struct cgraph_edge *e = id->src_node->callees; e; e = e->next_callee)
    5126         4293 :       if (gimple_maybe_alloca_call_p (e->call_stmt)
    5127         4293 :           && !gimple_call_alloca_for_var_p (e->call_stmt))
    5128              :         {
    5129           88 :           tree fn = builtin_decl_implicit (BUILT_IN_STACK_SAVE);
    5130           88 :           gcall *call = gimple_build_call (fn, 0);
    5131           88 :           save_stack = make_ssa_name (ptr_type_node);
    5132           88 :           gimple_call_set_lhs (call, save_stack);
    5133           88 :           gimple_stmt_iterator si = gsi_last_bb (bb);
    5134           88 :           gsi_insert_after (&si, call, GSI_NEW_STMT);
    5135           88 :           struct cgraph_node *dest = cgraph_node::get_create (fn);
    5136           88 :           id->dst_node->create_edge (dest, call, bb->count)->inline_failed
    5137           88 :             = CIF_BODY_NOT_AVAILABLE;
    5138           88 :           break;
    5139              :         }
    5140              : 
    5141      4646466 :   if (DECL_INITIAL (fn))
    5142              :     {
    5143      4646466 :       if (gimple_block (stmt))
    5144              :         {
    5145      4619558 :           tree *var;
    5146              : 
    5147      4619558 :           prepend_lexical_block (id->block,
    5148      4619558 :                                  remap_blocks (DECL_INITIAL (fn), id));
    5149      4619558 :           gcc_checking_assert (BLOCK_SUBBLOCKS (id->block)
    5150              :                                && (BLOCK_CHAIN (BLOCK_SUBBLOCKS (id->block))
    5151              :                                    == NULL_TREE));
    5152              :           /* Move vars for PARM_DECLs from DECL_INITIAL block to id->block,
    5153              :              otherwise for DWARF DW_TAG_formal_parameter will not be children of
    5154              :              DW_TAG_inlined_subroutine, but of a DW_TAG_lexical_block
    5155              :              under it.  The parameters can be then evaluated in the debugger,
    5156              :              but don't show in backtraces.  */
    5157      6792310 :           for (var = &BLOCK_VARS (BLOCK_SUBBLOCKS (id->block)); *var; )
    5158      2172752 :             if (TREE_CODE (DECL_ORIGIN (*var)) == PARM_DECL)
    5159              :               {
    5160       587210 :                 tree v = *var;
    5161       587210 :                 *var = TREE_CHAIN (v);
    5162       587210 :                 TREE_CHAIN (v) = BLOCK_VARS (id->block);
    5163       587210 :                 BLOCK_VARS (id->block) = v;
    5164              :               }
    5165              :             else
    5166      1585542 :               var = &TREE_CHAIN (*var);
    5167              :         }
    5168              :       else
    5169        26908 :         remap_blocks_to_null (DECL_INITIAL (fn), id);
    5170              :     }
    5171              : 
    5172              :   /* Return statements in the function body will be replaced by jumps
    5173              :      to the RET_LABEL.  */
    5174      4646466 :   gcc_assert (DECL_INITIAL (fn));
    5175      4646466 :   gcc_assert (TREE_CODE (DECL_INITIAL (fn)) == BLOCK);
    5176              : 
    5177              :   /* Find the LHS to which the result of this call is assigned.  */
    5178      4646466 :   return_slot = NULL;
    5179      4646466 :   if (gimple_call_lhs (stmt))
    5180              :     {
    5181      2339587 :       modify_dest = gimple_call_lhs (stmt);
    5182              : 
    5183              :       /* The function which we are inlining might not return a value,
    5184              :          in which case we should issue a warning that the function
    5185              :          does not return a value.  In that case the optimizers will
    5186              :          see that the variable to which the value is assigned was not
    5187              :          initialized.  We do not want to issue a warning about that
    5188              :          uninitialized variable.  */
    5189      2339587 :       if (DECL_P (modify_dest))
    5190       503192 :         suppress_warning (modify_dest, OPT_Wuninitialized);
    5191              : 
    5192              :       /* If we have a return slot, we can assign it the result directly,
    5193              :          except in the case where it is a global variable that is only
    5194              :          written to because, the callee being permitted to read or take
    5195              :          the address of its DECL_RESULT, this could invalidate the flag
    5196              :          on the global variable; instead we preventively remove the store,
    5197              :          which would have happened later if the call was not inlined.  */
    5198      2339587 :       if (gimple_call_return_slot_opt_p (call_stmt))
    5199              :         {
    5200       143027 :           tree base = get_base_address (modify_dest);
    5201              : 
    5202       143027 :           if (VAR_P (base)
    5203       118360 :               && (TREE_STATIC (base) || DECL_EXTERNAL (base))
    5204       143099 :               && varpool_node::get (base)->writeonly)
    5205              :             return_slot = NULL;
    5206              :           else
    5207              :             return_slot = modify_dest;
    5208              : 
    5209              :           modify_dest = NULL;
    5210              :         }
    5211              :     }
    5212              :   else
    5213              :     modify_dest = NULL;
    5214              : 
    5215              :   /* If we are inlining a call to the C++ operator new, we don't want
    5216              :      to use type based alias analysis on the return value.  Otherwise
    5217              :      we may get confused if the compiler sees that the inlined new
    5218              :      function returns a pointer which was just deleted.  See bug
    5219              :      33407.  */
    5220      4646466 :   if (DECL_IS_OPERATOR_NEW_P (fn))
    5221              :     {
    5222        20284 :       return_slot = NULL;
    5223        20284 :       modify_dest = NULL;
    5224              :     }
    5225              : 
    5226              :   /* Declare the return variable for the function.  */
    5227      4646466 :   use_retvar = declare_return_variable (id, return_slot, modify_dest, bb);
    5228              : 
    5229              :   /* Add local vars in this inlined callee to caller.  */
    5230      4646466 :   add_local_variables (id->src_cfun, cfun, id);
    5231              : 
    5232      4646466 :   if (dump_enabled_p ())
    5233              :     {
    5234          680 :       char buf[128];
    5235          680 :       snprintf (buf, sizeof(buf), "%4.2f",
    5236          680 :                 cg_edge->sreal_frequency ().to_double ());
    5237          680 :       dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
    5238          680 :                        call_stmt,
    5239              :                        "Inlining %C to %C with frequency %s\n",
    5240              :                        id->src_node, id->dst_node, buf);
    5241          680 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5242              :         {
    5243          198 :           id->src_node->dump (dump_file);
    5244          198 :           id->dst_node->dump (dump_file);
    5245              :         }
    5246              :     }
    5247              : 
    5248              :   /* This is it.  Duplicate the callee body.  Assume callee is
    5249              :      pre-gimplified.  Note that we must not alter the caller
    5250              :      function in any way before this point, as this CALL_EXPR may be
    5251              :      a self-referential call; if we're calling ourselves, we need to
    5252              :      duplicate our body before altering anything.  */
    5253      4646466 :   copy_body (id, bb, return_block, NULL);
    5254              : 
    5255      4646466 :   reset_debug_bindings (id, stmt_gsi);
    5256              : 
    5257      4646466 :   if (flag_stack_reuse != SR_NONE)
    5258     12562143 :     for (tree p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
    5259      7922615 :       if (!TREE_THIS_VOLATILE (p))
    5260              :         {
    5261              :           /* The value associated with P is a local temporary only if
    5262              :              there is no value associated with P in the debug map.  */
    5263      7922342 :           tree *varp = id->decl_map->get (p);
    5264      7922342 :           if (varp
    5265      7922342 :               && VAR_P (*varp)
    5266      7756106 :               && !is_gimple_reg (*varp)
    5267      8362962 :               && !(id->debug_map && id->debug_map->get (p)))
    5268              :             {
    5269       440205 :               tree clobber = build_clobber (TREE_TYPE (*varp),
    5270              :                                             CLOBBER_STORAGE_END);
    5271       440205 :               gimple *clobber_stmt;
    5272       440205 :               clobber_stmt = gimple_build_assign (*varp, clobber);
    5273       440205 :               gimple_set_location (clobber_stmt, gimple_location (stmt));
    5274       440205 :               gsi_insert_before (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
    5275              :             }
    5276              :         }
    5277              : 
    5278      4646466 :   if (save_stack)
    5279              :     {
    5280           88 :       tree fn = builtin_decl_implicit (BUILT_IN_STACK_RESTORE);
    5281           88 :       gcall *call = gimple_build_call (fn, 1, save_stack);
    5282           88 :       gsi_insert_before (&stmt_gsi, call, GSI_SAME_STMT);
    5283           88 :       struct cgraph_node *dest = cgraph_node::get_create (fn);
    5284           88 :       id->dst_node->create_edge (dest, call,
    5285              :                                  return_block->count)->inline_failed
    5286           88 :         = CIF_BODY_NOT_AVAILABLE;
    5287              :     }
    5288              : 
    5289              :   /* Reset the escaped solution.  */
    5290      4646466 :   if (cfun->gimple_df)
    5291              :     {
    5292      4646466 :       pt_solution_reset (&cfun->gimple_df->escaped);
    5293      4646466 :       pt_solution_reset (&cfun->gimple_df->escaped_return);
    5294              :     }
    5295              : 
    5296              :   /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
    5297      4646466 :   if (id->dst_simt_vars && id->dst_simt_vars->length () > 0)
    5298              :     {
    5299            0 :       size_t nargs = gimple_call_num_args (simtenter_stmt);
    5300            0 :       vec<tree> *vars = id->dst_simt_vars;
    5301            0 :       auto_vec<tree> newargs (nargs + vars->length ());
    5302            0 :       for (size_t i = 0; i < nargs; i++)
    5303            0 :         newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
    5304            0 :       for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
    5305              :         {
    5306            0 :           tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
    5307            0 :           newargs.quick_push (build1 (ADDR_EXPR, ptrtype, *pvar));
    5308              :         }
    5309            0 :       gcall *g = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
    5310            0 :       gimple_call_set_lhs (g, gimple_call_lhs (simtenter_stmt));
    5311            0 :       gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
    5312            0 :       gsi_replace (&gsi, g, false);
    5313            0 :     }
    5314      4646466 :   vec_free (id->dst_simt_vars);
    5315      4646466 :   id->dst_simt_vars = simtvars_save;
    5316              : 
    5317              :   /* Clean up.  */
    5318      4646466 :   if (id->debug_map)
    5319              :     {
    5320       161859 :       delete id->debug_map;
    5321       161859 :       id->debug_map = dst;
    5322              :     }
    5323      9292932 :   delete id->decl_map;
    5324      4646466 :   id->decl_map = st;
    5325              : 
    5326              :   /* Unlink the calls virtual operands before replacing it.  */
    5327      4646466 :   unlink_stmt_vdef (stmt);
    5328      4646466 :   if (gimple_vdef (stmt)
    5329      4646466 :       && TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
    5330      2162548 :     release_ssa_name (gimple_vdef (stmt));
    5331              : 
    5332              :   /* If the inlined function returns a result that we care about,
    5333              :      substitute the GIMPLE_CALL with an assignment of the return
    5334              :      variable to the LHS of the call.  That is, if STMT was
    5335              :      'a = foo (...)', substitute the call with 'a = USE_RETVAR'.  */
    5336      4646466 :   if (use_retvar && gimple_call_lhs (stmt))
    5337              :     {
    5338      1916809 :       gimple *old_stmt = stmt;
    5339      1916809 :       tree lhs = gimple_call_lhs (stmt);
    5340      1916809 :       if (!is_gimple_reg (lhs)
    5341       107330 :           && !is_gimple_reg (use_retvar)
    5342      2024097 :           && is_gimple_reg_type (TREE_TYPE (lhs)))
    5343              :         {
    5344              :           /* If both lhs and use_retvar aren't gimple regs, yet have
    5345              :              gimple reg type, copy through a temporary SSA_NAME.  */
    5346            0 :           gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (lhs)),
    5347              :                                            use_retvar);
    5348            0 :           gimple_set_location (g, gimple_location (old_stmt));
    5349            0 :           gsi_insert_before (&stmt_gsi, g, GSI_SAME_STMT);
    5350            0 :           use_retvar = gimple_assign_lhs (g);
    5351              :         }
    5352      1916809 :       stmt = gimple_build_assign (lhs, use_retvar);
    5353      1916809 :       gimple_set_location (stmt, gimple_location (old_stmt));
    5354      1916809 :       gsi_replace (&stmt_gsi, stmt, false);
    5355      1916809 :       maybe_clean_or_replace_eh_stmt (old_stmt, stmt);
    5356              :       /* Append a clobber for id->retvar if easily possible.  */
    5357      1916809 :       if (flag_stack_reuse != SR_NONE
    5358      1914661 :           && id->retvar
    5359      1914661 :           && VAR_P (id->retvar)
    5360      1914661 :           && id->retvar != return_slot
    5361      1914661 :           && id->retvar != modify_dest
    5362      1914661 :           && !TREE_THIS_VOLATILE (id->retvar)
    5363      1914631 :           && !is_gimple_reg (id->retvar)
    5364      2025155 :           && !stmt_ends_bb_p (stmt))
    5365              :         {
    5366       108343 :           tree clobber = build_clobber (TREE_TYPE (id->retvar),
    5367              :                                         CLOBBER_STORAGE_END);
    5368       108343 :           gimple *clobber_stmt;
    5369       108343 :           clobber_stmt = gimple_build_assign (id->retvar, clobber);
    5370       108343 :           gimple_set_location (clobber_stmt, gimple_location (old_stmt));
    5371       108343 :           gsi_insert_after (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
    5372              :         }
    5373              :     }
    5374              :   else
    5375              :     {
    5376              :       /* Handle the case of inlining a function with no return
    5377              :          statement, which causes the return value to become undefined.  */
    5378      2729657 :       if (gimple_call_lhs (stmt)
    5379      2729657 :           && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
    5380              :         {
    5381           21 :           tree name = gimple_call_lhs (stmt);
    5382           21 :           tree var = SSA_NAME_VAR (name);
    5383            6 :           tree def = var ? ssa_default_def (cfun, var) : NULL;
    5384              : 
    5385            6 :           if (def)
    5386              :             {
    5387              :               /* If the variable is used undefined, make this name
    5388              :                  undefined via a move.  */
    5389            0 :               stmt = gimple_build_assign (gimple_call_lhs (stmt), def);
    5390            0 :               gsi_replace (&stmt_gsi, stmt, true);
    5391              :             }
    5392              :           else
    5393              :             {
    5394           21 :               if (!var)
    5395              :                 {
    5396           15 :                   var = create_tmp_reg_fn (cfun, TREE_TYPE (name), NULL);
    5397           30 :                   SET_SSA_NAME_VAR_OR_IDENTIFIER (name, var);
    5398              :                 }
    5399              :               /* Otherwise make this variable undefined.  */
    5400           21 :               gsi_remove (&stmt_gsi, true);
    5401           21 :               set_ssa_default_def (cfun, var, name);
    5402           21 :               SSA_NAME_DEF_STMT (name) = gimple_build_nop ();
    5403              :             }
    5404              :         }
    5405              :       /* Replace with a clobber for id->retvar.  */
    5406      2729636 :       else if (flag_stack_reuse != SR_NONE
    5407      2724846 :                && id->retvar
    5408       572002 :                && VAR_P (id->retvar)
    5409       480418 :                && id->retvar != return_slot
    5410       428087 :                && id->retvar != modify_dest
    5411       149716 :                && !TREE_THIS_VOLATILE (id->retvar)
    5412      2879352 :                && !is_gimple_reg (id->retvar))
    5413              :         {
    5414        15172 :           tree clobber = build_clobber (TREE_TYPE (id->retvar));
    5415        15172 :           gimple *clobber_stmt;
    5416        15172 :           clobber_stmt = gimple_build_assign (id->retvar, clobber);
    5417        15172 :           gimple_set_location (clobber_stmt, gimple_location (stmt));
    5418        15172 :           gsi_replace (&stmt_gsi, clobber_stmt, false);
    5419        15172 :           maybe_clean_or_replace_eh_stmt (stmt, clobber_stmt);
    5420              :         }
    5421              :       else
    5422      2714464 :         gsi_remove (&stmt_gsi, true);
    5423              :     }
    5424              : 
    5425      4646466 :   if (purge_dead_abnormal_edges)
    5426      1132568 :     bitmap_set_bit (to_purge, return_block->index);
    5427              : 
    5428              :   /* If the value of the new expression is ignored, that's OK.  We
    5429              :      don't warn about this for CALL_EXPRs, so we shouldn't warn about
    5430              :      the equivalent inlined version either.  */
    5431      4646466 :   if (is_gimple_assign (stmt))
    5432              :     {
    5433      1916809 :       gcc_assert (gimple_assign_single_p (stmt)
    5434              :                   || CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)));
    5435      1916809 :       TREE_USED (gimple_assign_rhs1 (stmt)) = 1;
    5436              :     }
    5437              : 
    5438      4646466 :   id->add_clobbers_to_eh_landing_pads = 0;
    5439              : 
    5440              :   /* Output the inlining info for this abstract function, since it has been
    5441              :      inlined.  If we don't do this now, we can lose the information about the
    5442              :      variables in the function when the blocks get blown away as soon as we
    5443              :      remove the cgraph node.  */
    5444      4646466 :   if (gimple_block (stmt))
    5445      4619558 :     (*debug_hooks->outlining_inline_function) (fn);
    5446              : 
    5447              :   /* Update callgraph if needed.  */
    5448      4646466 :   cg_edge->callee->remove ();
    5449              : 
    5450      4646466 :   id->block = NULL_TREE;
    5451      4646466 :   id->retvar = NULL_TREE;
    5452      4646466 :   successfully_inlined = true;
    5453              : 
    5454     15948101 :  egress:
    5455     15948101 :   input_location = saved_location;
    5456     15948101 :   return successfully_inlined;
    5457              : }
    5458              : 
    5459              : /* Expand call statements reachable from STMT_P.
    5460              :    We can only have CALL_EXPRs as the "toplevel" tree code or nested
    5461              :    in a MODIFY_EXPR.  */
    5462              : 
    5463              : static bool
    5464     32849170 : gimple_expand_calls_inline (basic_block bb, copy_body_data *id,
    5465              :                             bitmap to_purge)
    5466              : {
    5467     32849170 :   gimple_stmt_iterator gsi;
    5468     32849170 :   bool inlined = false;
    5469              : 
    5470    244240220 :   for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi);)
    5471              :     {
    5472    178541880 :       gimple *stmt = gsi_stmt (gsi);
    5473    178541880 :       gsi_prev (&gsi);
    5474              : 
    5475    178541880 :       if (is_gimple_call (stmt)
    5476    178541880 :           && !gimple_call_internal_p (stmt))
    5477     15947903 :         inlined |= expand_call_inline (bb, stmt, id, to_purge);
    5478              :     }
    5479              : 
    5480     32849170 :   return inlined;
    5481              : }
    5482              : 
    5483              : 
    5484              : /* Walk all basic blocks created after FIRST and try to fold every statement
    5485              :    in the STATEMENTS pointer set.  */
    5486              : 
    5487              : static void
    5488      1669662 : fold_marked_statements (int first, hash_set<gimple *> *statements)
    5489              : {
    5490      1669662 :   auto_bitmap to_purge;
    5491      1669662 :   auto_bitmap to_purge_abnormal;
    5492              : 
    5493      1669662 :   auto_vec<edge, 20> stack (n_basic_blocks_for_fn (cfun) + 2);
    5494      1669662 :   auto_sbitmap visited (last_basic_block_for_fn (cfun));
    5495      1669662 :   bitmap_clear (visited);
    5496              : 
    5497      1669662 :   stack.quick_push (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    5498     40604498 :   while (!stack.is_empty ())
    5499              :     {
    5500              :       /* Look at the edge on the top of the stack.  */
    5501     38934836 :       edge e = stack.pop ();
    5502     38934836 :       basic_block dest = e->dest;
    5503              : 
    5504     47933798 :       if (dest == EXIT_BLOCK_PTR_FOR_FN (cfun)
    5505     37290789 :           || bitmap_bit_p (visited, dest->index))
    5506      8998962 :         continue;
    5507              : 
    5508     29935874 :       bitmap_set_bit (visited, dest->index);
    5509              : 
    5510     29935874 :       if (dest->index >= first)
    5511     38931840 :         for (gimple_stmt_iterator gsi = gsi_start_bb (dest);
    5512    136901469 :              !gsi_end_p (gsi); gsi_next (&gsi))
    5513              :           {
    5514    117435549 :             if (!statements->contains (gsi_stmt (gsi)))
    5515     24044849 :               continue;
    5516              : 
    5517     93390700 :             gimple *old_stmt = gsi_stmt (gsi);
    5518     93390700 :             bool can_make_abnormal_goto = false;
    5519     93390700 :             tree old_decl = NULL_TREE;
    5520              : 
    5521     93390700 :             if (is_gimple_call (old_stmt))
    5522              :               {
    5523      4153602 :                 old_decl = gimple_call_fndecl (old_stmt);
    5524      4153602 :                 if (stmt_can_make_abnormal_goto (old_stmt))
    5525              :                   can_make_abnormal_goto = true;
    5526              :               }
    5527              : 
    5528      4153602 :             if (old_decl && fndecl_built_in_p (old_decl))
    5529              :               {
    5530              :                 /* Folding builtins can create multiple instructions,
    5531              :                    we need to look at all of them.  */
    5532      1568715 :                 gimple_stmt_iterator i2 = gsi;
    5533      1568715 :                 gsi_prev (&i2);
    5534      1568715 :                 if (fold_stmt (&gsi))
    5535              :                   {
    5536        93544 :                     gimple *new_stmt;
    5537              :                     /* If a builtin at the end of a bb folded into nothing,
    5538              :                        the following loop won't work.  */
    5539        93544 :                     if (gsi_end_p (gsi))
    5540              :                       {
    5541            0 :                         cgraph_update_edges_for_call_stmt (old_stmt,
    5542              :                                                            old_decl, NULL);
    5543            0 :                         if (can_make_abnormal_goto)
    5544            0 :                           bitmap_set_bit (to_purge_abnormal, dest->index);
    5545     19465920 :                         break;
    5546              :                       }
    5547        93544 :                     if (gsi_end_p (i2))
    5548       125456 :                       i2 = gsi_start_bb (dest);
    5549              :                     else
    5550        30816 :                       gsi_next (&i2);
    5551        94472 :                     while (1)
    5552              :                       {
    5553          464 :                         new_stmt = gsi_stmt (i2);
    5554        94008 :                         update_stmt (new_stmt);
    5555        94008 :                         cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
    5556              :                                                            new_stmt);
    5557              : 
    5558        94008 :                         if (new_stmt == gsi_stmt (gsi))
    5559              :                           {
    5560              :                             /* It is okay to check only for the very last
    5561              :                                of these statements.  If it is a throwing
    5562              :                                statement nothing will change.  If it isn't
    5563              :                                this can remove EH edges.  If that weren't
    5564              :                                correct then because some intermediate stmts
    5565              :                                throw, but not the last one.  That would mean
    5566              :                                we'd have to split the block, which we can't
    5567              :                                here and we'd loose anyway.  And as builtins
    5568              :                                probably never throw, this all
    5569              :                                is mood anyway.  */
    5570        93544 :                             if (maybe_clean_or_replace_eh_stmt (old_stmt,
    5571              :                                                                 new_stmt))
    5572            7 :                               bitmap_set_bit (to_purge, dest->index);
    5573        93544 :                             if (can_make_abnormal_goto
    5574        93544 :                                 && !stmt_can_make_abnormal_goto (new_stmt))
    5575            0 :                               bitmap_set_bit (to_purge_abnormal, dest->index);
    5576              :                             break;
    5577              :                           }
    5578          464 :                         gsi_next (&i2);
    5579              :                       }
    5580              :                   }
    5581              :               }
    5582     91821985 :             else if (fold_stmt (&gsi))
    5583              :               {
    5584              :                 /* Re-read the statement from GSI as fold_stmt() may
    5585              :                    have changed it.  */
    5586      3010004 :                 gimple *new_stmt = gsi_stmt (gsi);
    5587      3010004 :                 update_stmt (new_stmt);
    5588              : 
    5589      3010004 :                 if (is_gimple_call (old_stmt)
    5590      3010004 :                     || is_gimple_call (new_stmt))
    5591         4867 :                   cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
    5592              :                                                      new_stmt);
    5593              : 
    5594      3010004 :                 if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt))
    5595          154 :                   bitmap_set_bit (to_purge, dest->index);
    5596      3010004 :                 if (can_make_abnormal_goto
    5597      3010004 :                     && !stmt_can_make_abnormal_goto (new_stmt))
    5598            7 :                   bitmap_set_bit (to_purge_abnormal, dest->index);
    5599              :               }
    5600              :           }
    5601              : 
    5602     58205861 :       if (EDGE_COUNT (dest->succs) > 0)
    5603              :         {
    5604              :           /* Avoid warnings emitted from folding statements that
    5605              :              became unreachable because of inlined function parameter
    5606              :              propagation.  */
    5607     28269987 :           e = find_taken_edge (dest, NULL_TREE);
    5608     28269987 :           if (e)
    5609     19467391 :             stack.quick_push (e);
    5610              :           else
    5611              :             {
    5612      8802596 :               edge_iterator ei;
    5613     26600379 :               FOR_EACH_EDGE (e, ei, dest->succs)
    5614     17797783 :                 stack.safe_push (e);
    5615              :             }
    5616              :         }
    5617              :     }
    5618              : 
    5619      1669662 :   gimple_purge_all_dead_eh_edges (to_purge);
    5620      1669662 :   gimple_purge_all_dead_abnormal_call_edges (to_purge_abnormal);
    5621      1669662 : }
    5622              : 
    5623              : /* Expand calls to inline functions in the body of FN.  */
    5624              : 
    5625              : unsigned int
    5626      2041742 : optimize_inline_calls (tree fn)
    5627              : {
    5628      2041742 :   copy_body_data id;
    5629      2041742 :   basic_block bb;
    5630      2041742 :   int last = n_basic_blocks_for_fn (cfun);
    5631      2041742 :   bool inlined_p = false;
    5632              : 
    5633              :   /* Clear out ID.  */
    5634      2041742 :   memset (&id, 0, sizeof (id));
    5635              : 
    5636      2041742 :   id.src_node = id.dst_node = cgraph_node::get (fn);
    5637      2041742 :   gcc_assert (id.dst_node->definition);
    5638      2041742 :   id.dst_fn = fn;
    5639              :   /* Or any functions that aren't finished yet.  */
    5640      2041742 :   if (current_function_decl)
    5641      2041742 :     id.dst_fn = current_function_decl;
    5642              : 
    5643      2041742 :   id.copy_decl = copy_decl_maybe_to_var;
    5644      2041742 :   id.transform_call_graph_edges = CB_CGE_DUPLICATE;
    5645      2041742 :   id.transform_new_cfg = false;
    5646      2041742 :   id.transform_return_to_modify = true;
    5647      2041742 :   id.transform_parameter = true;
    5648      2041742 :   id.statements_to_fold = new hash_set<gimple *>;
    5649              : 
    5650      2041742 :   push_gimplify_context ();
    5651              : 
    5652              :   /* We make no attempts to keep dominance info up-to-date.  */
    5653      2041742 :   free_dominance_info (CDI_DOMINATORS);
    5654      2041742 :   free_dominance_info (CDI_POST_DOMINATORS);
    5655              : 
    5656              :   /* Register specific gimple functions.  */
    5657      2041742 :   gimple_register_cfg_hooks ();
    5658              : 
    5659              :   /* Reach the trees by walking over the CFG, and note the
    5660              :      enclosing basic-blocks in the call edges.  */
    5661              :   /* We walk the blocks going forward, because inlined function bodies
    5662              :      will split id->current_basic_block, and the new blocks will
    5663              :      follow it; we'll trudge through them, processing their CALL_EXPRs
    5664              :      along the way.  */
    5665      2041742 :   auto_bitmap to_purge;
    5666     34890912 :   FOR_EACH_BB_FN (bb, cfun)
    5667     32849170 :     inlined_p |= gimple_expand_calls_inline (bb, &id, to_purge);
    5668              : 
    5669      2041742 :   pop_gimplify_context (NULL);
    5670              : 
    5671      2041742 :   if (flag_checking)
    5672              :     {
    5673      2041724 :       struct cgraph_edge *e;
    5674              : 
    5675      2041724 :       id.dst_node->verify ();
    5676              : 
    5677              :       /* Double check that we inlined everything we are supposed to inline.  */
    5678     12389986 :       for (e = id.dst_node->callees; e; e = e->next_callee)
    5679     10348262 :         gcc_assert (e->inline_failed);
    5680              :     }
    5681              : 
    5682              :   /* If we didn't inline into the function there is nothing to do.  */
    5683      2041742 :   if (!inlined_p)
    5684              :     {
    5685      1250508 :       delete id.statements_to_fold;
    5686              :       return 0;
    5687              :     }
    5688              : 
    5689              :   /* Fold queued statements.  */
    5690      1416488 :   update_max_bb_count ();
    5691      1416488 :   fold_marked_statements (last, id.statements_to_fold);
    5692      2832976 :   delete id.statements_to_fold;
    5693              : 
    5694              :   /* Finally purge EH and abnormal edges from the call stmts we inlined.
    5695              :      We need to do this after fold_marked_statements since that may walk
    5696              :      the SSA use-def chain.  */
    5697      1416488 :   unsigned i;
    5698      1416488 :   bitmap_iterator bi;
    5699      2549056 :   EXECUTE_IF_SET_IN_BITMAP (to_purge, 0, i, bi)
    5700              :     {
    5701      1132568 :       basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
    5702      1132568 :       if (bb)
    5703              :         {
    5704      1132568 :           gimple_purge_dead_eh_edges (bb);
    5705      1132568 :           gimple_purge_dead_abnormal_call_edges (bb);
    5706              :         }
    5707              :     }
    5708              : 
    5709      1416488 :   gcc_assert (!id.debug_stmts.exists ());
    5710              : 
    5711              :   /* Renumber the lexical scoping (non-code) blocks consecutively.  */
    5712      1416488 :   number_blocks (fn);
    5713              : 
    5714      1416488 :   delete_unreachable_blocks_update_callgraph (id.dst_node, false);
    5715      1416488 :   id.dst_node->calls_comdat_local = id.dst_node->check_calls_comdat_local_p ();
    5716              : 
    5717      1416488 :   if (flag_checking)
    5718      1416479 :     id.dst_node->verify ();
    5719              : 
    5720              :   /* It would be nice to check SSA/CFG/statement consistency here, but it is
    5721              :      not possible yet - the IPA passes might make various functions to not
    5722              :      throw and they don't care to proactively update local EH info.  This is
    5723              :      done later in fixup_cfg pass that also execute the verification.  */
    5724      1416488 :   return (TODO_update_ssa
    5725              :           | TODO_cleanup_cfg
    5726      1416488 :           | (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0)
    5727      2832976 :           | (gimple_in_ssa_p (cfun) ? TODO_update_address_taken : 0));
    5728      2041742 : }
    5729              : 
    5730              : /* Passed to walk_tree.  Copies the node pointed to, if appropriate.  */
    5731              : 
    5732              : tree
    5733   1987613217 : copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
    5734              : {
    5735   1987613217 :   enum tree_code code = TREE_CODE (*tp);
    5736   1987613217 :   enum tree_code_class cl = TREE_CODE_CLASS (code);
    5737              : 
    5738              :   /* We make copies of most nodes.  */
    5739   1987613217 :   if (IS_EXPR_CODE_CLASS (cl)
    5740              :       || code == TREE_LIST
    5741    304128766 :       || code == TREE_VEC
    5742    299143816 :       || code == TYPE_DECL
    5743    299143816 :       || code == OMP_CLAUSE)
    5744              :     {
    5745              :       /* Because the chain gets clobbered when we make a copy, we save it
    5746              :          here.  */
    5747   1688493864 :       tree chain = NULL_TREE, new_tree;
    5748              : 
    5749   1688493864 :       if (CODE_CONTAINS_STRUCT (code, TS_COMMON))
    5750      5009413 :         chain = TREE_CHAIN (*tp);
    5751              : 
    5752              :       /* Copy the node.  */
    5753   1688493864 :       new_tree = copy_node (*tp);
    5754              : 
    5755   1688493864 :       *tp = new_tree;
    5756              : 
    5757              :       /* Now, restore the chain, if appropriate.  That will cause
    5758              :          walk_tree to walk into the chain as well.  */
    5759   1688493864 :       if (code == PARM_DECL
    5760   1688493864 :           || code == TREE_LIST
    5761   1686266751 :           || code == OMP_CLAUSE)
    5762      2251548 :         TREE_CHAIN (*tp) = chain;
    5763              : 
    5764              :       /* For now, we don't update BLOCKs when we make copies.  So, we
    5765              :          have to nullify all BIND_EXPRs.  */
    5766   1688493864 :       if (TREE_CODE (*tp) == BIND_EXPR)
    5767     20570612 :         BIND_EXPR_BLOCK (*tp) = NULL_TREE;
    5768              :     }
    5769    299119353 :   else if (code == CONSTRUCTOR)
    5770              :     {
    5771              :       /* CONSTRUCTOR nodes need special handling because
    5772              :          we need to duplicate the vector of elements.  */
    5773     18300018 :       tree new_tree;
    5774              : 
    5775     18300018 :       new_tree = copy_node (*tp);
    5776     26667790 :       CONSTRUCTOR_ELTS (new_tree) = vec_safe_copy (CONSTRUCTOR_ELTS (*tp));
    5777     18300018 :       *tp = new_tree;
    5778              :     }
    5779    280819335 :   else if (code == STATEMENT_LIST)
    5780              :     /* We used to just abort on STATEMENT_LIST, but we can run into them
    5781              :        with statement-expressions (c++/40975).  */
    5782          198 :     copy_statement_list (tp);
    5783    280819137 :   else if (TREE_CODE_CLASS (code) == tcc_type)
    5784          278 :     *walk_subtrees = 0;
    5785    280818859 :   else if (TREE_CODE_CLASS (code) == tcc_declaration)
    5786    121368302 :     *walk_subtrees = 0;
    5787    159450557 :   else if (TREE_CODE_CLASS (code) == tcc_constant)
    5788            0 :     *walk_subtrees = 0;
    5789   1987613217 :   return NULL_TREE;
    5790              : }
    5791              : 
    5792              : /* The SAVE_EXPR pointed to by TP is being copied.  If ST contains
    5793              :    information indicating to what new SAVE_EXPR this one should be mapped,
    5794              :    use that one.  Otherwise, create a new node and enter it in ST.  FN is
    5795              :    the function into which the copy will be placed.  */
    5796              : 
    5797              : static void
    5798     15698316 : remap_save_expr (tree *tp, hash_map<tree, tree> *st, int *walk_subtrees)
    5799              : {
    5800     15698316 :   tree *n;
    5801     15698316 :   tree t;
    5802              : 
    5803              :   /* See if we already encountered this SAVE_EXPR.  */
    5804     15698316 :   n = st->get (*tp);
    5805              : 
    5806              :   /* If we didn't already remap this SAVE_EXPR, do so now.  */
    5807     15698316 :   if (!n)
    5808              :     {
    5809     15240381 :       t = copy_node (*tp);
    5810              : 
    5811              :       /* Remember this SAVE_EXPR.  */
    5812     15240381 :       st->put (*tp, t);
    5813              :       /* Make sure we don't remap an already-remapped SAVE_EXPR.  */
    5814     15240381 :       st->put (t, t);
    5815              :     }
    5816              :   else
    5817              :     {
    5818              :       /* We've already walked into this SAVE_EXPR; don't do it again.  */
    5819       457935 :       *walk_subtrees = 0;
    5820       457935 :       t = *n;
    5821              :     }
    5822              : 
    5823              :   /* Replace this SAVE_EXPR with the copy.  */
    5824     15698316 :   *tp = t;
    5825     15698316 : }
    5826              : 
    5827              : /* Called via walk_gimple_seq.  If *GSIP points to a GIMPLE_LABEL for a local
    5828              :    label, copies the declaration and enters it in the splay_tree in DATA (which
    5829              :    is really a 'copy_body_data *'.  */
    5830              : 
    5831              : static tree
    5832      1635702 : mark_local_labels_stmt (gimple_stmt_iterator *gsip,
    5833              :                         bool *handled_ops_p ATTRIBUTE_UNUSED,
    5834              :                         struct walk_stmt_info *wi)
    5835              : {
    5836      1635702 :   copy_body_data *id = (copy_body_data *) wi->info;
    5837      1635702 :   glabel *stmt = dyn_cast <glabel *> (gsi_stmt (*gsip));
    5838              : 
    5839        53944 :   if (stmt)
    5840              :     {
    5841        53944 :       tree decl = gimple_label_label (stmt);
    5842              : 
    5843              :       /* Copy the decl and remember the copy.  */
    5844        53944 :       insert_decl_map (id, decl, id->copy_decl (decl, id));
    5845              :     }
    5846              : 
    5847      1635702 :   return NULL_TREE;
    5848              : }
    5849              : 
    5850              : static gimple_seq duplicate_remap_omp_clause_seq (gimple_seq seq,
    5851              :                                                   struct walk_stmt_info *wi);
    5852              : 
    5853              : /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
    5854              :    Using the splay_tree pointed to by ST (which is really a `splay_tree'),
    5855              :    remaps all local declarations to appropriate replacements in gimple
    5856              :    operands. */
    5857              : 
    5858              : static tree
    5859      3576578 : replace_locals_op (tree *tp, int *walk_subtrees, void *data)
    5860              : {
    5861      3576578 :   struct walk_stmt_info *wi = (struct walk_stmt_info*) data;
    5862      3576578 :   copy_body_data *id = (copy_body_data *) wi->info;
    5863      3576578 :   hash_map<tree, tree> *st = id->decl_map;
    5864      3576578 :   tree *n;
    5865      3576578 :   tree expr = *tp;
    5866              : 
    5867              :   /* For recursive invocations this is no longer the LHS itself.  */
    5868      3576578 :   bool is_lhs = wi->is_lhs;
    5869      3576578 :   wi->is_lhs = false;
    5870              : 
    5871      3576578 :   if (TREE_CODE (expr) == SSA_NAME)
    5872              :     {
    5873        24614 :       *tp = remap_ssa_name (*tp, id);
    5874        24614 :       *walk_subtrees = 0;
    5875        24614 :       if (is_lhs)
    5876        12307 :         SSA_NAME_DEF_STMT (*tp) = gsi_stmt (wi->gsi);
    5877              :     }
    5878              :   /* Only a local declaration (variable or label).  */
    5879      3551964 :   else if ((VAR_P (expr) && !TREE_STATIC (expr))
    5880      2071405 :            || TREE_CODE (expr) == LABEL_DECL)
    5881              :     {
    5882              :       /* Lookup the declaration.  */
    5883      1590503 :       n = st->get (expr);
    5884              : 
    5885              :       /* If it's there, remap it.  */
    5886      1590503 :       if (n)
    5887       108602 :         *tp = *n;
    5888      1590503 :       *walk_subtrees = 0;
    5889              :     }
    5890      1961461 :   else if (TREE_CODE (expr) == STATEMENT_LIST
    5891      1961461 :            || TREE_CODE (expr) == BIND_EXPR
    5892      1961461 :            || TREE_CODE (expr) == SAVE_EXPR)
    5893            0 :     gcc_unreachable ();
    5894      1961461 :   else if (TREE_CODE (expr) == TARGET_EXPR)
    5895              :     {
    5896              :       /* Don't mess with a TARGET_EXPR that hasn't been expanded.
    5897              :          It's OK for this to happen if it was part of a subtree that
    5898              :          isn't immediately expanded, such as operand 2 of another
    5899              :          TARGET_EXPR.  */
    5900            0 :       if (!TREE_OPERAND (expr, 1))
    5901              :         {
    5902            0 :           TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
    5903            0 :           TREE_OPERAND (expr, 3) = NULL_TREE;
    5904              :         }
    5905              :     }
    5906      1961461 :   else if (TREE_CODE (expr) == OMP_CLAUSE)
    5907              :     {
    5908              :       /* Before the omplower pass completes, some OMP clauses can contain
    5909              :          sequences that are neither copied by gimple_seq_copy nor walked by
    5910              :          walk_gimple_seq.  To make copy_gimple_seq_and_replace_locals work even
    5911              :          in those situations, we have to copy and process them explicitly.  */
    5912              : 
    5913          588 :       if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LASTPRIVATE)
    5914              :         {
    5915           14 :           gimple_seq seq = OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr);
    5916           14 :           seq = duplicate_remap_omp_clause_seq (seq, wi);
    5917           14 :           OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr) = seq;
    5918              :         }
    5919          574 :       else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LINEAR)
    5920              :         {
    5921           85 :           gimple_seq seq = OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr);
    5922           85 :           seq = duplicate_remap_omp_clause_seq (seq, wi);
    5923           85 :           OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr) = seq;
    5924              :         }
    5925          489 :       else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_REDUCTION)
    5926              :         {
    5927          107 :           gimple_seq seq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr);
    5928          107 :           seq = duplicate_remap_omp_clause_seq (seq, wi);
    5929          107 :           OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr) = seq;
    5930          107 :           seq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr);
    5931          107 :           seq = duplicate_remap_omp_clause_seq (seq, wi);
    5932          107 :           OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr) = seq;
    5933              :         }
    5934              :     }
    5935              : 
    5936              :   /* Keep iterating.  */
    5937      3576578 :   return NULL_TREE;
    5938              : }
    5939              : 
    5940              : 
    5941              : /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
    5942              :    Using the splay_tree pointed to by ST (which is really a `splay_tree'),
    5943              :    remaps all local declarations to appropriate replacements in gimple
    5944              :    statements. */
    5945              : 
    5946              : static tree
    5947      1635702 : replace_locals_stmt (gimple_stmt_iterator *gsip,
    5948              :                      bool *handled_ops_p ATTRIBUTE_UNUSED,
    5949              :                      struct walk_stmt_info *wi)
    5950              : {
    5951      1635702 :   copy_body_data *id = (copy_body_data *) wi->info;
    5952      1635702 :   gimple *gs = gsi_stmt (*gsip);
    5953              : 
    5954      1635702 :   if (gbind *stmt = dyn_cast <gbind *> (gs))
    5955              :     {
    5956          357 :       tree block = gimple_bind_block (stmt);
    5957              : 
    5958          357 :       if (block)
    5959              :         {
    5960          277 :           remap_block (&block, id);
    5961          277 :           gimple_bind_set_block (stmt, block);
    5962              :         }
    5963              : 
    5964              :       /* This will remap a lot of the same decls again, but this should be
    5965              :          harmless.  */
    5966          357 :       if (gimple_bind_vars (stmt))
    5967              :         {
    5968              :           tree old_var, decls = gimple_bind_vars (stmt);
    5969              : 
    5970          528 :           for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
    5971          360 :             if (!can_be_nonlocal (old_var, id)
    5972          360 :                 && ! variably_modified_type_p (TREE_TYPE (old_var), id->src_fn))
    5973          360 :               remap_decl (old_var, id);
    5974              : 
    5975          168 :           gcc_checking_assert (!id->prevent_decl_creation_for_types);
    5976          168 :           id->prevent_decl_creation_for_types = true;
    5977          168 :           gimple_bind_set_vars (stmt, remap_decls (decls, NULL, id));
    5978          168 :           id->prevent_decl_creation_for_types = false;
    5979              :         }
    5980              :     }
    5981              : 
    5982              :   /* Keep iterating.  */
    5983      1635702 :   return NULL_TREE;
    5984              : }
    5985              : 
    5986              : /* Create a copy of SEQ and remap all decls in it.  */
    5987              : 
    5988              : static gimple_seq
    5989          313 : duplicate_remap_omp_clause_seq (gimple_seq seq, struct walk_stmt_info *wi)
    5990              : {
    5991          313 :   if (!seq)
    5992              :     return NULL;
    5993              : 
    5994              :   /* If there are any labels in OMP sequences, they can be only referred to in
    5995              :      the sequence itself and therefore we can do both here.  */
    5996           60 :   walk_gimple_seq (seq, mark_local_labels_stmt, NULL, wi);
    5997           60 :   gimple_seq copy = gimple_seq_copy (seq);
    5998           60 :   walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, wi);
    5999           60 :   return copy;
    6000              : }
    6001              : 
    6002              : /* Copies everything in SEQ and replaces variables and labels local to
    6003              :    current_function_decl.  */
    6004              : 
    6005              : gimple_seq
    6006      1105484 : copy_gimple_seq_and_replace_locals (gimple_seq seq)
    6007              : {
    6008      1105484 :   copy_body_data id;
    6009      1105484 :   struct walk_stmt_info wi;
    6010      1105484 :   gimple_seq copy;
    6011              : 
    6012              :   /* There's nothing to do for NULL_TREE.  */
    6013      1105484 :   if (seq == NULL)
    6014              :     return seq;
    6015              : 
    6016              :   /* Set up ID.  */
    6017      1105468 :   memset (&id, 0, sizeof (id));
    6018      1105468 :   id.src_fn = current_function_decl;
    6019      1105468 :   id.dst_fn = current_function_decl;
    6020      1105468 :   id.src_cfun = cfun;
    6021      1105468 :   id.decl_map = new hash_map<tree, tree>;
    6022      1105468 :   id.debug_map = NULL;
    6023              : 
    6024      1105468 :   id.copy_decl = copy_decl_no_change;
    6025      1105468 :   id.transform_call_graph_edges = CB_CGE_DUPLICATE;
    6026      1105468 :   id.transform_new_cfg = false;
    6027      1105468 :   id.transform_return_to_modify = false;
    6028      1105468 :   id.transform_parameter = false;
    6029              : 
    6030              :   /* Walk the tree once to find local labels.  */
    6031      1105468 :   memset (&wi, 0, sizeof (wi));
    6032      1105468 :   hash_set<tree> visited;
    6033      1105468 :   wi.info = &id;
    6034      1105468 :   wi.pset = &visited;
    6035      1105468 :   walk_gimple_seq (seq, mark_local_labels_stmt, NULL, &wi);
    6036              : 
    6037      1105468 :   copy = gimple_seq_copy (seq);
    6038              : 
    6039              :   /* Walk the copy, remapping decls.  */
    6040      1105468 :   memset (&wi, 0, sizeof (wi));
    6041      1105468 :   wi.info = &id;
    6042      1105468 :   walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, &wi);
    6043              : 
    6044              :   /* Clean up.  */
    6045      2210936 :   delete id.decl_map;
    6046      1105468 :   if (id.debug_map)
    6047            0 :     delete id.debug_map;
    6048      1105468 :   if (id.dependence_map)
    6049              :     {
    6050            0 :       delete id.dependence_map;
    6051            0 :       id.dependence_map = NULL;
    6052              :     }
    6053              : 
    6054      1105468 :   return copy;
    6055      1105468 : }
    6056              : 
    6057              : 
    6058              : /* Declare the variables created by the inliner.  Add all the variables in
    6059              :    VARS to BIND_EXPR.  */
    6060              : 
    6061              : static void
    6062      7011885 : declare_inline_vars (tree block, tree vars)
    6063              : {
    6064      7011885 :   tree t;
    6065     17250569 :   for (t = vars; t; t = DECL_CHAIN (t))
    6066              :     {
    6067     10238684 :       DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
    6068     10238684 :       gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t));
    6069     10238684 :       add_local_decl (cfun, t);
    6070              :     }
    6071              : 
    6072      7011885 :   if (block)
    6073      6928263 :     BLOCK_VARS (block) = chainon (BLOCK_VARS (block), vars);
    6074      7011885 : }
    6075              : 
    6076              : /* Copy NODE (which must be a DECL).  The DECL originally was in the FROM_FN,
    6077              :    but now it will be in the TO_FN.  PARM_TO_VAR means enable PARM_DECL to
    6078              :    VAR_DECL translation.  */
    6079              : 
    6080              : tree
    6081    138618439 : copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy)
    6082              : {
    6083              :   /* Don't generate debug information for the copy if we wouldn't have
    6084              :      generated it for the copy either.  */
    6085    138618439 :   DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (decl);
    6086    138618439 :   DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl);
    6087              : 
    6088              :   /* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what
    6089              :      declaration inspired this copy.  */
    6090    171900405 :   DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl);
    6091              : 
    6092              :   /* The new variable/label has no RTL, yet.  */
    6093    138618439 :   if (HAS_RTL_P (copy)
    6094    138618439 :       && !TREE_STATIC (copy) && !DECL_EXTERNAL (copy))
    6095    138612903 :     SET_DECL_RTL (copy, 0);
    6096              :   /* For vector typed decls make sure to update DECL_MODE according
    6097              :      to the new function context.  */
    6098    138618439 :   if (VECTOR_TYPE_P (TREE_TYPE (copy)))
    6099       336492 :     SET_DECL_MODE (copy, TYPE_MODE (TREE_TYPE (copy)));
    6100              : 
    6101              :   /* These args would always appear unused, if not for this.  */
    6102    138618439 :   TREE_USED (copy) = 1;
    6103              : 
    6104              :   /* Set the context for the new declaration.  */
    6105    138618439 :   if (!DECL_CONTEXT (decl))
    6106              :     /* Globals stay global.  */
    6107              :     ;
    6108    138618288 :   else if (DECL_CONTEXT (decl) != id->src_fn)
    6109              :     /* Things that weren't in the scope of the function we're inlining
    6110              :        from aren't in the scope we're inlining to, either.  */
    6111              :     ;
    6112    138615556 :   else if (TREE_STATIC (decl))
    6113              :     /* Function-scoped static variables should stay in the original
    6114              :        function.  */
    6115              :     ;
    6116              :   else
    6117              :     {
    6118              :       /* Ordinary automatic local variables are now in the scope of the
    6119              :          new function.  */
    6120    138612022 :       DECL_CONTEXT (copy) = id->dst_fn;
    6121    138612022 :       if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
    6122              :         {
    6123            0 :           if (!lookup_attribute ("omp simt private", DECL_ATTRIBUTES (copy)))
    6124            0 :             DECL_ATTRIBUTES (copy)
    6125            0 :               = tree_cons (get_identifier ("omp simt private"), NULL,
    6126            0 :                            DECL_ATTRIBUTES (copy));
    6127            0 :           id->dst_simt_vars->safe_push (copy);
    6128              :         }
    6129              :     }
    6130              : 
    6131    138618439 :   return copy;
    6132              : }
    6133              : 
    6134              : /* Create a new VAR_DECL that is identical in all respect to DECL except that
    6135              :    DECL can be either a VAR_DECL, a PARM_DECL or RESULT_DECL.  The original
    6136              :    DECL must come from ID->src_fn and the copy will be part of ID->dst_fn.  */
    6137              : 
    6138              : tree
    6139      8140715 : copy_decl_to_var (tree decl, copy_body_data *id)
    6140              : {
    6141      8140715 :   tree copy, type;
    6142              : 
    6143      8140715 :   gcc_assert (TREE_CODE (decl) == PARM_DECL
    6144              :               || TREE_CODE (decl) == RESULT_DECL);
    6145              : 
    6146      8140715 :   type = TREE_TYPE (decl);
    6147              : 
    6148      8140715 :   copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
    6149      8140715 :                      VAR_DECL, DECL_NAME (decl), type);
    6150      8140715 :   if (DECL_PT_UID_SET_P (decl))
    6151          289 :     SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
    6152      8140715 :   TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
    6153      8140715 :   TREE_READONLY (copy) = TREE_READONLY (decl);
    6154      8140715 :   TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
    6155      8140715 :   DECL_NOT_GIMPLE_REG_P (copy) = DECL_NOT_GIMPLE_REG_P (decl);
    6156      8140715 :   DECL_BY_REFERENCE (copy) = DECL_BY_REFERENCE (decl);
    6157              : 
    6158      8140715 :   return copy_decl_for_dup_finish (id, decl, copy);
    6159              : }
    6160              : 
    6161              : /* Like copy_decl_to_var, but create a return slot object instead of a
    6162              :    pointer variable for return by invisible reference.  */
    6163              : 
    6164              : static tree
    6165      2112245 : copy_result_decl_to_var (tree decl, copy_body_data *id)
    6166              : {
    6167      2112245 :   tree copy, type;
    6168              : 
    6169      2112245 :   gcc_assert (TREE_CODE (decl) == PARM_DECL
    6170              :               || TREE_CODE (decl) == RESULT_DECL);
    6171              : 
    6172      2112245 :   type = TREE_TYPE (decl);
    6173      2112245 :   if (DECL_BY_REFERENCE (decl))
    6174          141 :     type = TREE_TYPE (type);
    6175              : 
    6176      2112245 :   copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
    6177      2112245 :                      VAR_DECL, DECL_NAME (decl), type);
    6178      2112245 :   if (DECL_PT_UID_SET_P (decl))
    6179            0 :     SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
    6180      2112245 :   TREE_READONLY (copy) = TREE_READONLY (decl);
    6181      2112245 :   TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
    6182      2112245 :   if (!DECL_BY_REFERENCE (decl))
    6183              :     {
    6184      2112104 :       TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
    6185      2112104 :       DECL_NOT_GIMPLE_REG_P (copy)
    6186      4224208 :         = (DECL_NOT_GIMPLE_REG_P (decl)
    6187              :            /* RESULT_DECLs are treated special by needs_to_live_in_memory,
    6188              :               mirror that to the created VAR_DECL.  */
    6189      2112104 :            || (TREE_CODE (decl) == RESULT_DECL
    6190      2111937 :                && aggregate_value_p (decl, id->src_fn)));
    6191              :     }
    6192              : 
    6193      2112245 :   return copy_decl_for_dup_finish (id, decl, copy);
    6194              : }
    6195              : 
    6196              : tree
    6197    128365236 : copy_decl_no_change (tree decl, copy_body_data *id)
    6198              : {
    6199    128365236 :   tree copy;
    6200              : 
    6201    128365236 :   copy = copy_node (decl);
    6202              : 
    6203              :   /* The COPY is not abstract; it will be generated in DST_FN.  */
    6204    128365236 :   DECL_ABSTRACT_P (copy) = false;
    6205    128365236 :   lang_hooks.dup_lang_specific_decl (copy);
    6206              : 
    6207              :   /* TREE_ADDRESSABLE isn't used to indicate that a label's address has
    6208              :      been taken; it's for internal bookkeeping in expand_goto_internal.  */
    6209    128365236 :   if (TREE_CODE (copy) == LABEL_DECL)
    6210              :     {
    6211      1695906 :       TREE_ADDRESSABLE (copy) = 0;
    6212      1695906 :       LABEL_DECL_UID (copy) = -1;
    6213              :     }
    6214              : 
    6215    128365236 :   return copy_decl_for_dup_finish (id, decl, copy);
    6216              : }
    6217              : 
    6218              : static tree
    6219     24577016 : copy_decl_maybe_to_var (tree decl, copy_body_data *id)
    6220              : {
    6221     24577016 :   if (TREE_CODE (decl) == PARM_DECL || TREE_CODE (decl) == RESULT_DECL)
    6222        14276 :     return copy_decl_to_var (decl, id);
    6223              :   else
    6224     24562740 :     return copy_decl_no_change (decl, id);
    6225              : }
    6226              : 
    6227              : /* Return a copy of the function's argument tree without any modifications.  */
    6228              : 
    6229              : static tree
    6230        80424 : copy_arguments_nochange (tree orig_parm, copy_body_data * id)
    6231              : {
    6232        80424 :   tree arg, *parg;
    6233        80424 :   tree new_parm = NULL;
    6234              : 
    6235        80424 :   parg = &new_parm;
    6236       242887 :   for (arg = orig_parm; arg; arg = DECL_CHAIN (arg))
    6237              :     {
    6238       162463 :       tree new_tree = remap_decl (arg, id);
    6239       162463 :       if (TREE_CODE (new_tree) != PARM_DECL)
    6240         3880 :         new_tree = id->copy_decl (arg, id);
    6241       162463 :       lang_hooks.dup_lang_specific_decl (new_tree);
    6242       162463 :       *parg = new_tree;
    6243       162463 :       parg = &DECL_CHAIN (new_tree);
    6244              :     }
    6245        80424 :   return new_parm;
    6246              : }
    6247              : 
    6248              : /* Return a copy of the function's static chain.  */
    6249              : static tree
    6250          984 : copy_static_chain (tree static_chain, copy_body_data * id)
    6251              : {
    6252          984 :   tree *chain_copy, *pvar;
    6253              : 
    6254          984 :   chain_copy = &static_chain;
    6255         1968 :   for (pvar = chain_copy; *pvar; pvar = &DECL_CHAIN (*pvar))
    6256              :     {
    6257          984 :       tree new_tree = remap_decl (*pvar, id);
    6258          984 :       lang_hooks.dup_lang_specific_decl (new_tree);
    6259          984 :       DECL_CHAIN (new_tree) = DECL_CHAIN (*pvar);
    6260          984 :       *pvar = new_tree;
    6261              :     }
    6262          984 :   return static_chain;
    6263              : }
    6264              : 
    6265              : /* Return true if the function is allowed to be versioned.
    6266              :    This is a guard for the versioning functionality.  */
    6267              : 
    6268              : bool
    6269     12887955 : tree_versionable_function_p (tree fndecl)
    6270              : {
    6271     12887955 :   return (!lookup_attribute ("noclone", DECL_ATTRIBUTES (fndecl))
    6272     25496258 :           && copy_forbidden (DECL_STRUCT_FUNCTION (fndecl)) == NULL);
    6273              : }
    6274              : 
    6275              : /* Update clone info after duplication.  */
    6276              : 
    6277              : static void
    6278       253174 : update_clone_info (copy_body_data * id)
    6279              : {
    6280       253174 :   struct cgraph_node *this_node = id->dst_node;
    6281       253174 :   if (!this_node->clones)
    6282              :     return;
    6283       644312 :   for (cgraph_node *node = this_node->clones; node != this_node;)
    6284              :     {
    6285              :       /* First update replace maps to match the new body.  */
    6286       559304 :       clone_info *info = clone_info::get (node);
    6287       559304 :       if (info && info->tree_map)
    6288              :         {
    6289              :           unsigned int i;
    6290            0 :           for (i = 0; i < vec_safe_length (info->tree_map); i++)
    6291              :             {
    6292            0 :               struct ipa_replace_map *replace_info;
    6293            0 :               replace_info = (*info->tree_map)[i];
    6294            0 :               walk_tree (&replace_info->new_tree, copy_tree_body_r, id, NULL);
    6295              :             }
    6296              :         }
    6297              : 
    6298       559304 :       if (node->clones)
    6299              :         node = node->clones;
    6300       534311 :       else if (node->next_sibling_clone)
    6301              :         node = node->next_sibling_clone;
    6302              :       else
    6303              :         {
    6304       207550 :           while (node != id->dst_node && !node->next_sibling_clone)
    6305       110001 :             node = node->clone_of;
    6306        97549 :           if (node != id->dst_node)
    6307        12541 :             node = node->next_sibling_clone;
    6308              :         }
    6309              :     }
    6310              : }
    6311              : 
    6312              : /* Create a copy of a function's tree.
    6313              :    OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes
    6314              :    of the original function and the new copied function
    6315              :    respectively.  In case we want to replace a DECL
    6316              :    tree with another tree while duplicating the function's
    6317              :    body, TREE_MAP represents the mapping between these
    6318              :    trees. If UPDATE_CLONES is set, the call_stmt fields
    6319              :    of edges of clones of the function will be updated.
    6320              : 
    6321              :    If non-NULL PARAM_ADJUSTMENTS determines how function prototype (i.e. the
    6322              :    function parameters and return value) should be modified).
    6323              :    If non-NULL BLOCKS_TO_COPY determine what basic blocks to copy.
    6324              :    If non_NULL NEW_ENTRY determine new entry BB of the clone.
    6325              : */
    6326              : void
    6327       253174 : tree_function_versioning (tree old_decl, tree new_decl,
    6328              :                           vec<ipa_replace_map *, va_gc> *tree_map,
    6329              :                           ipa_param_adjustments *param_adjustments,
    6330              :                           bool update_clones, bitmap blocks_to_copy,
    6331              :                           basic_block new_entry)
    6332              : {
    6333       253174 :   struct cgraph_node *old_version_node;
    6334       253174 :   struct cgraph_node *new_version_node;
    6335       253174 :   copy_body_data id;
    6336       253174 :   tree p;
    6337       253174 :   unsigned i;
    6338       253174 :   struct ipa_replace_map *replace_info;
    6339       253174 :   basic_block old_entry_block, bb;
    6340       253174 :   auto_vec<gimple *, 10> init_stmts;
    6341       253174 :   tree vars = NULL_TREE;
    6342              : 
    6343              :   /* We can get called recursively from expand_call_inline via clone
    6344              :      materialization.  While expand_call_inline maintains input_location
    6345              :      we cannot tolerate it to leak into the materialized clone.  */
    6346       253174 :   location_t saved_location = input_location;
    6347       253174 :   input_location = UNKNOWN_LOCATION;
    6348              : 
    6349       253174 :   gcc_assert (TREE_CODE (old_decl) == FUNCTION_DECL
    6350              :               && TREE_CODE (new_decl) == FUNCTION_DECL);
    6351       253174 :   DECL_POSSIBLY_INLINED (old_decl) = 1;
    6352              : 
    6353       253174 :   old_version_node = cgraph_node::get (old_decl);
    6354       253174 :   gcc_checking_assert (old_version_node);
    6355       253174 :   new_version_node = cgraph_node::get (new_decl);
    6356       253174 :   gcc_checking_assert (new_version_node);
    6357              : 
    6358              :   /* Copy over debug args.  */
    6359       253174 :   if (DECL_HAS_DEBUG_ARGS_P (old_decl))
    6360              :     {
    6361         3961 :       vec<tree, va_gc> **new_debug_args, **old_debug_args;
    6362         3961 :       gcc_checking_assert (decl_debug_args_lookup (new_decl) == NULL);
    6363         3961 :       DECL_HAS_DEBUG_ARGS_P (new_decl) = 0;
    6364         3961 :       old_debug_args = decl_debug_args_lookup (old_decl);
    6365         3961 :       if (old_debug_args)
    6366              :         {
    6367         3961 :           new_debug_args = decl_debug_args_insert (new_decl);
    6368         7922 :           *new_debug_args = vec_safe_copy (*old_debug_args);
    6369              :         }
    6370              :     }
    6371              : 
    6372              :   /* Output the inlining info for this abstract function, since it has been
    6373              :      inlined.  If we don't do this now, we can lose the information about the
    6374              :      variables in the function when the blocks get blown away as soon as we
    6375              :      remove the cgraph node.  */
    6376       253174 :   (*debug_hooks->outlining_inline_function) (old_decl);
    6377              : 
    6378       253174 :   DECL_ARTIFICIAL (new_decl) = 1;
    6379       458109 :   DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl);
    6380       458109 :   if (DECL_ORIGIN (old_decl) == old_decl)
    6381       224547 :     old_version_node->used_as_abstract_origin = true;
    6382       253174 :   DECL_FUNCTION_PERSONALITY (new_decl) = DECL_FUNCTION_PERSONALITY (old_decl);
    6383              : 
    6384              :   /* Prepare the data structures for the tree copy.  */
    6385       253174 :   memset (&id, 0, sizeof (id));
    6386              : 
    6387              :   /* Generate a new name for the new version. */
    6388       253174 :   id.statements_to_fold = new hash_set<gimple *>;
    6389              : 
    6390       253174 :   id.decl_map = new hash_map<tree, tree>;
    6391       253174 :   id.debug_map = NULL;
    6392       253174 :   id.src_fn = old_decl;
    6393       253174 :   id.dst_fn = new_decl;
    6394       253174 :   id.src_node = old_version_node;
    6395       253174 :   id.dst_node = new_version_node;
    6396       253174 :   id.src_cfun = DECL_STRUCT_FUNCTION (old_decl);
    6397       253174 :   id.blocks_to_copy = blocks_to_copy;
    6398              : 
    6399       253174 :   id.copy_decl = copy_decl_no_change;
    6400       253174 :   id.transform_call_graph_edges
    6401       253174 :     = update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE;
    6402       253174 :   id.transform_new_cfg = true;
    6403       253174 :   id.transform_return_to_modify = false;
    6404       253174 :   id.transform_parameter = false;
    6405              : 
    6406       253174 :   old_entry_block = ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (old_decl));
    6407       253174 :   DECL_RESULT (new_decl) = DECL_RESULT (old_decl);
    6408       253174 :   DECL_ARGUMENTS (new_decl) = DECL_ARGUMENTS (old_decl);
    6409       253174 :   initialize_cfun (new_decl, old_decl,
    6410       253174 :                    new_entry ? new_entry->count : old_entry_block->count);
    6411       253174 :   new_version_node->has_omp_variant_constructs
    6412       253174 :     = old_version_node->has_omp_variant_constructs;
    6413       253174 :   if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
    6414       253174 :     DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
    6415       253174 :       = id.src_cfun->gimple_df->ipa_pta;
    6416              : 
    6417              :   /* Copy the function's static chain.  */
    6418       253174 :   p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl;
    6419       253174 :   if (p)
    6420         1968 :     DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl
    6421          984 :       = copy_static_chain (p, &id);
    6422              : 
    6423       253174 :   auto_vec<int, 16> new_param_indices;
    6424       253174 :   clone_info *info = clone_info::get (old_version_node);
    6425       260075 :   ipa_param_adjustments *old_param_adjustments
    6426       253174 :     = info ? info->param_adjustments : NULL;
    6427         6901 :   if (old_param_adjustments)
    6428         6812 :     old_param_adjustments->get_updated_indices (&new_param_indices);
    6429              : 
    6430              :   /* If there's a tree_map, prepare for substitution.  */
    6431       253174 :   if (tree_map)
    6432        39394 :     for (i = 0; i < tree_map->length (); i++)
    6433              :       {
    6434        24913 :         gimple *init;
    6435        24913 :         replace_info = (*tree_map)[i];
    6436              : 
    6437        24913 :         int p = replace_info->parm_num;
    6438        24913 :         if (old_param_adjustments)
    6439            0 :           p = new_param_indices[p];
    6440              : 
    6441        24913 :         tree parm;
    6442        71801 :         for (parm = DECL_ARGUMENTS (old_decl); p;
    6443        46888 :              parm = DECL_CHAIN (parm))
    6444        46888 :           p--;
    6445        24913 :         gcc_assert (parm);
    6446        24913 :         init = setup_one_parameter (&id, parm, replace_info->new_tree,
    6447              :                                     id.src_fn, NULL, &vars);
    6448        24913 :         if (init)
    6449         7310 :           init_stmts.safe_push (init);
    6450              :       }
    6451              : 
    6452       253174 :   ipa_param_body_adjustments *param_body_adjs = NULL;
    6453       253174 :   if (param_adjustments)
    6454              :     {
    6455       333312 :       param_body_adjs = new ipa_param_body_adjustments (param_adjustments,
    6456              :                                                         new_decl, old_decl,
    6457       166656 :                                                         &id, &vars, tree_map);
    6458       166656 :       id.param_body_adjs = param_body_adjs;
    6459       166656 :       DECL_ARGUMENTS (new_decl) = param_body_adjs->get_new_param_chain ();
    6460              :     }
    6461        86518 :   else if (DECL_ARGUMENTS (old_decl) != NULL_TREE)
    6462       160848 :     DECL_ARGUMENTS (new_decl)
    6463        80424 :       = copy_arguments_nochange (DECL_ARGUMENTS (old_decl), &id);
    6464              : 
    6465       253174 :   DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), &id);
    6466       253174 :   BLOCK_SUPERCONTEXT (DECL_INITIAL (new_decl)) = new_decl;
    6467              : 
    6468       253174 :   declare_inline_vars (DECL_INITIAL (new_decl), vars);
    6469              : 
    6470       253174 :   if (!vec_safe_is_empty (DECL_STRUCT_FUNCTION (old_decl)->local_decls))
    6471              :     /* Add local vars.  */
    6472       151868 :     add_local_variables (DECL_STRUCT_FUNCTION (old_decl), cfun, &id);
    6473              : 
    6474       253174 :   if (DECL_RESULT (old_decl) == NULL_TREE)
    6475              :     ;
    6476       166656 :   else if (param_adjustments && param_adjustments->m_skip_return
    6477       306099 :            && !VOID_TYPE_P (TREE_TYPE (DECL_RESULT (old_decl))))
    6478              :     {
    6479        45501 :       tree resdecl_repl = copy_result_decl_to_var (DECL_RESULT (old_decl),
    6480              :                                                    &id);
    6481        45501 :       declare_inline_vars (NULL, resdecl_repl);
    6482        45501 :       if (DECL_BY_REFERENCE (DECL_RESULT (old_decl)))
    6483          141 :         resdecl_repl = build_fold_addr_expr (resdecl_repl);
    6484        45501 :       insert_decl_map (&id, DECL_RESULT (old_decl), resdecl_repl);
    6485              : 
    6486        91002 :       DECL_RESULT (new_decl)
    6487        45501 :         = build_decl (DECL_SOURCE_LOCATION (DECL_RESULT (old_decl)),
    6488              :                       RESULT_DECL, NULL_TREE, void_type_node);
    6489        45501 :       DECL_CONTEXT (DECL_RESULT (new_decl)) = new_decl;
    6490        45501 :       DECL_IS_MALLOC (new_decl) = false;
    6491        45501 :       cfun->returns_struct = 0;
    6492        45501 :       cfun->returns_pcc_struct = 0;
    6493              :     }
    6494              :   else
    6495              :     {
    6496       207673 :       tree old_name;
    6497       207673 :       DECL_RESULT (new_decl) = remap_decl (DECL_RESULT (old_decl), &id);
    6498       207673 :       lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl));
    6499       415346 :       if (gimple_in_ssa_p (id.src_cfun)
    6500       207673 :           && DECL_BY_REFERENCE (DECL_RESULT (old_decl))
    6501         4884 :           && (old_name = ssa_default_def (id.src_cfun, DECL_RESULT (old_decl))))
    6502              :         {
    6503         4865 :           tree new_name = make_ssa_name (DECL_RESULT (new_decl));
    6504         4865 :           insert_decl_map (&id, old_name, new_name);
    6505         4865 :           SSA_NAME_DEF_STMT (new_name) = gimple_build_nop ();
    6506         4865 :           set_ssa_default_def (cfun, DECL_RESULT (new_decl), new_name);
    6507              :         }
    6508              :     }
    6509              : 
    6510              :   /* Set up the destination functions loop tree.  */
    6511       253174 :   if (loops_for_fn (DECL_STRUCT_FUNCTION (old_decl)) != NULL)
    6512              :     {
    6513       253174 :       cfun->curr_properties &= ~PROP_loops;
    6514       253174 :       loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
    6515       253174 :       cfun->curr_properties |= PROP_loops;
    6516              :     }
    6517              : 
    6518              :   /* Copy the Function's body.  */
    6519       253174 :   copy_body (&id, ENTRY_BLOCK_PTR_FOR_FN (cfun), EXIT_BLOCK_PTR_FOR_FN (cfun),
    6520              :              new_entry);
    6521              : 
    6522              :   /* Renumber the lexical scoping (non-code) blocks consecutively.  */
    6523       253174 :   number_blocks (new_decl);
    6524              : 
    6525              :   /* We want to create the BB unconditionally, so that the addition of
    6526              :      debug stmts doesn't affect BB count, which may in the end cause
    6527              :      codegen differences.  */
    6528       253174 :   bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    6529       513658 :   while (init_stmts.length ())
    6530         7310 :     insert_init_stmt (&id, bb, init_stmts.pop ());
    6531       253174 :   if (param_body_adjs)
    6532       166656 :     param_body_adjs->append_init_stmts (bb);
    6533       253174 :   update_clone_info (&id);
    6534              : 
    6535              :   /* Remap the nonlocal_goto_save_area, if any.  */
    6536       253174 :   if (cfun->nonlocal_goto_save_area)
    6537              :     {
    6538            0 :       struct walk_stmt_info wi;
    6539              : 
    6540            0 :       memset (&wi, 0, sizeof (wi));
    6541            0 :       wi.info = &id;
    6542            0 :       walk_tree (&cfun->nonlocal_goto_save_area, remap_gimple_op_r, &wi, NULL);
    6543              :     }
    6544              : 
    6545              :   /* Clean up.  */
    6546       506348 :   delete id.decl_map;
    6547       253174 :   if (id.debug_map)
    6548         1248 :     delete id.debug_map;
    6549       253174 :   free_dominance_info (CDI_DOMINATORS);
    6550       253174 :   free_dominance_info (CDI_POST_DOMINATORS);
    6551              : 
    6552       253174 :   update_max_bb_count ();
    6553       253174 :   fold_marked_statements (0, id.statements_to_fold);
    6554       506348 :   delete id.statements_to_fold;
    6555       253174 :   delete_unreachable_blocks_update_callgraph (id.dst_node, update_clones);
    6556       253174 :   if (id.dst_node->definition)
    6557       248667 :     cgraph_edge::rebuild_references ();
    6558       253174 :   if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
    6559              :     {
    6560       253174 :       calculate_dominance_info (CDI_DOMINATORS);
    6561       253174 :       fix_loop_structure (NULL);
    6562              :     }
    6563       253174 :   update_ssa (TODO_update_ssa);
    6564              : 
    6565              :   /* After partial cloning we need to rescale frequencies, so they are
    6566              :      within proper range in the cloned function.  */
    6567       253174 :   if (new_entry)
    6568              :     {
    6569        52605 :       struct cgraph_edge *e;
    6570        52605 :       rebuild_frequencies ();
    6571              : 
    6572        52605 :       new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
    6573       236967 :       for (e = new_version_node->callees; e; e = e->next_callee)
    6574              :         {
    6575       184362 :           basic_block bb = gimple_bb (e->call_stmt);
    6576       184362 :           e->count = bb->count;
    6577              :         }
    6578        57615 :       for (e = new_version_node->indirect_calls; e; e = e->next_callee)
    6579              :         {
    6580         5010 :           basic_block bb = gimple_bb (e->call_stmt);
    6581         5010 :           e->count = bb->count;
    6582              :         }
    6583              :     }
    6584              : 
    6585       253174 :   if (param_body_adjs && MAY_HAVE_DEBUG_BIND_STMTS)
    6586              :     {
    6587       142690 :       vec<tree, va_gc> **debug_args = NULL;
    6588       142690 :       unsigned int len = 0;
    6589       142690 :       unsigned reset_len = param_body_adjs->m_reset_debug_decls.length ();
    6590              : 
    6591       254869 :       for (i = 0; i < reset_len; i++)
    6592              :         {
    6593       112179 :           tree parm = param_body_adjs->m_reset_debug_decls[i];
    6594       112179 :           gcc_assert (is_gimple_reg (parm));
    6595       112179 :           tree ddecl;
    6596              : 
    6597       112179 :           if (debug_args == NULL)
    6598              :             {
    6599        85236 :               debug_args = decl_debug_args_insert (new_decl);
    6600        85236 :               len = vec_safe_length (*debug_args);
    6601              :             }
    6602       112179 :           ddecl = build_debug_expr_decl (TREE_TYPE (parm));
    6603              :           /* FIXME: Is setting the mode really necessary? */
    6604       112179 :           SET_DECL_MODE (ddecl, DECL_MODE (parm));
    6605       112179 :           vec_safe_push (*debug_args, DECL_ORIGIN (parm));
    6606       112179 :           vec_safe_push (*debug_args, ddecl);
    6607              :         }
    6608       142690 :       if (debug_args != NULL)
    6609              :         {
    6610              :           /* On the callee side, add
    6611              :              DEBUG D#Y s=> parm
    6612              :              DEBUG var => D#Y
    6613              :              stmts to the first bb where var is a VAR_DECL created for the
    6614              :              optimized away parameter in DECL_INITIAL block.  This hints
    6615              :              in the debug info that var (whole DECL_ORIGIN is the parm
    6616              :              PARM_DECL) is optimized away, but could be looked up at the
    6617              :              call site as value of D#X there.  */
    6618        85236 :           gimple_stmt_iterator cgsi
    6619        85236 :             = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    6620        85236 :           gimple *def_temp;
    6621        85236 :           tree var = vars;
    6622       170472 :           i = vec_safe_length (*debug_args);
    6623       112179 :           do
    6624              :             {
    6625       112179 :               tree vexpr = NULL_TREE;
    6626       112179 :               i -= 2;
    6627       112179 :               while (var != NULL_TREE
    6628       148442 :                      && DECL_ABSTRACT_ORIGIN (var) != (**debug_args)[i])
    6629        36263 :                 var = TREE_CHAIN (var);
    6630       112179 :               if (var == NULL_TREE)
    6631              :                 break;
    6632       112179 :               tree parm = (**debug_args)[i];
    6633       112179 :               if (tree parm_ddef = ssa_default_def (id.src_cfun, parm))
    6634       149910 :                 if (tree *d
    6635        74955 :                     = param_body_adjs->m_dead_ssa_debug_equiv.get (parm_ddef))
    6636        30908 :                   vexpr = *d;
    6637       112179 :               if (!vexpr)
    6638              :                 {
    6639        81271 :                   vexpr = build_debug_expr_decl (TREE_TYPE (parm));
    6640              :                   /* FIXME: Is setting the mode really necessary? */
    6641        81271 :                   SET_DECL_MODE (vexpr, DECL_MODE (parm));
    6642              :                 }
    6643       112179 :               def_temp = gimple_build_debug_bind (var, vexpr, NULL);
    6644       112179 :               gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
    6645       112179 :               def_temp = gimple_build_debug_source_bind (vexpr, parm, NULL);
    6646       112179 :               gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
    6647              :             }
    6648       112179 :           while (i > len);
    6649              :         }
    6650              :     }
    6651       166656 :   delete param_body_adjs;
    6652       253174 :   free_dominance_info (CDI_DOMINATORS);
    6653       253174 :   free_dominance_info (CDI_POST_DOMINATORS);
    6654              : 
    6655       253174 :   gcc_assert (!id.debug_stmts.exists ());
    6656       253174 :   pop_cfun ();
    6657       253174 :   input_location = saved_location;
    6658       253174 :   return;
    6659       253174 : }
    6660              : 
    6661              : /* EXP is CALL_EXPR present in a GENERIC expression tree.  Try to integrate
    6662              :    the callee and return the inlined body on success.  */
    6663              : 
    6664              : tree
    6665            0 : maybe_inline_call_in_expr (tree exp)
    6666              : {
    6667            0 :   tree fn = get_callee_fndecl (exp);
    6668              : 
    6669              :   /* We can only try to inline "const" functions.  */
    6670            0 :   if (fn && TREE_READONLY (fn) && DECL_SAVED_TREE (fn))
    6671              :     {
    6672            0 :       call_expr_arg_iterator iter;
    6673            0 :       copy_body_data id;
    6674            0 :       tree param, arg, t;
    6675            0 :       hash_map<tree, tree> decl_map;
    6676              : 
    6677              :       /* Remap the parameters.  */
    6678            0 :       for (param = DECL_ARGUMENTS (fn), arg = first_call_expr_arg (exp, &iter);
    6679            0 :            param;
    6680            0 :            param = DECL_CHAIN (param), arg = next_call_expr_arg (&iter))
    6681            0 :         decl_map.put (param, arg);
    6682              : 
    6683            0 :       memset (&id, 0, sizeof (id));
    6684            0 :       id.src_fn = fn;
    6685            0 :       id.dst_fn = current_function_decl;
    6686            0 :       id.src_cfun = DECL_STRUCT_FUNCTION (fn);
    6687            0 :       id.decl_map = &decl_map;
    6688              : 
    6689            0 :       id.copy_decl = copy_decl_no_change;
    6690            0 :       id.transform_call_graph_edges = CB_CGE_DUPLICATE;
    6691            0 :       id.transform_new_cfg = false;
    6692            0 :       id.transform_return_to_modify = true;
    6693            0 :       id.transform_parameter = true;
    6694              : 
    6695              :       /* Make sure not to unshare trees behind the front-end's back
    6696              :          since front-end specific mechanisms may rely on sharing.  */
    6697            0 :       id.regimplify = false;
    6698            0 :       id.do_not_unshare = true;
    6699              : 
    6700              :       /* We're not inside any EH region.  */
    6701            0 :       id.eh_lp_nr = 0;
    6702              : 
    6703            0 :       t = copy_tree_body (&id);
    6704              : 
    6705              :       /* We can only return something suitable for use in a GENERIC
    6706              :          expression tree.  */
    6707            0 :       if (TREE_CODE (t) == MODIFY_EXPR)
    6708            0 :         return TREE_OPERAND (t, 1);
    6709            0 :     }
    6710              : 
    6711              :    return NULL_TREE;
    6712              : }
    6713              : 
    6714              : /* Duplicate a type, fields and all.  */
    6715              : 
    6716              : tree
    6717           63 : build_duplicate_type (tree type)
    6718              : {
    6719           63 :   struct copy_body_data id;
    6720              : 
    6721           63 :   memset (&id, 0, sizeof (id));
    6722           63 :   id.src_fn = current_function_decl;
    6723           63 :   id.dst_fn = current_function_decl;
    6724           63 :   id.src_cfun = cfun;
    6725           63 :   id.decl_map = new hash_map<tree, tree>;
    6726           63 :   id.debug_map = NULL;
    6727           63 :   id.copy_decl = copy_decl_no_change;
    6728              : 
    6729           63 :   type = remap_type_1 (type, &id);
    6730              : 
    6731          126 :   delete id.decl_map;
    6732           63 :   if (id.debug_map)
    6733            0 :     delete id.debug_map;
    6734              : 
    6735           63 :   TYPE_CANONICAL (type) = type;
    6736              : 
    6737           63 :   return type;
    6738              : }
    6739              : 
    6740              : /* Unshare the entire DECL_SAVED_TREE of FN and return the remapped
    6741              :    parameters and RESULT_DECL in PARMS and RESULT.  Used by C++ constexpr
    6742              :    evaluation.  */
    6743              : 
    6744              : tree
    6745     31020400 : copy_fn (tree fn, tree& parms, tree& result)
    6746              : {
    6747     31020400 :   copy_body_data id;
    6748     31020400 :   tree param;
    6749     31020400 :   hash_map<tree, tree> decl_map;
    6750              : 
    6751     31020400 :   tree *p = &parms;
    6752     31020400 :   *p = NULL_TREE;
    6753              : 
    6754     31020400 :   memset (&id, 0, sizeof (id));
    6755     31020400 :   id.src_fn = fn;
    6756     31020400 :   id.dst_fn = current_function_decl;
    6757     31020400 :   id.src_cfun = DECL_STRUCT_FUNCTION (fn);
    6758     31020400 :   id.decl_map = &decl_map;
    6759              : 
    6760    130407729 :   id.copy_decl = [] (tree decl, copy_body_data *id)
    6761              :     {
    6762     99387329 :       if (TREE_CODE (decl) == TYPE_DECL || TREE_CODE (decl) == CONST_DECL)
    6763              :         /* Don't make copies of local types or injected enumerators,
    6764              :            the C++ constexpr evaluator doesn't need them and they
    6765              :            confuse modules streaming.  */
    6766              :         return decl;
    6767     97247238 :       return copy_decl_no_change (decl, id);
    6768              :     };
    6769     31020400 :   id.transform_call_graph_edges = CB_CGE_DUPLICATE;
    6770     31020400 :   id.transform_new_cfg = false;
    6771     31020400 :   id.transform_return_to_modify = false;
    6772     31020400 :   id.transform_parameter = true;
    6773              : 
    6774              :   /* Make sure not to unshare trees behind the front-end's back
    6775              :      since front-end specific mechanisms may rely on sharing.  */
    6776     31020400 :   id.regimplify = false;
    6777     31020400 :   id.do_not_unshare = true;
    6778     31020400 :   id.do_not_fold = true;
    6779              : 
    6780              :   /* We're not inside any EH region.  */
    6781     31020400 :   id.eh_lp_nr = 0;
    6782              : 
    6783              :   /* Remap the parameters and result and return them to the caller.  */
    6784     31020400 :   for (param = DECL_ARGUMENTS (fn);
    6785     74725434 :        param;
    6786     43705034 :        param = DECL_CHAIN (param))
    6787              :     {
    6788     43705034 :       *p = remap_decl (param, &id);
    6789     43705034 :       p = &DECL_CHAIN (*p);
    6790              :     }
    6791              : 
    6792     31020400 :   if (DECL_RESULT (fn))
    6793     31020400 :     result = remap_decl (DECL_RESULT (fn), &id);
    6794              :   else
    6795              :     result = NULL_TREE;
    6796              : 
    6797     31020400 :   return copy_tree_body (&id);
    6798     31020400 : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.