LCOV - code coverage report
Current view: top level - gcc - tree-cfg.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 82.4 % 5168 4256
Test Date: 2026-11-14 16:39:50 Functions: 90.4 % 209 189
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Control flow functions for trees.
       2              :    Copyright (C) 2001-2026 Free Software Foundation, Inc.
       3              :    Contributed by Diego Novillo <dnovillo@redhat.com>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify
       8              : it under the terms of the GNU General Public License as published by
       9              : the Free Software Foundation; either version 3, or (at your option)
      10              : any later version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful,
      13              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      14              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15              : GNU General Public License for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "target.h"
      26              : #include "rtl.h"
      27              : #include "tree.h"
      28              : #include "gimple.h"
      29              : #include "cfghooks.h"
      30              : #include "tree-pass.h"
      31              : #include "ssa.h"
      32              : #include "cgraph.h"
      33              : #include "gimple-pretty-print.h"
      34              : #include "diagnostic-core.h"
      35              : #include "fold-const.h"
      36              : #include "trans-mem.h"
      37              : #include "stor-layout.h"
      38              : #include "print-tree.h"
      39              : #include "cfganal.h"
      40              : #include "gimple-iterator.h"
      41              : #include "gimple-fold.h"
      42              : #include "tree-eh.h"
      43              : #include "gimplify-me.h"
      44              : #include "gimple-walk.h"
      45              : #include "tree-cfg.h"
      46              : #include "tree-ssa-loop-manip.h"
      47              : #include "tree-ssa-loop-niter.h"
      48              : #include "tree-into-ssa.h"
      49              : #include "tree-dfa.h"
      50              : #include "tree-ssa.h"
      51              : #include "except.h"
      52              : #include "cfgloop.h"
      53              : #include "tree-ssa-propagate.h"
      54              : #include "value-prof.h"
      55              : #include "tree-inline.h"
      56              : #include "tree-ssa-live.h"
      57              : #include "tree-ssa-dce.h"
      58              : #include "omp-general.h"
      59              : #include "omp-expand.h"
      60              : #include "tree-cfgcleanup.h"
      61              : #include "gimplify.h"
      62              : #include "internal-fn.h"
      63              : #include "attribs.h"
      64              : #include "selftest.h"
      65              : #include "opts.h"
      66              : #include "asan.h"
      67              : #include "profile.h"
      68              : #include "sreal.h"
      69              : #include "gcc-urlifier.h"
      70              : 
      71              : /* This file contains functions for building the Control Flow Graph (CFG)
      72              :    for a function tree.  */
      73              : 
      74              : /* Local declarations.  */
      75              : 
      76              : /* Initial capacity for the basic block array.  */
      77              : static const int initial_cfg_capacity = 20;
      78              : 
      79              : /* This hash table allows us to efficiently lookup all CASE_LABEL_EXPRs
      80              :    which use a particular edge.  The CASE_LABEL_EXPRs are chained together
      81              :    via their CASE_CHAIN field, which we clear after we're done with the
      82              :    hash table to prevent problems with duplication of GIMPLE_SWITCHes.
      83              : 
      84              :    Access to this list of CASE_LABEL_EXPRs allows us to efficiently
      85              :    update the case vector in response to edge redirections.
      86              : 
      87              :    Right now this table is set up and torn down at key points in the
      88              :    compilation process.  It would be nice if we could make the table
      89              :    more persistent.  The key is getting notification of changes to
      90              :    the CFG (particularly edge removal, creation and redirection).  */
      91              : 
      92              : static hash_map<edge, tree> *edge_to_cases;
      93              : 
      94              : /* If we record edge_to_cases, this bitmap will hold indexes
      95              :    of basic blocks that end in a GIMPLE_SWITCH which we touched
      96              :    due to edge manipulations.  */
      97              : 
      98              : static bitmap touched_switch_bbs;
      99              : 
     100              : /* OpenMP region idxs for blocks during cfg pass.  */
     101              : static vec<int> bb_to_omp_idx;
     102              : 
     103              : /* CFG statistics.  */
     104              : struct cfg_stats_d
     105              : {
     106              :   long num_merged_labels;
     107              : };
     108              : 
     109              : static struct cfg_stats_d cfg_stats;
     110              : 
     111              : /* Data to pass to replace_block_vars_by_duplicates_1.  */
     112              : struct replace_decls_d
     113              : {
     114              :   hash_map<tree, tree> *vars_map;
     115              :   tree to_context;
     116              : };
     117              : 
     118              : /* Basic blocks and flowgraphs.  */
     119              : static void make_blocks (gimple_seq);
     120              : 
     121              : /* Edges.  */
     122              : static void make_edges (void);
     123              : static void assign_discriminators (void);
     124              : static void make_cond_expr_edges (basic_block);
     125              : static void make_gimple_switch_edges (gswitch *, basic_block);
     126              : static bool make_goto_expr_edges (basic_block);
     127              : static void make_gimple_asm_edges (basic_block);
     128              : static edge gimple_redirect_edge_and_branch (edge, basic_block);
     129              : static edge gimple_try_redirect_by_replacing_jump (edge, basic_block);
     130              : 
     131              : /* Various helpers.  */
     132              : static inline bool stmt_starts_bb_p (gimple *, gimple *);
     133              : static bool gimple_verify_flow_info (void);
     134              : static void gimple_make_forwarder_block (edge);
     135              : static bool verify_gimple_transaction (gtransaction *);
     136              : static bool call_can_make_abnormal_goto (gimple *);
     137              : static bool gimple_empty_block_p (basic_block);
     138              : 
     139              : /* Flowgraph optimization and cleanup.  */
     140              : static void gimple_merge_blocks (basic_block, basic_block);
     141              : static bool gimple_can_merge_blocks_p (basic_block, basic_block);
     142              : static void remove_bb (basic_block);
     143              : static edge find_taken_edge_computed_goto (basic_block, tree);
     144              : static edge find_taken_edge_cond_expr (const gcond *, tree);
     145              : 
     146              : void
     147      3414781 : init_empty_tree_cfg_for_function (struct function *fn)
     148              : {
     149              :   /* Initialize the basic block array.  */
     150      3414781 :   init_flow (fn);
     151      3414781 :   profile_status_for_fn (fn) = PROFILE_ABSENT;
     152      3414781 :   n_basic_blocks_for_fn (fn) = NUM_FIXED_BLOCKS;
     153      3414781 :   last_basic_block_for_fn (fn) = NUM_FIXED_BLOCKS;
     154      3414781 :   vec_safe_grow_cleared (basic_block_info_for_fn (fn),
     155              :                          initial_cfg_capacity, true);
     156              : 
     157              :   /* Build a mapping of labels to their associated blocks.  */
     158      3414781 :   vec_safe_grow_cleared (label_to_block_map_for_fn (fn),
     159              :                          initial_cfg_capacity, true);
     160              : 
     161      3414781 :   SET_BASIC_BLOCK_FOR_FN (fn, ENTRY_BLOCK, ENTRY_BLOCK_PTR_FOR_FN (fn));
     162      3414781 :   SET_BASIC_BLOCK_FOR_FN (fn, EXIT_BLOCK, EXIT_BLOCK_PTR_FOR_FN (fn));
     163              : 
     164      3414781 :   ENTRY_BLOCK_PTR_FOR_FN (fn)->next_bb
     165      3414781 :     = EXIT_BLOCK_PTR_FOR_FN (fn);
     166      3414781 :   EXIT_BLOCK_PTR_FOR_FN (fn)->prev_bb
     167      3414781 :     = ENTRY_BLOCK_PTR_FOR_FN (fn);
     168      3414781 : }
     169              : 
     170              : void
     171      3328842 : init_empty_tree_cfg (void)
     172              : {
     173      3328842 :   init_empty_tree_cfg_for_function (cfun);
     174      3328842 : }
     175              : 
     176              : /*---------------------------------------------------------------------------
     177              :                               Create basic blocks
     178              : ---------------------------------------------------------------------------*/
     179              : 
     180              : /* Entry point to the CFG builder for trees.  SEQ is the sequence of
     181              :    statements to be added to the flowgraph.  */
     182              : 
     183              : static void
     184      3010153 : build_gimple_cfg (gimple_seq seq)
     185              : {
     186              :   /* Register specific gimple functions.  */
     187      3010153 :   gimple_register_cfg_hooks ();
     188              : 
     189      3010153 :   memset ((void *) &cfg_stats, 0, sizeof (cfg_stats));
     190              : 
     191      3010153 :   init_empty_tree_cfg ();
     192              : 
     193      3010153 :   make_blocks (seq);
     194              : 
     195              :   /* Make sure there is always at least one block, even if it's empty.  */
     196      3010153 :   if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
     197            0 :     create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (cfun));
     198              : 
     199              :   /* Adjust the size of the array.  */
     200      3010153 :   if (basic_block_info_for_fn (cfun)->length ()
     201      3010153 :       < (size_t) n_basic_blocks_for_fn (cfun))
     202            0 :     vec_safe_grow_cleared (basic_block_info_for_fn (cfun),
     203              :                            n_basic_blocks_for_fn (cfun));
     204              : 
     205              :   /* To speed up statement iterator walks, we first purge dead labels.  */
     206      3010153 :   cleanup_dead_labels ();
     207              : 
     208              :   /* Group case nodes to reduce the number of edges.
     209              :      We do this after cleaning up dead labels because otherwise we miss
     210              :      a lot of obvious case merging opportunities.  */
     211      3010153 :   group_case_labels ();
     212              : 
     213              :   /* Create the edges of the flowgraph.  */
     214      3010153 :   make_edges ();
     215      3010153 :   assign_discriminators ();
     216      3010153 :   cleanup_dead_labels ();
     217      3010153 : }
     218              : 
     219              : /* Look for ANNOTATE calls with loop annotation kind in BB; if found, remove
     220              :    them and propagate the information to LOOP.  We assume that the annotations
     221              :    come immediately before the condition in BB, if any.  */
     222              : 
     223              : static void
     224      1589123 : replace_loop_annotate_in_block (basic_block bb, class loop *loop)
     225              : {
     226      1589123 :   gimple_stmt_iterator gsi = gsi_last_bb (bb);
     227      1589123 :   gimple *stmt = gsi_stmt (gsi);
     228              : 
     229      1589123 :   if (!(stmt && gimple_code (stmt) == GIMPLE_COND))
     230       581690 :     return;
     231              : 
     232      1013238 :   for (gsi_prev_nondebug (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
     233              :     {
     234       552509 :       stmt = gsi_stmt (gsi);
     235       552509 :       if (gimple_code (stmt) != GIMPLE_CALL)
     236              :         break;
     237        58909 :       if (!gimple_call_internal_p (stmt)
     238        58909 :           || gimple_call_internal_fn (stmt) != IFN_ANNOTATE)
     239              :         break;
     240              : 
     241         5750 :       switch ((annot_expr_kind) tree_to_shwi (gimple_call_arg (stmt, 1)))
     242              :         {
     243          480 :         case annot_expr_ivdep_kind:
     244          480 :           loop->safelen = INT_MAX;
     245          480 :           break;
     246         2025 :         case annot_expr_unroll_kind:
     247         2025 :           loop->unroll
     248         2025 :             = (unsigned short) tree_to_shwi (gimple_call_arg (stmt, 2));
     249         2025 :           cfun->has_unroll = true;
     250         2025 :           break;
     251         3178 :         case annot_expr_no_vector_kind:
     252         3178 :           loop->dont_vectorize = true;
     253         3178 :           break;
     254            3 :         case annot_expr_vector_kind:
     255            3 :           loop->force_vectorize = true;
     256            3 :           cfun->has_force_vectorize_loops = true;
     257            3 :           break;
     258            0 :         case annot_expr_parallel_kind:
     259            0 :           loop->can_be_parallel = true;
     260            0 :           loop->safelen = INT_MAX;
     261            0 :           break;
     262           64 :         case annot_expr_maybe_infinite_kind:
     263           64 :           loop->finite_p = false;
     264           64 :           break;
     265            0 :         default:
     266            0 :           gcc_unreachable ();
     267              :         }
     268              : 
     269         5750 :       stmt = gimple_build_assign (gimple_call_lhs (stmt),
     270              :                                   gimple_call_arg (stmt, 0));
     271         5750 :       gsi_replace (&gsi, stmt, true);
     272              :     }
     273              : }
     274              : 
     275              : /* Look for ANNOTATE calls with loop annotation kind; if found, remove
     276              :    them and propagate the information to the loop.  We assume that the
     277              :    annotations come immediately before the condition of the loop.  */
     278              : 
     279              : static void
     280      3010153 : replace_loop_annotate (void)
     281              : {
     282      3010153 :   basic_block bb;
     283      3010153 :   gimple_stmt_iterator gsi;
     284      3010153 :   gimple *stmt;
     285              : 
     286      9667367 :   for (auto loop : loops_list (cfun, 0))
     287              :     {
     288              :       /* Push the global flag_finite_loops state down to individual loops.  */
     289       636908 :       loop->finite_p = flag_finite_loops;
     290              : 
     291              :       /* Check all exit source blocks for annotations.  */
     292      3490845 :       for (auto e : get_loop_exit_edges (loop))
     293      2226031 :         replace_loop_annotate_in_block (e->src, loop);
     294      3010153 :     }
     295              : 
     296              :   /* Remove IFN_ANNOTATE.  Safeguard for the case loop->latch == NULL.  */
     297     21218961 :   FOR_EACH_BB_FN (bb, cfun)
     298              :     {
     299    122942531 :       for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
     300              :         {
     301     68404472 :           stmt = gsi_stmt (gsi);
     302     68404472 :           if (gimple_code (stmt) != GIMPLE_CALL)
     303     57651522 :             continue;
     304     10752950 :           if (!gimple_call_internal_p (stmt)
     305     10752950 :               || gimple_call_internal_fn (stmt) != IFN_ANNOTATE)
     306     10752950 :             continue;
     307              : 
     308            0 :           switch ((annot_expr_kind) tree_to_shwi (gimple_call_arg (stmt, 1)))
     309              :             {
     310            0 :             case annot_expr_ivdep_kind:
     311            0 :             case annot_expr_unroll_kind:
     312            0 :             case annot_expr_no_vector_kind:
     313            0 :             case annot_expr_vector_kind:
     314            0 :             case annot_expr_parallel_kind:
     315            0 :             case annot_expr_maybe_infinite_kind:
     316            0 :               break;
     317            0 :             default:
     318            0 :               gcc_unreachable ();
     319              :             }
     320              : 
     321            0 :           warning_at (gimple_location (stmt), 0, "ignoring loop annotation");
     322            0 :           stmt = gimple_build_assign (gimple_call_lhs (stmt),
     323              :                                       gimple_call_arg (stmt, 0));
     324            0 :           gsi_replace (&gsi, stmt, true);
     325              :         }
     326              :     }
     327      3010153 : }
     328              : 
     329              : static unsigned int
     330      3010153 : execute_build_cfg (void)
     331              : {
     332      3010153 :   gimple_seq body = gimple_body (current_function_decl);
     333              : 
     334      3010153 :   build_gimple_cfg (body);
     335      3010153 :   gimple_set_body (current_function_decl, NULL);
     336      3010153 :   if (dump_file && (dump_flags & TDF_DETAILS))
     337              :     {
     338            5 :       fprintf (dump_file, "Scope blocks:\n");
     339            5 :       dump_scope_blocks (dump_file, dump_flags);
     340              :     }
     341      3010153 :   cleanup_tree_cfg ();
     342              : 
     343      3010153 :   bb_to_omp_idx.release ();
     344              : 
     345      3010153 :   loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
     346      3010153 :   replace_loop_annotate ();
     347      3010153 :   return 0;
     348              : }
     349              : 
     350              : namespace {
     351              : 
     352              : const pass_data pass_data_build_cfg =
     353              : {
     354              :   GIMPLE_PASS, /* type */
     355              :   "cfg", /* name */
     356              :   OPTGROUP_NONE, /* optinfo_flags */
     357              :   TV_TREE_CFG, /* tv_id */
     358              :   PROP_gimple_leh, /* properties_required */
     359              :   ( PROP_cfg | PROP_loops ), /* properties_provided */
     360              :   0, /* properties_destroyed */
     361              :   0, /* todo_flags_start */
     362              :   0, /* todo_flags_finish */
     363              : };
     364              : 
     365              : class pass_build_cfg : public gimple_opt_pass
     366              : {
     367              : public:
     368       297196 :   pass_build_cfg (gcc::context *ctxt)
     369       594392 :     : gimple_opt_pass (pass_data_build_cfg, ctxt)
     370              :   {}
     371              : 
     372              :   /* opt_pass methods: */
     373      3010153 :   unsigned int execute (function *) final override
     374              :   {
     375      3010153 :     return execute_build_cfg ();
     376              :   }
     377              : 
     378              : }; // class pass_build_cfg
     379              : 
     380              : } // anon namespace
     381              : 
     382              : gimple_opt_pass *
     383       297196 : make_pass_build_cfg (gcc::context *ctxt)
     384              : {
     385       297196 :   return new pass_build_cfg (ctxt);
     386              : }
     387              : 
     388              : 
     389              : /* Return true if T is a computed goto.  */
     390              : 
     391              : bool
     392    688656119 : computed_goto_p (gimple *t)
     393              : {
     394    688656119 :   return (gimple_code (t) == GIMPLE_GOTO
     395    688656119 :           && TREE_CODE (gimple_goto_dest (t)) != LABEL_DECL);
     396              : }
     397              : 
     398              : /* Returns true if the sequence of statements STMTS only contains
     399              :    a call to __builtin_unreachable ().  */
     400              : 
     401              : bool
     402      3769972 : gimple_seq_unreachable_p (gimple_seq stmts)
     403              : {
     404      3769972 :   if (stmts == NULL
     405              :       /* Return false if -fsanitize=unreachable, we don't want to
     406              :          optimize away those calls, but rather turn them into
     407              :          __ubsan_handle_builtin_unreachable () or __builtin_trap ()
     408              :          later.  */
     409      3769972 :       || sanitize_flags_p (SANITIZE_UNREACHABLE))
     410              :     return false;
     411              : 
     412      3767291 :   gimple_stmt_iterator gsi = gsi_last (stmts);
     413              : 
     414      3767291 :   if (!gimple_call_builtin_p (gsi_stmt (gsi), BUILT_IN_UNREACHABLE))
     415              :     return false;
     416              : 
     417      1162065 :   for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
     418              :     {
     419         7119 :       gimple *stmt = gsi_stmt (gsi);
     420         7119 :       if (gimple_code (stmt) != GIMPLE_LABEL
     421         4298 :           && !is_gimple_debug (stmt)
     422         8207 :           && !gimple_clobber_p (stmt))
     423              :       return false;
     424              :     }
     425              :   return true;
     426              : }
     427              : 
     428              : /* Returns true for edge E where e->src ends with a GIMPLE_COND and
     429              :    the other edge points to a bb with just __builtin_unreachable ().
     430              :    I.e. return true for C->M edge in:
     431              :    <bb C>:
     432              :    ...
     433              :    if (something)
     434              :      goto <bb N>;
     435              :    else
     436              :      goto <bb M>;
     437              :    <bb N>:
     438              :    __builtin_unreachable ();
     439              :    <bb M>:  */
     440              : 
     441              : bool
     442     12358159 : assert_unreachable_fallthru_edge_p (edge e)
     443              : {
     444     12358159 :   basic_block pred_bb = e->src;
     445     24716318 :   if (safe_is_a <gcond *> (*gsi_last_bb (pred_bb)))
     446              :     {
     447     12358159 :       basic_block other_bb = EDGE_SUCC (pred_bb, 0)->dest;
     448     12358159 :       if (other_bb == e->dest)
     449      6631228 :         other_bb = EDGE_SUCC (pred_bb, 1)->dest;
     450     12358159 :       if (EDGE_COUNT (other_bb->succs) == 0)
     451      3784516 :         return gimple_seq_unreachable_p (bb_seq (other_bb));
     452              :     }
     453              :   return false;
     454              : }
     455              : 
     456              : 
     457              : /* Initialize GF_CALL_CTRL_ALTERING flag, which indicates the call
     458              :    could alter control flow except via eh. We initialize the flag at
     459              :    CFG build time and only ever clear it later.  */
     460              : 
     461              : static void
     462     11447012 : gimple_call_initialize_ctrl_altering (gimple *stmt)
     463              : {
     464     11447012 :   int flags = gimple_call_flags (stmt);
     465              : 
     466              :   /* A call alters control flow if it can make an abnormal goto.  */
     467     11447012 :   if (call_can_make_abnormal_goto (stmt)
     468              :       /* A call also alters control flow if it does not return.  */
     469     11439599 :       || flags & ECF_NORETURN
     470              :       /* TM ending statements have backedges out of the transaction.
     471              :          Return true so we split the basic block containing them.
     472              :          Note that the TM_BUILTIN test is merely an optimization.  */
     473      9754697 :       || ((flags & ECF_TM_BUILTIN)
     474          985 :           && is_tm_ending_fndecl (gimple_call_fndecl (stmt)))
     475              :       /* BUILT_IN_RETURN call is same as return statement.  */
     476      9753719 :       || gimple_call_builtin_p (stmt, BUILT_IN_RETURN)
     477              :       /* IFN_UNIQUE should be the last insn, to make checking for it
     478              :          as cheap as possible.  */
     479     21200731 :       || (gimple_call_internal_p (stmt)
     480       530689 :           && gimple_call_internal_unique_p (stmt)))
     481      1779086 :     gimple_call_set_ctrl_altering (stmt, true);
     482              :   else
     483      9667926 :     gimple_call_set_ctrl_altering (stmt, false);
     484     11447012 : }
     485              : 
     486              : 
     487              : /* Insert SEQ after BB and build a flowgraph.  */
     488              : 
     489              : static basic_block
     490      3062539 : make_blocks_1 (gimple_seq seq, basic_block bb)
     491              : {
     492      3062539 :   gimple_stmt_iterator i = gsi_start (seq);
     493      3062539 :   gimple *stmt = NULL;
     494      3062539 :   gimple *prev_stmt = NULL;
     495      3062539 :   bool start_new_block = true;
     496      3062539 :   bool first_stmt_of_seq = true;
     497      3062539 :   bool suppress_coverage_p = false;
     498              : 
     499     97156786 :   while (!gsi_end_p (i))
     500              :     {
     501              :       /* PREV_STMT should only be set to a debug stmt if the debug
     502              :          stmt is before nondebug stmts.  Once stmt reaches a nondebug
     503              :          nonlabel, prev_stmt will be set to it, so that
     504              :          stmt_starts_bb_p will know to start a new block if a label is
     505              :          found.  However, if stmt was a label after debug stmts only,
     506              :          keep the label in prev_stmt even if we find further debug
     507              :          stmts, for there may be other labels after them, and they
     508              :          should land in the same block.  */
     509     94094247 :       if (!prev_stmt || !stmt || !is_gimple_debug (stmt))
     510              :         prev_stmt = stmt;
     511     94094247 :       stmt = gsi_stmt (i);
     512              : 
     513     94094247 :       if (stmt && is_gimple_call (stmt))
     514     11447012 :         gimple_call_initialize_ctrl_altering (stmt);
     515              : 
     516     94094247 :       suppress_coverage_p = in_pragma_suppress_coverage_p (stmt,
     517              :                                                            suppress_coverage_p);
     518     94094247 :       if (suppress_coverage_p && !coverage_suppressed_p (bb))
     519              :         start_new_block = true;
     520     94093880 :       else if (!suppress_coverage_p && coverage_suppressed_p (bb))
     521              :         start_new_block = true;
     522              : 
     523              :       /* If the statement starts a new basic block or if we have determined
     524              :          in a previous pass that we need to create a new block for STMT, do
     525              :          so now.  */
     526     94093545 :       if (start_new_block || stmt_starts_bb_p (stmt, prev_stmt))
     527              :         {
     528     22488763 :           if (!first_stmt_of_seq)
     529     19426224 :             gsi_split_seq_before (&i, &seq);
     530     22488763 :           bb = create_basic_block (seq, bb);
     531              : 
     532     22488763 :           start_new_block = false;
     533     22488763 :           prev_stmt = NULL;
     534              :         }
     535              : 
     536              :       /* Now add STMT to BB and create the subgraphs for special statement
     537              :          codes.  */
     538     94094247 :       gimple_set_bb (stmt, bb);
     539              : 
     540     94094247 :       if (suppress_coverage_p)
     541         2096 :         suppress_coverage (bb);
     542              : 
     543              :       /* If STMT is a basic block terminator, set START_NEW_BLOCK for the
     544              :          next iteration.  */
     545     94094247 :       if (stmt_ends_bb_p (stmt))
     546              :         {
     547              :           /* If the stmt can make abnormal goto use a new temporary
     548              :              for the assignment to the LHS.  This makes sure the old value
     549              :              of the LHS is available on the abnormal edge.  Otherwise
     550              :              we will end up with overlapping life-ranges for abnormal
     551              :              SSA names.  */
     552     20504116 :           if (gimple_has_lhs (stmt)
     553      1678105 :               && stmt_can_make_abnormal_goto (stmt)
     554      3626196 :               && is_gimple_reg_type (TREE_TYPE (gimple_get_lhs (stmt))))
     555              :             {
     556         2530 :               tree lhs = gimple_get_lhs (stmt);
     557         2530 :               tree tmp = create_tmp_var (TREE_TYPE (lhs));
     558         2530 :               gimple *s = gimple_build_assign (lhs, tmp);
     559         2530 :               gimple_set_location (s, gimple_location (stmt));
     560         2530 :               gimple_set_block (s, gimple_block (stmt));
     561         2530 :               gimple_set_lhs (stmt, tmp);
     562         2530 :               gsi_insert_after (&i, s, GSI_SAME_STMT);
     563              :             }
     564              :           start_new_block = true;
     565              :         }
     566              : 
     567     94094247 :       gsi_next (&i);
     568     94094247 :       first_stmt_of_seq = false;
     569              :     }
     570      3062539 :   return bb;
     571              : }
     572              : 
     573              : /* Build a flowgraph for the sequence of stmts SEQ.  */
     574              : 
     575              : static void
     576      3010153 : make_blocks (gimple_seq seq)
     577              : {
     578              :   /* Look for debug markers right before labels, and move the debug
     579              :      stmts after the labels.  Accepting labels among debug markers
     580              :      adds no value, just complexity; if we wanted to annotate labels
     581              :      with view numbers (so sequencing among markers would matter) or
     582              :      somesuch, we're probably better off still moving the labels, but
     583              :      adding other debug annotations in their original positions or
     584              :      emitting nonbind or bind markers associated with the labels in
     585              :      the original position of the labels.
     586              : 
     587              :      Moving labels would probably be simpler, but we can't do that:
     588              :      moving labels assigns label ids to them, and doing so because of
     589              :      debug markers makes for -fcompare-debug and possibly even codegen
     590              :      differences.  So, we have to move the debug stmts instead.  To
     591              :      that end, we scan SEQ backwards, marking the position of the
     592              :      latest (earliest we find) label, and moving debug stmts that are
     593              :      not separated from it by nondebug nonlabel stmts after the
     594              :      label.  */
     595      3010153 :   if (MAY_HAVE_DEBUG_MARKER_STMTS)
     596              :     {
     597      1849072 :       gimple_stmt_iterator label = gsi_none ();
     598              : 
     599     55913631 :       for (gimple_stmt_iterator i = gsi_last (seq); !gsi_end_p (i); gsi_prev (&i))
     600              :         {
     601     50366415 :           gimple *stmt = gsi_stmt (i);
     602              : 
     603              :           /* If this is the first label we encounter (latest in SEQ)
     604              :              before nondebug stmts, record its position.  */
     605     50366415 :           if (is_a <glabel *> (stmt))
     606              :             {
     607      9467955 :               if (gsi_end_p (label))
     608      8799362 :                 label = i;
     609      9467955 :               continue;
     610              :             }
     611              : 
     612              :           /* Without a recorded label position to move debug stmts to,
     613              :              there's nothing to do.  */
     614     40898460 :           if (gsi_end_p (label))
     615     32081704 :             continue;
     616              : 
     617              :           /* Move the debug stmt at I after LABEL.  */
     618      8816756 :           if (is_gimple_debug (stmt))
     619              :             {
     620        22709 :               gcc_assert (gimple_debug_nonbind_marker_p (stmt));
     621              :               /* As STMT is removed, I advances to the stmt after
     622              :                  STMT, so the gsi_prev in the for "increment"
     623              :                  expression gets us to the stmt we're to visit after
     624              :                  STMT.  LABEL, however, would advance to the moved
     625              :                  stmt if we passed it to gsi_move_after, so pass it a
     626              :                  copy instead, so as to keep LABEL pointing to the
     627              :                  LABEL.  */
     628        22709 :               gimple_stmt_iterator copy = label;
     629        22709 :               gsi_move_after (&i, &copy);
     630        22709 :               continue;
     631        22709 :             }
     632              : 
     633              :           /* There aren't any (more?) debug stmts before label, so
     634              :              there isn't anything else to move after it.  */
     635              :           label = gsi_none ();
     636              :         }
     637              :     }
     638              : 
     639              :   /* If the function decl itself is in a #pragma GCC suppress_coverage block we
     640              :      insert a nop with the same location as the function so that we're
     641              :      guaranteed that the first block gets coverage suppressed and the function
     642              :      entry isn't counted.
     643              : 
     644              :      #pragma GCC suppress_coverage begin
     645              :      int foo (args) // count = suppressed
     646              :      {
     647              :      #pragma GCC suppress_coverage end
     648              :        ...
     649              :      }
     650              : 
     651              :      If it isn't and the first stmt is suppressed we insert a nop to ensure that
     652              :      the function entry is counted, but the first (real) stmt is not:
     653              : 
     654              :      int foo (args) // count = 1
     655              :      {
     656              :      #pragma GCC suppress_coverage begin
     657              :        bar (); // count = suppressed
     658              :      #pragma GCC suppress_coverage end
     659              :        ...
     660              :      }
     661              : 
     662              :      gimple_can_merge_blocks_p is aware of specific instruction.  */
     663      3010153 :   gimple_stmt_iterator gsi = gsi_start (seq);
     664      3010153 :   const location_t decl_loc = DECL_SOURCE_LOCATION (cfun->decl);
     665      3010153 :   if (location_in_pragma_suppress_coverage_p (decl_loc))
     666              :     {
     667           32 :       gimple *nop = gimple_build_nop ();
     668           32 :       gimple_set_location (nop, decl_loc);
     669           32 :       gsi_insert_seq_before (&gsi, nop, GSI_NEW_STMT);
     670              :     }
     671      3010121 :   else if (*gsi && in_pragma_suppress_coverage_p (*gsi, false))
     672              :     {
     673          103 :       gimple *nop = gimple_build_nop ();
     674          103 :       gsi_insert_seq_before (&gsi, nop, GSI_NEW_STMT);
     675              :     }
     676              : 
     677      3010153 :   make_blocks_1 (seq, ENTRY_BLOCK_PTR_FOR_FN (cfun));
     678      3010153 : }
     679              : 
     680              : /* Create and return a new empty basic block after bb AFTER.  */
     681              : 
     682              : static basic_block
     683     73859528 : create_bb (void *h, void *e, basic_block after)
     684              : {
     685     73859528 :   basic_block bb;
     686              : 
     687     73859528 :   gcc_assert (!e);
     688              : 
     689              :   /* Create and initialize a new basic block.  Since alloc_block uses
     690              :      GC allocation that clears memory to allocate a basic block, we do
     691              :      not have to clear the newly allocated basic block here.  */
     692     73859528 :   bb = alloc_block ();
     693              : 
     694     73859528 :   bb->index = last_basic_block_for_fn (cfun);
     695     73859528 :   bb->flags = BB_NEW;
     696     73859528 :   set_bb_seq (bb, h ? (gimple_seq) h : NULL);
     697              : 
     698              :   /* Add the new block to the linked list of blocks.  */
     699     73859528 :   link_block (bb, after);
     700              : 
     701              :   /* Grow the basic block array if needed.  */
     702     73859528 :   if ((size_t) last_basic_block_for_fn (cfun)
     703     73859528 :       == basic_block_info_for_fn (cfun)->length ())
     704     22411415 :     vec_safe_grow_cleared (basic_block_info_for_fn (cfun),
     705     22411415 :                            last_basic_block_for_fn (cfun) + 1);
     706              : 
     707              :   /* Add the newly created block to the array.  */
     708     73859528 :   SET_BASIC_BLOCK_FOR_FN (cfun, last_basic_block_for_fn (cfun), bb);
     709              : 
     710     73859528 :   n_basic_blocks_for_fn (cfun)++;
     711     73859528 :   last_basic_block_for_fn (cfun)++;
     712              : 
     713     73859528 :   return bb;
     714              : }
     715              : 
     716              : 
     717              : /*---------------------------------------------------------------------------
     718              :                                  Edge creation
     719              : ---------------------------------------------------------------------------*/
     720              : 
     721              : /* If basic block BB has an abnormal edge to a basic block
     722              :    containing IFN_ABNORMAL_DISPATCHER internal call, return
     723              :    that the dispatcher's basic block, otherwise return NULL.  */
     724              : 
     725              : basic_block
     726          665 : get_abnormal_succ_dispatcher (basic_block bb)
     727              : {
     728          665 :   edge e;
     729          665 :   edge_iterator ei;
     730              : 
     731         1409 :   FOR_EACH_EDGE (e, ei, bb->succs)
     732         1057 :     if ((e->flags & (EDGE_ABNORMAL | EDGE_EH)) == EDGE_ABNORMAL)
     733              :       {
     734          313 :         gimple_stmt_iterator gsi
     735          313 :           = gsi_start_nondebug_after_labels_bb (e->dest);
     736          313 :         gimple *g = gsi_stmt (gsi);
     737          313 :         if (g && gimple_call_internal_p (g, IFN_ABNORMAL_DISPATCHER))
     738          313 :           return e->dest;
     739              :       }
     740              :   return NULL;
     741              : }
     742              : 
     743              : /* Helper function for make_edges.  Create a basic block with
     744              :    with ABNORMAL_DISPATCHER internal call in it if needed, and
     745              :    create abnormal edges from BBS to it and from it to FOR_BB
     746              :    if COMPUTED_GOTO is false, otherwise factor the computed gotos.  */
     747              : 
     748              : static void
     749         5481 : handle_abnormal_edges (basic_block *dispatcher_bbs, basic_block for_bb,
     750              :                        auto_vec<basic_block> *bbs, bool computed_goto)
     751              : {
     752         5481 :   basic_block *dispatcher = dispatcher_bbs + (computed_goto ? 1 : 0);
     753         5481 :   unsigned int idx = 0;
     754         5481 :   basic_block bb;
     755         5481 :   bool inner = false;
     756              : 
     757         5481 :   if (!bb_to_omp_idx.is_empty ())
     758              :     {
     759           11 :       dispatcher = dispatcher_bbs + 2 * bb_to_omp_idx[for_bb->index];
     760           11 :       if (bb_to_omp_idx[for_bb->index] != 0)
     761            6 :         inner = true;
     762              :     }
     763              : 
     764              :   /* If the dispatcher has been created already, then there are basic
     765              :      blocks with abnormal edges to it, so just make a new edge to
     766              :      for_bb.  */
     767         5481 :   if (*dispatcher == NULL)
     768              :     {
     769              :       /* Check if there are any basic blocks that need to have
     770              :          abnormal edges to this dispatcher.  If there are none, return
     771              :          early.  */
     772         3563 :       if (bb_to_omp_idx.is_empty ())
     773              :         {
     774         3552 :           if (bbs->is_empty ())
     775         5481 :             return;
     776              :         }
     777              :       else
     778              :         {
     779           18 :           FOR_EACH_VEC_ELT (*bbs, idx, bb)
     780           18 :             if (bb_to_omp_idx[bb->index] == bb_to_omp_idx[for_bb->index])
     781              :               break;
     782           11 :           if (bb == NULL)
     783              :             return;
     784              :         }
     785              : 
     786              :       /* Create the dispatcher bb.  */
     787         2736 :       *dispatcher = create_basic_block (NULL, for_bb);
     788         2736 :       if (computed_goto)
     789              :         {
     790              :           /* Factor computed gotos into a common computed goto site.  Also
     791              :              record the location of that site so that we can un-factor the
     792              :              gotos after we have converted back to normal form.  */
     793          579 :           gimple_stmt_iterator gsi = gsi_start_bb (*dispatcher);
     794              : 
     795              :           /* Create the destination of the factored goto.  Each original
     796              :              computed goto will put its desired destination into this
     797              :              variable and jump to the label we create immediately below.  */
     798          579 :           tree var = create_tmp_var (ptr_type_node, "gotovar");
     799              : 
     800              :           /* Build a label for the new block which will contain the
     801              :              factored computed goto.  */
     802          579 :           tree factored_label_decl
     803          579 :             = create_artificial_label (UNKNOWN_LOCATION);
     804          579 :           gimple *factored_computed_goto_label
     805          579 :             = gimple_build_label (factored_label_decl);
     806          579 :           gsi_insert_after (&gsi, factored_computed_goto_label, GSI_NEW_STMT);
     807              : 
     808              :           /* Build our new computed goto.  */
     809          579 :           gimple *factored_computed_goto = gimple_build_goto (var);
     810          579 :           gsi_insert_after (&gsi, factored_computed_goto, GSI_NEW_STMT);
     811              : 
     812         2886 :           FOR_EACH_VEC_ELT (*bbs, idx, bb)
     813              :             {
     814         1149 :               if (!bb_to_omp_idx.is_empty ()
     815            0 :                   && bb_to_omp_idx[bb->index] != bb_to_omp_idx[for_bb->index])
     816            0 :                 continue;
     817              : 
     818         1149 :               gsi = gsi_last_bb (bb);
     819         1149 :               gimple *last = gsi_stmt (gsi);
     820              : 
     821         1149 :               gcc_assert (computed_goto_p (last));
     822              : 
     823              :               /* Copy the original computed goto's destination into VAR.  */
     824         1149 :               gimple *assignment
     825         1149 :                 = gimple_build_assign (var, gimple_goto_dest (last));
     826         1149 :               gsi_insert_before (&gsi, assignment, GSI_SAME_STMT);
     827              : 
     828         1149 :               edge e = make_edge (bb, *dispatcher, EDGE_FALLTHRU);
     829         1149 :               e->goto_locus = gimple_location (last);
     830         1149 :               gsi_remove (&gsi, true);
     831              :             }
     832              :         }
     833              :       else
     834              :         {
     835         2157 :           tree arg = inner ? boolean_true_node : boolean_false_node;
     836         2157 :           gcall *g = gimple_build_call_internal (IFN_ABNORMAL_DISPATCHER,
     837              :                                                  1, arg);
     838         2157 :           gimple_call_set_ctrl_altering (g, true);
     839         2157 :           gimple_stmt_iterator gsi = gsi_after_labels (*dispatcher);
     840         2157 :           gsi_insert_after (&gsi, g, GSI_NEW_STMT);
     841              : 
     842              :           /* Create predecessor edges of the dispatcher.  */
     843        13892 :           FOR_EACH_VEC_ELT (*bbs, idx, bb)
     844              :             {
     845         7421 :               if (!bb_to_omp_idx.is_empty ()
     846           44 :                   && bb_to_omp_idx[bb->index] != bb_to_omp_idx[for_bb->index])
     847           20 :                 continue;
     848         7401 :               make_edge (bb, *dispatcher, EDGE_ABNORMAL);
     849              :             }
     850              :         }
     851              :     }
     852              : 
     853         4654 :   make_edge (*dispatcher, for_bb, EDGE_ABNORMAL);
     854              : }
     855              : 
     856              : /* Creates outgoing edges for BB.  Returns 1 when it ends with an
     857              :    computed goto, returns 2 when it ends with a statement that
     858              :    might return to this function via an nonlocal goto, otherwise
     859              :    return 0.  Updates *PCUR_REGION with the OMP region this BB is in.  */
     860              : 
     861              : static int
     862     22488763 : make_edges_bb (basic_block bb, struct omp_region **pcur_region, int *pomp_index)
     863              : {
     864     22488763 :   gimple *last = *gsi_last_bb (bb);
     865     22488763 :   bool fallthru = false;
     866     22488763 :   int ret = 0;
     867              : 
     868     22488763 :   if (!last)
     869              :     return ret;
     870              : 
     871     22488763 :   switch (gimple_code (last))
     872              :     {
     873      6329979 :     case GIMPLE_GOTO:
     874      6329979 :       if (make_goto_expr_edges (bb))
     875              :         ret = 1;
     876              :       fallthru = false;
     877              :       break;
     878      3000347 :     case GIMPLE_RETURN:
     879      3000347 :       {
     880      3000347 :         edge e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
     881      3000347 :         e->goto_locus = gimple_location (last);
     882      3000347 :         fallthru = false;
     883              :       }
     884      3000347 :       break;
     885      5593360 :     case GIMPLE_COND:
     886      5593360 :       make_cond_expr_edges (bb);
     887      5593360 :       fallthru = false;
     888      5593360 :       break;
     889        54352 :     case GIMPLE_SWITCH:
     890        54352 :       make_gimple_switch_edges (as_a <gswitch *> (last), bb);
     891        54352 :       fallthru = false;
     892        54352 :       break;
     893       945985 :     case GIMPLE_RESX:
     894       945985 :       make_eh_edge (last);
     895       945985 :       fallthru = false;
     896       945985 :       break;
     897        44493 :     case GIMPLE_EH_DISPATCH:
     898        44493 :       fallthru = make_eh_dispatch_edges (as_a <geh_dispatch *> (last));
     899        44493 :       break;
     900              : 
     901      3965630 :     case GIMPLE_CALL:
     902              :       /* If this function receives a nonlocal goto, then we need to
     903              :          make edges from this call site to all the nonlocal goto
     904              :          handlers.  */
     905      3965630 :       if (stmt_can_make_abnormal_goto (last))
     906         7413 :         ret = 2;
     907              : 
     908              :       /* If this statement has reachable exception handlers, then
     909              :          create abnormal edges to them.  */
     910      3965630 :       make_eh_edge (last);
     911              : 
     912              :       /* BUILTIN_RETURN is really a return statement.  */
     913      3965630 :       if (gimple_call_builtin_p (last, BUILT_IN_RETURN))
     914              :         {
     915          368 :           make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
     916          368 :           fallthru = false;
     917              :         }
     918              :       /* Some calls are known not to return.  */
     919              :       else
     920      3965262 :         fallthru = !gimple_call_noreturn_p (last);
     921              :       break;
     922              : 
     923      2242989 :     case GIMPLE_ASSIGN:
     924              :       /* A GIMPLE_ASSIGN may throw internally and thus be considered
     925              :          control-altering.  */
     926      2242989 :       if (is_ctrl_altering_stmt (last))
     927       618556 :         make_eh_edge (last);
     928              :       fallthru = true;
     929              :       break;
     930              : 
     931         1628 :     case GIMPLE_ASM:
     932         1628 :       make_gimple_asm_edges (bb);
     933         1628 :       fallthru = true;
     934         1628 :       break;
     935              : 
     936       292441 :     CASE_GIMPLE_OMP:
     937       292441 :       fallthru = omp_make_gimple_edges (bb, pcur_region, pomp_index);
     938       292441 :       break;
     939              : 
     940          411 :     case GIMPLE_TRANSACTION:
     941          411 :       {
     942          411 :         gtransaction *txn = as_a <gtransaction *> (last);
     943          411 :         tree label1 = gimple_transaction_label_norm (txn);
     944          411 :         tree label2 = gimple_transaction_label_uninst (txn);
     945              : 
     946          411 :         if (label1)
     947          401 :           make_edge (bb, label_to_block (cfun, label1), EDGE_FALLTHRU);
     948          411 :         if (label2)
     949          397 :           make_edge (bb, label_to_block (cfun, label2),
     950          397 :                      EDGE_TM_UNINSTRUMENTED | (label1 ? 0 : EDGE_FALLTHRU));
     951              : 
     952          411 :         tree label3 = gimple_transaction_label_over (txn);
     953          411 :         if (gimple_transaction_subcode (txn)
     954              :             & (GTMA_HAVE_ABORT | GTMA_IS_OUTER))
     955           65 :           make_edge (bb, label_to_block (cfun, label3), EDGE_TM_ABORT);
     956              : 
     957              :         fallthru = false;
     958              :       }
     959              :       break;
     960              : 
     961        17148 :     default:
     962        17148 :       gcc_assert (!stmt_ends_bb_p (last));
     963              :       fallthru = true;
     964              :       break;
     965              :     }
     966              : 
     967     14516857 :   if (fallthru)
     968      4791331 :     make_edge (bb, bb->next_bb, EDGE_FALLTHRU);
     969              : 
     970              :   return ret;
     971              : }
     972              : 
     973              : /* Join all the blocks in the flowgraph.  */
     974              : 
     975              : static void
     976      3010153 : make_edges (void)
     977              : {
     978      3010153 :   basic_block bb;
     979      3010153 :   struct omp_region *cur_region = NULL;
     980      3010153 :   auto_vec<basic_block> ab_edge_goto;
     981      3010153 :   auto_vec<basic_block> ab_edge_call;
     982      3010153 :   int cur_omp_region_idx = 0;
     983              : 
     984              :   /* Create an edge from entry to the first block with executable
     985              :      statements in it.  */
     986      3010153 :   make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun),
     987      3010153 :              BASIC_BLOCK_FOR_FN (cfun, NUM_FIXED_BLOCKS),
     988              :              EDGE_FALLTHRU);
     989              : 
     990              :   /* Traverse the basic block array placing edges.  */
     991     25359239 :   FOR_EACH_BB_FN (bb, cfun)
     992              :     {
     993     22349086 :       int mer;
     994              : 
     995     22349086 :       if (!bb_to_omp_idx.is_empty ())
     996       731272 :         bb_to_omp_idx[bb->index] = cur_omp_region_idx;
     997              : 
     998     22349086 :       mer = make_edges_bb (bb, &cur_region, &cur_omp_region_idx);
     999     22349086 :       if (mer == 1)
    1000         1250 :         ab_edge_goto.safe_push (bb);
    1001     22347836 :       else if (mer == 2)
    1002         7413 :         ab_edge_call.safe_push (bb);
    1003              : 
    1004     22888151 :       if (cur_region && bb_to_omp_idx.is_empty ())
    1005        22573 :         bb_to_omp_idx.safe_grow_cleared (n_basic_blocks_for_fn (cfun), true);
    1006              :     }
    1007              : 
    1008              :   /* Computed gotos are hell to deal with, especially if there are
    1009              :      lots of them with a large number of destinations.  So we factor
    1010              :      them to a common computed goto location before we build the
    1011              :      edge list.  After we convert back to normal form, we will un-factor
    1012              :      the computed gotos since factoring introduces an unwanted jump.
    1013              :      For non-local gotos and abnormal edges from calls to calls that return
    1014              :      twice or forced labels, factor the abnormal edges too, by having all
    1015              :      abnormal edges from the calls go to a common artificial basic block
    1016              :      with ABNORMAL_DISPATCHER internal call and abnormal edges from that
    1017              :      basic block to all forced labels and calls returning twice.
    1018              :      We do this per-OpenMP structured block, because those regions
    1019              :      are guaranteed to be single entry single exit by the standard,
    1020              :      so it is not allowed to enter or exit such regions abnormally this way,
    1021              :      thus all computed gotos, non-local gotos and setjmp/longjmp calls
    1022              :      must not transfer control across SESE region boundaries.  */
    1023      3010153 :   if (!ab_edge_goto.is_empty () || !ab_edge_call.is_empty ())
    1024              :     {
    1025         2788 :       gimple_stmt_iterator gsi;
    1026         2788 :       basic_block dispatcher_bb_array[2] = { NULL, NULL };
    1027         2788 :       basic_block *dispatcher_bbs = dispatcher_bb_array;
    1028         2788 :       int count = n_basic_blocks_for_fn (cfun);
    1029              : 
    1030         2788 :       if (!bb_to_omp_idx.is_empty ())
    1031            9 :         dispatcher_bbs = XCNEWVEC (basic_block, 2 * count);
    1032              : 
    1033        32448 :       FOR_EACH_BB_FN (bb, cfun)
    1034              :         {
    1035        75412 :           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    1036              :             {
    1037        43958 :               glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (gsi));
    1038        16589 :               tree target;
    1039              : 
    1040        16589 :               if (!label_stmt)
    1041              :                 break;
    1042              : 
    1043        16589 :               target = gimple_label_label (label_stmt);
    1044              : 
    1045              :               /* Make an edge to every label block that has been marked as a
    1046              :                  potential target for a computed goto or a non-local goto.  */
    1047        16589 :               if (FORCED_LABEL (target))
    1048         2564 :                 handle_abnormal_edges (dispatcher_bbs, bb, &ab_edge_goto,
    1049              :                                        true);
    1050        16589 :               if (DECL_NONLOCAL (target))
    1051              :                 {
    1052          497 :                   handle_abnormal_edges (dispatcher_bbs, bb, &ab_edge_call,
    1053              :                                          false);
    1054          497 :                   break;
    1055              :                 }
    1056              :             }
    1057              : 
    1058        29660 :           if (!gsi_end_p (gsi) && is_gimple_debug (gsi_stmt (gsi)))
    1059         2352 :             gsi_next_nondebug (&gsi);
    1060        29660 :           if (!gsi_end_p (gsi))
    1061              :             {
    1062              :               /* Make an edge to every setjmp-like call.  */
    1063        27730 :               gimple *call_stmt = gsi_stmt (gsi);
    1064        27730 :               if (is_gimple_call (call_stmt)
    1065        27730 :                   && ((gimple_call_flags (call_stmt) & ECF_RETURNS_TWICE)
    1066         8381 :                       || gimple_call_builtin_p (call_stmt,
    1067              :                                                 BUILT_IN_SETJMP_RECEIVER)))
    1068         2420 :                 handle_abnormal_edges (dispatcher_bbs, bb, &ab_edge_call,
    1069              :                                        false);
    1070              :             }
    1071              :         }
    1072              : 
    1073         2797 :       if (!bb_to_omp_idx.is_empty ())
    1074            9 :         XDELETE (dispatcher_bbs);
    1075              :     }
    1076              : 
    1077      3010153 :   omp_free_regions ();
    1078      3010153 : }
    1079              : 
    1080              : /* Add SEQ after GSI.  Start new bb after GSI, and created further bbs as
    1081              :    needed.  Returns true if new bbs were created.
    1082              :    Note: This is transitional code, and should not be used for new code.  We
    1083              :    should be able to get rid of this by rewriting all target va-arg
    1084              :    gimplification hooks to use an interface gimple_build_cond_value as described
    1085              :    in https://gcc.gnu.org/ml/gcc-patches/2015-02/msg01194.html.  */
    1086              : 
    1087              : bool
    1088        52386 : gimple_find_sub_bbs (gimple_seq seq, gimple_stmt_iterator *gsi)
    1089              : {
    1090        52386 :   gimple *stmt = gsi_stmt (*gsi);
    1091        52386 :   basic_block bb = gimple_bb (stmt);
    1092        52386 :   basic_block lastbb, afterbb;
    1093        52386 :   int old_num_bbs = n_basic_blocks_for_fn (cfun);
    1094        52386 :   edge e;
    1095        52386 :   lastbb = make_blocks_1 (seq, bb);
    1096        52386 :   if (old_num_bbs == n_basic_blocks_for_fn (cfun))
    1097              :     return false;
    1098        52386 :   e = split_block (bb, stmt);
    1099              :   /* Move e->dest to come after the new basic blocks.  */
    1100        52386 :   afterbb = e->dest;
    1101        52386 :   unlink_block (afterbb);
    1102        52386 :   link_block (afterbb, lastbb);
    1103        52386 :   redirect_edge_succ (e, bb->next_bb);
    1104        52386 :   bb = bb->next_bb;
    1105       192063 :   while (bb != afterbb)
    1106              :     {
    1107       139677 :       struct omp_region *cur_region = NULL;
    1108       139677 :       profile_count cnt = profile_count::zero ();
    1109       139677 :       bool all = true;
    1110              : 
    1111       139677 :       int cur_omp_region_idx = 0;
    1112       139677 :       int mer = make_edges_bb (bb, &cur_region, &cur_omp_region_idx);
    1113       139677 :       gcc_assert (!mer && !cur_region);
    1114       139677 :       add_bb_to_loop (bb, afterbb->loop_father);
    1115              : 
    1116       139677 :       edge e;
    1117       139677 :       edge_iterator ei;
    1118       308710 :       FOR_EACH_EDGE (e, ei, bb->preds)
    1119              :         {
    1120       169033 :           if (e->count ().initialized_p ())
    1121        23553 :             cnt += e->count ();
    1122              :           else
    1123              :             all = false;
    1124              :         }
    1125       139677 :       tree_guess_outgoing_edge_probabilities (bb);
    1126       139677 :       if (all || profile_status_for_fn (cfun) == PROFILE_READ)
    1127        18905 :         bb->count = cnt;
    1128              : 
    1129       139677 :       bb = bb->next_bb;
    1130              :     }
    1131              :   return true;
    1132              : }
    1133              : 
    1134              : /* Auto-profile needs discriminator to distinguish statements with same line
    1135              :    number (file name is ignored) which are in different basic block.  This
    1136              :    map keeps track of current discriminator for a given line number.  */
    1137              : struct discrim_entry
    1138              : {
    1139              :   /* ID of basic block we saw line number last time.  */
    1140              :   unsigned int bb_id;
    1141              :   /* Discriminator we used.  */
    1142              :   unsigned int discrim;
    1143              : };
    1144              : 
    1145              : /* Return updated LOC with discriminator for use in basic block BB_ID.
    1146              :    MAP keeps track of current values.  */
    1147              : 
    1148              : location_t
    1149     75682060 : assign_discriminator (location_t loc, unsigned int bb_id,
    1150              :                       hash_map<int_hash <unsigned, -1U, -2U>,
    1151              :                                discrim_entry> &map)
    1152              : {
    1153     75682060 :   bool existed;
    1154     75682060 :   if ((unsigned) LOCATION_LINE (loc) >= -2U)
    1155              :     return loc;
    1156     75682058 :   discrim_entry &e
    1157     75682058 :     = map.get_or_insert ((unsigned) LOCATION_LINE (loc), &existed);
    1158     75682058 :   gcc_checking_assert (!has_discriminator (loc));
    1159     75682058 :   if (!existed)
    1160              :     {
    1161     14225975 :       e.bb_id = bb_id;
    1162     14225975 :       e.discrim = 0;
    1163     14225975 :       return loc;
    1164              :     }
    1165     61456083 :   if (e.bb_id != bb_id)
    1166              :     {
    1167     41282218 :       e.bb_id = bb_id;
    1168     41282218 :       e.discrim++;
    1169              :     }
    1170     61456083 :   if (e.discrim)
    1171     52091034 :     return location_with_discriminator (loc, e.discrim);
    1172              :   return loc;
    1173              : }
    1174              : 
    1175              : /* Assign discriminators to statement locations.  */
    1176              : 
    1177              : static void
    1178      3010153 : assign_discriminators (void)
    1179              : {
    1180      3010153 :   hash_map<int_hash <unsigned, -1U, -2U>, discrim_entry> map (13);
    1181      3010153 :   unsigned int bb_id = 0;
    1182      3010153 :   basic_block bb;
    1183     25361975 :   FOR_EACH_BB_FN (bb, cfun)
    1184              :     {
    1185     22351822 :       location_t prev_loc = UNKNOWN_LOCATION, prev_replacement = UNKNOWN_LOCATION;
    1186              :       /* Traverse the basic block, if two function calls within a basic block
    1187              :          are mapped to the same line, assign a new discriminator because a call
    1188              :          stmt could be a split point of a basic block.  */
    1189     44703644 :       for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    1190    107161797 :            !gsi_end_p (gsi); gsi_next (&gsi))
    1191              :         {
    1192     84809975 :           gimple *stmt = gsi_stmt (gsi);
    1193     84809975 :           location_t loc = gimple_location (stmt);
    1194     84809975 :           if (loc == UNKNOWN_LOCATION)
    1195      4645112 :             continue;
    1196     80164863 :           if (loc == prev_loc)
    1197     24037307 :             gimple_set_location (stmt, prev_replacement);
    1198              :           else
    1199              :             {
    1200     56127556 :               prev_loc = loc;
    1201     56127556 :               prev_replacement = assign_discriminator (loc, bb_id, map);
    1202    112255112 :               gimple_set_location (stmt, prev_replacement);
    1203              :             }
    1204              :           /* Break basic blocks after each call.  This is required so each
    1205              :              call site has unique discriminator.
    1206              :              More correctly, we can break after each statement that can possibly
    1207              :              terinate execution of the basic block, but for auto-profile this
    1208              :              precision is probably not useful.  */
    1209     80164863 :           if (gimple_code (stmt) == GIMPLE_CALL)
    1210              :             {
    1211     11445465 :               prev_loc = UNKNOWN_LOCATION;
    1212     11445465 :               bb_id++;
    1213              :             }
    1214              :         }
    1215              :       /* If basic block has multiple successors, consider every edge as a
    1216              :          separate block.  */
    1217     22351822 :       if (!single_succ_p (bb))
    1218     10024314 :         bb_id++;
    1219     92276749 :       for (edge e : bb->succs)
    1220              :         {
    1221     28890279 :           if (e->goto_locus != UNKNOWN_LOCATION)
    1222     19554504 :             e->goto_locus = assign_discriminator (e->goto_locus, bb_id, map);
    1223     28890279 :           for (gphi_iterator gpi = gsi_start_phis (bb);
    1224     28890279 :                !gsi_end_p (gpi); gsi_next (&gpi))
    1225              :             {
    1226            0 :               gphi *phi = gpi.phi ();
    1227            0 :               location_t phi_loc
    1228            0 :                 = gimple_phi_arg_location_from_edge (phi, e);
    1229            0 :               if (phi_loc == UNKNOWN_LOCATION)
    1230            0 :                 continue;
    1231            0 :               gimple_phi_arg_set_location
    1232            0 :                 (phi, e->dest_idx, assign_discriminator (phi_loc, bb_id, map));
    1233              :             }
    1234     28890279 :            bb_id++;
    1235              :         }
    1236     22351822 :       bb_id++;
    1237              :     }
    1238              : 
    1239      3010153 : }
    1240              : 
    1241              : /* Create the edges for a GIMPLE_COND starting at block BB.  */
    1242              : 
    1243              : static void
    1244      5593360 : make_cond_expr_edges (basic_block bb)
    1245              : {
    1246     11186720 :   gcond *entry = as_a <gcond *> (*gsi_last_bb (bb));
    1247      5593360 :   gimple *then_stmt, *else_stmt;
    1248      5593360 :   basic_block then_bb, else_bb;
    1249      5593360 :   tree then_label, else_label;
    1250      5593360 :   edge e;
    1251              : 
    1252      5593360 :   gcc_assert (entry);
    1253              : 
    1254              :   /* Entry basic blocks for each component.  */
    1255      5593360 :   then_label = gimple_cond_true_label (entry);
    1256      5593360 :   else_label = gimple_cond_false_label (entry);
    1257      5593360 :   then_bb = label_to_block (cfun, then_label);
    1258      5593360 :   else_bb = label_to_block (cfun, else_label);
    1259      5593360 :   then_stmt = first_stmt (then_bb);
    1260      5593360 :   else_stmt = first_stmt (else_bb);
    1261              : 
    1262      5593360 :   e = make_edge (bb, then_bb, EDGE_TRUE_VALUE);
    1263      5593360 :   e->goto_locus = gimple_location (then_stmt);
    1264      5593360 :   e = make_edge (bb, else_bb, EDGE_FALSE_VALUE);
    1265      5593360 :   if (e)
    1266      5587892 :     e->goto_locus = gimple_location (else_stmt);
    1267              : 
    1268              :   /* We do not need the labels anymore.  */
    1269      5593360 :   gimple_cond_set_true_label (entry, NULL_TREE);
    1270      5593360 :   gimple_cond_set_false_label (entry, NULL_TREE);
    1271      5593360 : }
    1272              : 
    1273              : 
    1274              : /* Called for each element in the hash table (P) as we delete the
    1275              :    edge to cases hash table.
    1276              : 
    1277              :    Clear all the CASE_CHAINs to prevent problems with copying of
    1278              :    SWITCH_EXPRs and structure sharing rules, then free the hash table
    1279              :    element.  */
    1280              : 
    1281              : bool
    1282      1173404 : edge_to_cases_cleanup (edge const &, tree const &value, void *)
    1283              : {
    1284      1173404 :   tree t, next;
    1285              : 
    1286      2501020 :   for (t = value; t; t = next)
    1287              :     {
    1288      1327616 :       next = CASE_CHAIN (t);
    1289      1327616 :       CASE_CHAIN (t) = NULL;
    1290              :     }
    1291              : 
    1292      1173404 :   return true;
    1293              : }
    1294              : 
    1295              : /* Start recording information mapping edges to case labels.  */
    1296              : 
    1297              : void
    1298     30515683 : start_recording_case_labels (void)
    1299              : {
    1300     30515683 :   gcc_assert (edge_to_cases == NULL);
    1301     30515683 :   edge_to_cases = new hash_map<edge, tree>;
    1302     30515683 :   touched_switch_bbs = BITMAP_ALLOC (NULL);
    1303     30515683 : }
    1304              : 
    1305              : /* Return nonzero if we are recording information for case labels.  */
    1306              : 
    1307              : static bool
    1308       416384 : recording_case_labels_p (void)
    1309              : {
    1310       416384 :   return (edge_to_cases != NULL);
    1311              : }
    1312              : 
    1313              : /* Stop recording information mapping edges to case labels and
    1314              :    remove any information we have recorded.  */
    1315              : void
    1316     30515683 : end_recording_case_labels (void)
    1317              : {
    1318     30515683 :   bitmap_iterator bi;
    1319     30515683 :   unsigned i;
    1320     31689087 :   edge_to_cases->traverse<void *, edge_to_cases_cleanup> (NULL);
    1321     61031366 :   delete edge_to_cases;
    1322     30515683 :   edge_to_cases = NULL;
    1323     30689600 :   EXECUTE_IF_SET_IN_BITMAP (touched_switch_bbs, 0, i, bi)
    1324              :     {
    1325       173917 :       basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
    1326       173917 :       if (bb)
    1327              :         {
    1328       517935 :           if (gswitch *stmt = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
    1329       171797 :             group_case_labels_stmt (stmt);
    1330              :         }
    1331              :     }
    1332     30515683 :   BITMAP_FREE (touched_switch_bbs);
    1333     30515683 : }
    1334              : 
    1335              : /* If we are inside a {start,end}_recording_cases block, then return
    1336              :    a chain of CASE_LABEL_EXPRs from T which reference E.
    1337              : 
    1338              :    Otherwise return NULL.  */
    1339              : 
    1340              : tree
    1341       416384 : get_cases_for_edge (edge e, gswitch *t)
    1342              : {
    1343       416384 :   tree *slot;
    1344       416384 :   size_t i, n;
    1345              : 
    1346              :   /* If we are not recording cases, then we do not have CASE_LABEL_EXPR
    1347              :      chains available.  Return NULL so the caller can detect this case.  */
    1348       416384 :   if (!recording_case_labels_p ())
    1349              :     return NULL;
    1350              : 
    1351       358006 :   slot = edge_to_cases->get (e);
    1352       358006 :   if (slot)
    1353       184237 :     return *slot;
    1354              : 
    1355              :   /* If we did not find E in the hash table, then this must be the first
    1356              :      time we have been queried for information about E & T.  Add all the
    1357              :      elements from T to the hash table then perform the query again.  */
    1358              : 
    1359       173769 :   n = gimple_switch_num_labels (t);
    1360      1492422 :   for (i = 0; i < n; i++)
    1361              :     {
    1362      1318653 :       tree elt = gimple_switch_label (t, i);
    1363      1318653 :       tree lab = CASE_LABEL (elt);
    1364      1318653 :       basic_block label_bb = label_to_block (cfun, lab);
    1365      1318653 :       edge this_edge = find_edge (e->src, label_bb);
    1366              : 
    1367              :       /* Add it to the chain of CASE_LABEL_EXPRs referencing E, or create
    1368              :          a new chain.  */
    1369      1318653 :       tree &s = edge_to_cases->get_or_insert (this_edge);
    1370      1318653 :       CASE_CHAIN (elt) = s;
    1371      1318653 :       s = elt;
    1372              :     }
    1373              : 
    1374       173769 :   return *edge_to_cases->get (e);
    1375              : }
    1376              : 
    1377              : /* Create the edges for a GIMPLE_SWITCH starting at block BB.  */
    1378              : 
    1379              : static void
    1380        54352 : make_gimple_switch_edges (gswitch *entry, basic_block bb)
    1381              : {
    1382        54352 :   size_t i, n;
    1383              : 
    1384        54352 :   n = gimple_switch_num_labels (entry);
    1385              : 
    1386       382760 :   for (i = 0; i < n; ++i)
    1387              :     {
    1388       328408 :       basic_block label_bb = gimple_switch_label_bb (cfun, entry, i);
    1389       328408 :       make_edge (bb, label_bb, 0);
    1390              :     }
    1391        54352 : }
    1392              : 
    1393              : 
    1394              : /* Return the basic block holding label DEST.  */
    1395              : 
    1396              : basic_block
    1397    475832834 : label_to_block (struct function *ifun, tree dest)
    1398              : {
    1399    475832834 :   int uid = LABEL_DECL_UID (dest);
    1400              : 
    1401              :   /* We would die hard when faced by an undefined label.  Emit a label to
    1402              :      the very first basic block.  This will hopefully make even the dataflow
    1403              :      and undefined variable warnings quite right.  */
    1404    475832834 :   if (seen_error () && uid < 0)
    1405              :     {
    1406           97 :       gimple_stmt_iterator gsi =
    1407           97 :         gsi_start_bb (BASIC_BLOCK_FOR_FN (cfun, NUM_FIXED_BLOCKS));
    1408           97 :       gimple *stmt;
    1409              : 
    1410           97 :       stmt = gimple_build_label (dest);
    1411           97 :       gsi_insert_before (&gsi, stmt, GSI_NEW_STMT);
    1412           97 :       uid = LABEL_DECL_UID (dest);
    1413              :     }
    1414    475832834 :   if (vec_safe_length (ifun->cfg->x_label_to_block_map) <= (unsigned int) uid)
    1415              :     return NULL;
    1416    475832834 :   return (*ifun->cfg->x_label_to_block_map)[uid];
    1417              : }
    1418              : 
    1419              : /* Create edges for a goto statement at block BB.  Returns true
    1420              :    if abnormal edges should be created.  */
    1421              : 
    1422              : static bool
    1423      6329979 : make_goto_expr_edges (basic_block bb)
    1424              : {
    1425      6329979 :   gimple_stmt_iterator last = gsi_last_bb (bb);
    1426      6329979 :   gimple *goto_t = gsi_stmt (last);
    1427              : 
    1428              :   /* A simple GOTO creates normal edges.  */
    1429      6329979 :   if (simple_goto_p (goto_t))
    1430              :     {
    1431      6328729 :       tree dest = gimple_goto_dest (goto_t);
    1432      6328729 :       basic_block label_bb = label_to_block (cfun, dest);
    1433      6328729 :       edge e = make_edge (bb, label_bb, EDGE_FALLTHRU);
    1434      6328729 :       e->goto_locus = gimple_location (goto_t);
    1435      6328729 :       gsi_remove (&last, true);
    1436      6328729 :       return false;
    1437              :     }
    1438              : 
    1439              :   /* A computed GOTO creates abnormal edges.  */
    1440              :   return true;
    1441              : }
    1442              : 
    1443              : /* Create edges for an asm statement with labels at block BB.  */
    1444              : 
    1445              : static void
    1446         1628 : make_gimple_asm_edges (basic_block bb)
    1447              : {
    1448         3256 :   gasm *stmt = as_a <gasm *> (*gsi_last_bb (bb));
    1449         1628 :   int i, n = gimple_asm_nlabels (stmt);
    1450              : 
    1451         2564 :   for (i = 0; i < n; ++i)
    1452              :     {
    1453          936 :       tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
    1454          936 :       basic_block label_bb = label_to_block (cfun, label);
    1455          936 :       make_edge (bb, label_bb, 0);
    1456              :     }
    1457         1628 : }
    1458              : 
    1459              : /*---------------------------------------------------------------------------
    1460              :                                Flowgraph analysis
    1461              : ---------------------------------------------------------------------------*/
    1462              : 
    1463              : /* Cleanup useless labels in basic blocks.  This is something we wish
    1464              :    to do early because it allows us to group case labels before creating
    1465              :    the edges for the CFG, and it speeds up block statement iterators in
    1466              :    all passes later on.
    1467              :    We rerun this pass after CFG is created, to get rid of the labels that
    1468              :    are no longer referenced.  After then we do not run it any more, since
    1469              :    (almost) no new labels should be created.  */
    1470              : 
    1471              : /* A map from basic block index to the leading label of that block.  */
    1472              : struct label_record
    1473              : {
    1474              :   /* The label.  */
    1475              :   tree label;
    1476              : 
    1477              :   /* True if the label is referenced from somewhere.  */
    1478              :   bool used;
    1479              : };
    1480              : 
    1481              : /* Given LABEL return the first label in the same basic block.  */
    1482              : 
    1483              : static tree
    1484     21122029 : main_block_label (tree label, label_record *label_for_bb)
    1485              : {
    1486     21122029 :   basic_block bb = label_to_block (cfun, label);
    1487     21122029 :   tree main_label = label_for_bb[bb->index].label;
    1488              : 
    1489              :   /* label_to_block possibly inserted undefined label into the chain.  */
    1490     21122029 :   if (!main_label)
    1491              :     {
    1492           88 :       label_for_bb[bb->index].label = label;
    1493           88 :       main_label = label;
    1494              :     }
    1495              : 
    1496     21122029 :   label_for_bb[bb->index].used = true;
    1497     21122029 :   return main_label;
    1498              : }
    1499              : 
    1500              : /* Clean up redundant labels within the exception tree.  */
    1501              : 
    1502              : static void
    1503      7538199 : cleanup_dead_labels_eh (label_record *label_for_bb)
    1504              : {
    1505      7538199 :   eh_landing_pad lp;
    1506      7538199 :   eh_region r;
    1507      7538199 :   tree lab;
    1508      7538199 :   int i;
    1509              : 
    1510      7538199 :   if (cfun->eh == NULL)
    1511      7538199 :     return;
    1512              : 
    1513     10930103 :   for (i = 1; vec_safe_iterate (cfun->eh->lp_array, i, &lp); ++i)
    1514      3391904 :     if (lp && lp->post_landing_pad)
    1515              :       {
    1516      2106580 :         lab = main_block_label (lp->post_landing_pad, label_for_bb);
    1517      2106580 :         if (lab != lp->post_landing_pad)
    1518              :           {
    1519            0 :             EH_LANDING_PAD_NR (lp->post_landing_pad) = 0;
    1520            0 :             lp->post_landing_pad = lab;
    1521            0 :             EH_LANDING_PAD_NR (lab) = lp->index;
    1522              :           }
    1523              :       }
    1524              : 
    1525     13225106 :   FOR_ALL_EH_REGION (r)
    1526      5686907 :     switch (r->type)
    1527              :       {
    1528              :       case ERT_CLEANUP:
    1529              :       case ERT_MUST_NOT_THROW:
    1530              :         break;
    1531              : 
    1532       142539 :       case ERT_TRY:
    1533       142539 :         {
    1534       142539 :           eh_catch c;
    1535       284356 :           for (c = r->u.eh_try.first_catch; c ; c = c->next_catch)
    1536              :             {
    1537       141817 :               lab = c->label;
    1538       141817 :               if (lab)
    1539        94212 :                 c->label = main_block_label (lab, label_for_bb);
    1540              :             }
    1541              :         }
    1542              :         break;
    1543              : 
    1544         9268 :       case ERT_ALLOWED_EXCEPTIONS:
    1545         9268 :         lab = r->u.allowed.label;
    1546         9268 :         if (lab)
    1547          914 :           r->u.allowed.label = main_block_label (lab, label_for_bb);
    1548              :         break;
    1549              :       }
    1550              : }
    1551              : 
    1552              : 
    1553              : /* Cleanup redundant labels.  This is a three-step process:
    1554              :      1) Find the leading label for each block.
    1555              :      2) Redirect all references to labels to the leading labels.
    1556              :      3) Cleanup all useless labels.  */
    1557              : 
    1558              : void
    1559      7538199 : cleanup_dead_labels (void)
    1560              : {
    1561      7538199 :   basic_block bb;
    1562      7538199 :   label_record *label_for_bb = XCNEWVEC (struct label_record,
    1563              :                                          last_basic_block_for_fn (cfun));
    1564              : 
    1565              :   /* Find a suitable label for each block.  We use the first user-defined
    1566              :      label if there is one, or otherwise just the first label we see.  */
    1567     65027752 :   FOR_EACH_BB_FN (bb, cfun)
    1568              :     {
    1569     57489553 :       gimple_stmt_iterator i;
    1570              : 
    1571    150379054 :       for (i = gsi_start_bb (bb); !gsi_end_p (i); gsi_next (&i))
    1572              :         {
    1573     91559957 :           tree label;
    1574     92899751 :           glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (i));
    1575              : 
    1576     35410198 :           if (!label_stmt)
    1577              :             break;
    1578              : 
    1579     35410198 :           label = gimple_label_label (label_stmt);
    1580              : 
    1581              :           /* If we have not yet seen a label for the current block,
    1582              :              remember this one and see if there are more labels.  */
    1583     35410198 :           if (!label_for_bb[bb->index].label)
    1584              :             {
    1585     33131081 :               label_for_bb[bb->index].label = label;
    1586     33131081 :               continue;
    1587              :             }
    1588              : 
    1589              :           /* If we did see a label for the current block already, but it
    1590              :              is an artificially created label, replace it if the current
    1591              :              label is a user defined label.  */
    1592      2279117 :           if (!DECL_ARTIFICIAL (label)
    1593      2279117 :               && DECL_ARTIFICIAL (label_for_bb[bb->index].label))
    1594              :             {
    1595        10250 :               label_for_bb[bb->index].label = label;
    1596        10250 :               break;
    1597              :             }
    1598              :         }
    1599              :     }
    1600              : 
    1601              :   /* Now redirect all jumps/branches to the selected label.
    1602              :      First do so for each block ending in a control statement.  */
    1603     65027752 :   FOR_EACH_BB_FN (bb, cfun)
    1604              :     {
    1605     57489553 :       gimple *stmt = *gsi_last_bb (bb);
    1606     57489553 :       tree label, new_label;
    1607              : 
    1608     57489553 :       if (!stmt)
    1609       427617 :         continue;
    1610              : 
    1611     57061936 :       switch (gimple_code (stmt))
    1612              :         {
    1613     16908696 :         case GIMPLE_COND:
    1614     16908696 :           {
    1615     16908696 :             gcond *cond_stmt = as_a <gcond *> (stmt);
    1616     16908696 :             label = gimple_cond_true_label (cond_stmt);
    1617     16908696 :             if (label)
    1618              :               {
    1619      5564004 :                 new_label = main_block_label (label, label_for_bb);
    1620      5564004 :                 if (new_label != label)
    1621         9570 :                   gimple_cond_set_true_label (cond_stmt, new_label);
    1622              :               }
    1623              : 
    1624     16908696 :             label = gimple_cond_false_label (cond_stmt);
    1625     16908696 :             if (label)
    1626              :               {
    1627      5564004 :                 new_label = main_block_label (label, label_for_bb);
    1628      5564004 :                 if (new_label != label)
    1629       111476 :                   gimple_cond_set_false_label (cond_stmt, new_label);
    1630              :               }
    1631              :           }
    1632              :           break;
    1633              : 
    1634       116363 :         case GIMPLE_SWITCH:
    1635       116363 :           {
    1636       116363 :             gswitch *switch_stmt = as_a <gswitch *> (stmt);
    1637       116363 :             size_t i, n = gimple_switch_num_labels (switch_stmt);
    1638              : 
    1639              :             /* Replace all destination labels.  */
    1640      1603707 :             for (i = 0; i < n; ++i)
    1641              :               {
    1642      1487344 :                 tree case_label = gimple_switch_label (switch_stmt, i);
    1643      1487344 :                 label = CASE_LABEL (case_label);
    1644      1487344 :                 new_label = main_block_label (label, label_for_bb);
    1645      1487344 :                 if (new_label != label)
    1646       790662 :                   CASE_LABEL (case_label) = new_label;
    1647              :               }
    1648              :             break;
    1649              :           }
    1650              : 
    1651         5633 :         case GIMPLE_ASM:
    1652         5633 :           {
    1653         5633 :             gasm *asm_stmt = as_a <gasm *> (stmt);
    1654         5633 :             int i, n = gimple_asm_nlabels (asm_stmt);
    1655              : 
    1656         8429 :             for (i = 0; i < n; ++i)
    1657              :               {
    1658         2796 :                 tree cons = gimple_asm_label_op (asm_stmt, i);
    1659         2796 :                 tree label = main_block_label (TREE_VALUE (cons), label_for_bb);
    1660         2796 :                 TREE_VALUE (cons) = label;
    1661              :               }
    1662              :             break;
    1663              :           }
    1664              : 
    1665              :         /* We have to handle gotos until they're removed, and we don't
    1666              :            remove them until after we've created the CFG edges.  */
    1667      6302117 :         case GIMPLE_GOTO:
    1668      6302117 :           if (!computed_goto_p (stmt))
    1669              :             {
    1670      6299765 :               ggoto *goto_stmt = as_a <ggoto *> (stmt);
    1671      6299765 :               label = gimple_goto_dest (goto_stmt);
    1672      6299765 :               new_label = main_block_label (label, label_for_bb);
    1673      6299765 :               if (new_label != label)
    1674      1121465 :                 gimple_goto_set_dest (goto_stmt, new_label);
    1675              :             }
    1676              :           break;
    1677              : 
    1678          822 :         case GIMPLE_TRANSACTION:
    1679          822 :           {
    1680          822 :             gtransaction *txn = as_a <gtransaction *> (stmt);
    1681              : 
    1682          822 :             label = gimple_transaction_label_norm (txn);
    1683          822 :             if (label)
    1684              :               {
    1685          802 :                 new_label = main_block_label (label, label_for_bb);
    1686          802 :                 if (new_label != label)
    1687            0 :                   gimple_transaction_set_label_norm (txn, new_label);
    1688              :               }
    1689              : 
    1690          822 :             label = gimple_transaction_label_uninst (txn);
    1691          822 :             if (label)
    1692              :               {
    1693          794 :                 new_label = main_block_label (label, label_for_bb);
    1694          794 :                 if (new_label != label)
    1695            0 :                   gimple_transaction_set_label_uninst (txn, new_label);
    1696              :               }
    1697              : 
    1698          822 :             label = gimple_transaction_label_over (txn);
    1699          822 :             if (label)
    1700              :               {
    1701          814 :                 new_label = main_block_label (label, label_for_bb);
    1702          814 :                 if (new_label != label)
    1703            7 :                   gimple_transaction_set_label_over (txn, new_label);
    1704              :               }
    1705              :           }
    1706              :           break;
    1707              : 
    1708              :         default:
    1709              :           break;
    1710              :       }
    1711              :     }
    1712              : 
    1713              :   /* Do the same for the exception region tree labels.  */
    1714      7538199 :   cleanup_dead_labels_eh (label_for_bb);
    1715              : 
    1716              :   /* Finally, purge dead labels.  All user-defined labels and labels that
    1717              :      can be the target of non-local gotos and labels which have their
    1718              :      address taken are preserved.  */
    1719     65027752 :   FOR_EACH_BB_FN (bb, cfun)
    1720              :     {
    1721     57489553 :       gimple_stmt_iterator i;
    1722     57489553 :       tree label_for_this_bb = label_for_bb[bb->index].label;
    1723              : 
    1724     57489553 :       if (!label_for_this_bb)
    1725     24358384 :         continue;
    1726              : 
    1727              :       /* If the main label of the block is unused, we may still remove it.  */
    1728     33131169 :       if (!label_for_bb[bb->index].used)
    1729     15474861 :         label_for_this_bb = NULL;
    1730              : 
    1731    101672636 :       for (i = gsi_start_bb (bb); !gsi_end_p (i); )
    1732              :         {
    1733     67639009 :           tree label;
    1734     68541467 :           glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (i));
    1735              : 
    1736     35410298 :           if (!label_stmt)
    1737              :             break;
    1738              : 
    1739     35410298 :           label = gimple_label_label (label_stmt);
    1740              : 
    1741     35410298 :           if (label == label_for_this_bb
    1742     17753990 :               || !DECL_ARTIFICIAL (label)
    1743     17024606 :               || DECL_NONLOCAL (label)
    1744     52433385 :               || FORCED_LABEL (label))
    1745     18414125 :             gsi_next (&i);
    1746              :           else
    1747              :             {
    1748     16996173 :               gcc_checking_assert (EH_LANDING_PAD_NR (label) == 0);
    1749     16996173 :               gsi_remove (&i, true);
    1750              :             }
    1751              :         }
    1752              :     }
    1753              : 
    1754      7538199 :   free (label_for_bb);
    1755      7538199 : }
    1756              : 
    1757              : /* Scan the sorted vector of cases in STMT (a GIMPLE_SWITCH) and combine
    1758              :    the ones jumping to the same label.
    1759              :    Eg. three separate entries 1: 2: 3: become one entry 1..3:  */
    1760              : 
    1761              : bool
    1762       292123 : group_case_labels_stmt (gswitch *stmt)
    1763              : {
    1764       292123 :   int old_size = gimple_switch_num_labels (stmt);
    1765       292123 :   int i, next_index, new_size;
    1766       292123 :   basic_block default_bb = NULL;
    1767              : 
    1768       292123 :   default_bb = gimple_switch_default_bb (cfun, stmt);
    1769              : 
    1770              :   /* Look for possible opportunities to merge cases.  */
    1771       292123 :   new_size = i = 1;
    1772      2385343 :   while (i < old_size)
    1773              :     {
    1774      1801097 :       tree base_case, base_high;
    1775      1801097 :       basic_block base_bb;
    1776              : 
    1777      1801097 :       base_case = gimple_switch_label (stmt, i);
    1778              : 
    1779      1801097 :       gcc_assert (base_case);
    1780      1801097 :       base_bb = label_to_block (cfun, CASE_LABEL (base_case));
    1781              : 
    1782              :       /* Discard cases that have the same destination as the default case.  */
    1783      1809241 :       if (base_bb == NULL
    1784      1801097 :           || base_bb == default_bb)
    1785              :         {
    1786         8144 :           i++;
    1787         8144 :           continue;
    1788              :         }
    1789              : 
    1790      3585906 :       base_high = CASE_HIGH (base_case)
    1791      1792953 :           ? CASE_HIGH (base_case)
    1792      1710701 :           : CASE_LOW (base_case);
    1793      1792953 :       next_index = i + 1;
    1794              : 
    1795              :       /* Try to merge case labels.  Break out when we reach the end
    1796              :          of the label vector or when we cannot merge the next case
    1797              :          label with the current one.  */
    1798      2540257 :       while (next_index < old_size)
    1799              :         {
    1800      2250148 :           tree merge_case = gimple_switch_label (stmt, next_index);
    1801      2250148 :           basic_block merge_bb = label_to_block (cfun, CASE_LABEL (merge_case));
    1802      2250148 :           wide_int bhp1 = wi::to_wide (base_high) + 1;
    1803              : 
    1804              :           /* Merge the cases if they jump to the same place,
    1805              :              and their ranges are consecutive.  */
    1806      2250148 :           if (merge_bb == base_bb
    1807      3096920 :               && wi::to_wide (CASE_LOW (merge_case)) == bhp1)
    1808              :             {
    1809       747304 :               base_high
    1810      1494608 :                 = (CASE_HIGH (merge_case)
    1811       747304 :                    ? CASE_HIGH (merge_case) : CASE_LOW (merge_case));
    1812       747304 :               CASE_HIGH (base_case) = base_high;
    1813       747304 :               next_index++;
    1814              :             }
    1815              :           else
    1816              :             break;
    1817      2250148 :         }
    1818              : 
    1819      1792953 :       if (new_size < i)
    1820        72965 :         gimple_switch_set_label (stmt, new_size,
    1821              :                                  gimple_switch_label (stmt, i));
    1822      1792953 :       i = next_index;
    1823      1792953 :       new_size++;
    1824              :     }
    1825              : 
    1826       292123 :   gcc_assert (new_size <= old_size);
    1827              : 
    1828       292123 :   if (new_size < old_size)
    1829        19971 :     gimple_switch_set_num_labels (stmt, new_size);
    1830              : 
    1831       292123 :   return new_size < old_size;
    1832              : }
    1833              : 
    1834              : /* Look for blocks ending in a multiway branch (a GIMPLE_SWITCH),
    1835              :    and scan the sorted vector of cases.  Combine the ones jumping to the
    1836              :    same label.  */
    1837              : 
    1838              : bool
    1839      4528046 : group_case_labels (void)
    1840              : {
    1841      4528046 :   basic_block bb;
    1842      4528046 :   bool changed = false;
    1843              : 
    1844     39665777 :   FOR_EACH_BB_FN (bb, cfun)
    1845              :     {
    1846    105193369 :       if (gswitch *stmt = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
    1847        62011 :         changed |= group_case_labels_stmt (stmt);
    1848              :     }
    1849              : 
    1850      4528046 :   return changed;
    1851              : }
    1852              : 
    1853              : /* Checks whether we can merge block B into block A.  */
    1854              : 
    1855              : static bool
    1856    432113962 : gimple_can_merge_blocks_p (basic_block a, basic_block b)
    1857              : {
    1858    432113962 :   gimple *stmt;
    1859              : 
    1860    432113962 :   if (!single_succ_p (a))
    1861              :     return false;
    1862              : 
    1863    205107852 :   if (single_succ_edge (a)->flags & EDGE_COMPLEX)
    1864              :     return false;
    1865              : 
    1866    192935158 :   if (single_succ (a) != b)
    1867              :     return false;
    1868              : 
    1869    192935158 :   if (!single_pred_p (b))
    1870              :     return false;
    1871              : 
    1872     94433173 :   if (a == ENTRY_BLOCK_PTR_FOR_FN (cfun)
    1873     61656313 :       || b == EXIT_BLOCK_PTR_FOR_FN (cfun))
    1874              :     return false;
    1875              : 
    1876              :   /* If A ends by a statement causing exceptions or something similar, we
    1877              :      cannot merge the blocks.  */
    1878     34435405 :   stmt = *gsi_last_nondebug_bb (a);
    1879     34435405 :   if (stmt && stmt_ends_bb_p (stmt))
    1880              :     return false;
    1881              : 
    1882              :   /* Check if the compiler-inserted nop should block merges.  */
    1883     26503268 :   if (stmt && gimple_nop_p (stmt)
    1884         9648 :       && gimple_location (stmt) == DECL_SOURCE_LOCATION (cfun->decl)
    1885     32541548 :       && coverage_suppressed_p (a) && !coverage_suppressed_p (b))
    1886              :     return false;
    1887              : 
    1888              :   /* We cannot merge blocks if one has suppressed coverage and the other hasn't.
    1889              :      The exception is when the merge-into block is empty AND is the one with
    1890              :      coverage suppressed.  This can happen for code like this:
    1891              : 
    1892              :      while (cond)
    1893              :        #pragma GCC suppress_coverage begin
    1894              :        foo ();
    1895              :        #pragma GCC suppress_coverage end
    1896              :   */
    1897     32541512 :   if (coverage_suppressed_p (a) != coverage_suppressed_p (b)
    1898     32541512 :       && !(coverage_suppressed_p (a) && gimple_empty_block_p (a)))
    1899              :       return false;
    1900              : 
    1901              :   /* Examine the labels at the beginning of B.  */
    1902     65276458 :   for (gimple_stmt_iterator gsi = gsi_start_bb (b); !gsi_end_p (gsi);
    1903       200298 :        gsi_next (&gsi))
    1904              :     {
    1905     29107745 :       tree lab;
    1906     29107745 :       glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (gsi));
    1907      1908712 :       if (!label_stmt)
    1908              :         break;
    1909      1908712 :       lab = gimple_label_label (label_stmt);
    1910              : 
    1911              :       /* Do not remove user forced labels or for -O0 any user labels.  */
    1912      1967415 :       if (!DECL_ARTIFICIAL (lab) && (!optimize || FORCED_LABEL (lab)))
    1913    432113962 :         return false;
    1914              :     }
    1915              : 
    1916              :   /* Protect simple loop latches.  We only want to avoid merging
    1917              :      the latch with the loop header or with a block in another
    1918              :      loop in this case.  */
    1919     30829666 :   if (current_loops
    1920     27997237 :       && b->loop_father->latch == b
    1921       439634 :       && loops_state_satisfies_p (LOOPS_HAVE_SIMPLE_LATCHES)
    1922     30867353 :       && (b->loop_father->header == a
    1923        25870 :           || b->loop_father != a->loop_father))
    1924              :     return false;
    1925              : 
    1926              :   /* It must be possible to eliminate all phi nodes in B.  If ssa form
    1927              :      is not up-to-date and a name-mapping is registered, we cannot eliminate
    1928              :      any phis.  Symbols marked for renaming are never a problem though.  */
    1929     36891464 :   for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (gsi);
    1930      6073739 :        gsi_next (&gsi))
    1931              :     {
    1932      6073739 :       gphi *phi = gsi.phi ();
    1933              :       /* Technically only new names matter.  */
    1934      6073739 :       if (name_registered_for_update_p (PHI_RESULT (phi)))
    1935            0 :         return false;
    1936              :     }
    1937              : 
    1938              :   /* When not optimizing, don't merge if we'd lose goto_locus.  */
    1939     30817725 :   if (!optimize
    1940     30817725 :       && single_succ_edge (a)->goto_locus != UNKNOWN_LOCATION)
    1941              :     {
    1942       595353 :       location_t goto_locus = single_succ_edge (a)->goto_locus;
    1943       595353 :       gimple_stmt_iterator prev, next;
    1944       595353 :       prev = gsi_last_nondebug_bb (a);
    1945       595353 :       next = gsi_after_labels (b);
    1946       595353 :       if (!gsi_end_p (next) && is_gimple_debug (gsi_stmt (next)))
    1947            6 :         gsi_next_nondebug (&next);
    1948       595353 :       if ((gsi_end_p (prev)
    1949       448259 :            || gimple_location (gsi_stmt (prev)) != goto_locus)
    1950       999518 :           && (gsi_end_p (next)
    1951       501795 :               || gimple_location (gsi_stmt (next)) != goto_locus))
    1952       551067 :         return false;
    1953              :     }
    1954              : 
    1955              :   return true;
    1956              : }
    1957              : 
    1958              : /* Replaces all uses of NAME by VAL.  */
    1959              : 
    1960              : void
    1961      4055702 : replace_uses_by (tree name, tree val)
    1962              : {
    1963      4055702 :   imm_use_iterator imm_iter;
    1964      4055702 :   use_operand_p use;
    1965      4055702 :   gimple *stmt;
    1966      4055702 :   edge e;
    1967      4055702 :   auto_bitmap eh_cleanup_bbs;
    1968              : 
    1969     11391785 :   FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
    1970              :     {
    1971              :       /* Mark the block if we change the last stmt in it.  */
    1972      7336083 :       if (cfgcleanup_altered_bbs
    1973      7336083 :           && stmt_ends_bb_p (stmt))
    1974       804600 :         bitmap_set_bit (cfgcleanup_altered_bbs, gimple_bb (stmt)->index);
    1975              : 
    1976     14986504 :       FOR_EACH_IMM_USE_ON_STMT (use, imm_iter)
    1977              :         {
    1978      7493252 :           replace_exp (use, val);
    1979              : 
    1980      7493252 :           if (gimple_code (stmt) == GIMPLE_PHI)
    1981              :             {
    1982      2498709 :               e = gimple_phi_arg_edge (as_a <gphi *> (stmt),
    1983      2498709 :                                        PHI_ARG_INDEX_FROM_USE (use));
    1984      2498709 :               if (e->flags & EDGE_ABNORMAL
    1985      2498709 :                   && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val))
    1986              :                 {
    1987              :                   /* This can only occur for virtual operands, since
    1988              :                      for the real ones SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
    1989              :                      would prevent replacement.  */
    1990            0 :                   gcc_checking_assert (virtual_operand_p (name));
    1991            0 :                   SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val) = 1;
    1992              :                 }
    1993              :             }
    1994              :         }
    1995              : 
    1996      7336083 :       if (gimple_code (stmt) != GIMPLE_PHI)
    1997              :         {
    1998      4986625 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    1999      4986625 :           gimple *orig_stmt = stmt;
    2000      4986625 :           size_t i;
    2001              : 
    2002              :           /* FIXME.  It shouldn't be required to keep TREE_CONSTANT
    2003              :              on ADDR_EXPRs up-to-date on GIMPLE.  Propagation will
    2004              :              only change sth from non-invariant to invariant, and only
    2005              :              when propagating constants.  */
    2006      4986625 :           if (is_gimple_min_invariant (val))
    2007      1784960 :             for (i = 0; i < gimple_num_ops (stmt); i++)
    2008              :               {
    2009      1292295 :                 tree op = gimple_op (stmt, i);
    2010              :                 /* Operands may be empty here.  For example, the labels
    2011              :                    of a GIMPLE_COND are nulled out following the creation
    2012              :                    of the corresponding CFG edges.  */
    2013      1292295 :                 if (op && TREE_CODE (op) == ADDR_EXPR)
    2014        66888 :                   recompute_tree_invariant_for_addr_expr (op);
    2015              :               }
    2016      4986625 :           update_stmt (stmt);
    2017              : 
    2018      4986625 :           if (fold_stmt (&gsi))
    2019              :             {
    2020       527114 :               stmt = gsi_stmt (gsi);
    2021       527114 :               update_stmt (stmt);
    2022              :             }
    2023              : 
    2024      4986625 :           if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
    2025          977 :             bitmap_set_bit (eh_cleanup_bbs, gimple_bb (stmt)->index);
    2026              :         }
    2027      4055702 :     }
    2028              : 
    2029      4055702 :   if (!bitmap_empty_p (eh_cleanup_bbs))
    2030          938 :     gimple_purge_all_dead_eh_edges (eh_cleanup_bbs);
    2031              : 
    2032      4055702 :   gcc_checking_assert (has_zero_uses (name));
    2033              : 
    2034              :   /* Also update the trees stored in loop structures.  */
    2035      4055702 :   if (current_loops)
    2036              :     {
    2037    140084170 :       for (auto loop : loops_list (cfun, 0))
    2038    131972766 :           substitute_in_loop_info (loop, name, val);
    2039              :     }
    2040      4055702 : }
    2041              : 
    2042              : /* Merge block B into block A.  */
    2043              : 
    2044              : static void
    2045     17881878 : gimple_merge_blocks (basic_block a, basic_block b)
    2046              : {
    2047     17881878 :   gimple_stmt_iterator last, gsi;
    2048     17881878 :   gphi_iterator psi;
    2049              : 
    2050     17882049 :   if (coverage_suppressed_p (a) && !coverage_suppressed_p (b)
    2051     17881901 :       && gimple_empty_block_p (a))
    2052           23 :     suppress_coverage_unset (a);
    2053              : 
    2054     17881878 :   if (dump_file)
    2055        20704 :     fprintf (dump_file, "Merging blocks %d and %d\n", a->index, b->index);
    2056              : 
    2057              :   /* Remove all single-valued PHI nodes from block B of the form
    2058              :      V_i = PHI <V_j> by propagating V_j to all the uses of V_i.  */
    2059     17881878 :   gsi = gsi_last_bb (a);
    2060     21922369 :   for (psi = gsi_start_phis (b); !gsi_end_p (psi); )
    2061              :     {
    2062      4040491 :       gimple *phi = gsi_stmt (psi);
    2063      4040491 :       tree def = gimple_phi_result (phi), use = gimple_phi_arg_def (phi, 0);
    2064      4040491 :       gimple *copy;
    2065      4040491 :       bool may_replace_uses = (virtual_operand_p (def)
    2066      4040491 :                                || may_propagate_copy (def, use));
    2067              : 
    2068              :       /* In case we maintain loop closed ssa form, do not propagate arguments
    2069              :          of loop exit phi nodes.  */
    2070      4040491 :       if (current_loops
    2071      4040491 :           && loops_state_satisfies_p (LOOP_CLOSED_SSA)
    2072       655855 :           && !virtual_operand_p (def)
    2073       478811 :           && TREE_CODE (use) == SSA_NAME
    2074      4399355 :           && a->loop_father != b->loop_father)
    2075              :         may_replace_uses = false;
    2076              : 
    2077      4040183 :       if (!may_replace_uses)
    2078              :         {
    2079         1260 :           gcc_assert (!virtual_operand_p (def));
    2080              : 
    2081              :           /* Note that just emitting the copies is fine -- there is no problem
    2082              :              with ordering of phi nodes.  This is because A is the single
    2083              :              predecessor of B, therefore results of the phi nodes cannot
    2084              :              appear as arguments of the phi nodes.  */
    2085          630 :           copy = gimple_build_assign (def, use);
    2086          630 :           gsi_insert_after (&gsi, copy, GSI_NEW_STMT);
    2087          630 :           remove_phi_node (&psi, false);
    2088              :         }
    2089              :       else
    2090              :         {
    2091              :           /* If we deal with a PHI for virtual operands, we can simply
    2092              :              propagate these without fussing with folding or updating
    2093              :              the stmt.  */
    2094      8079722 :           if (virtual_operand_p (def))
    2095              :             {
    2096      1559780 :               imm_use_iterator iter;
    2097      1559780 :               use_operand_p use_p;
    2098      1559780 :               gimple *stmt;
    2099              : 
    2100      4486752 :               FOR_EACH_IMM_USE_STMT (stmt, iter, def)
    2101      5885814 :                 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
    2102      2942907 :                   SET_USE (use_p, use);
    2103              : 
    2104      1559780 :               if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
    2105          281 :                 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use) = 1;
    2106              :             }
    2107              :           else
    2108      2480081 :             replace_uses_by (def, use);
    2109              : 
    2110      4039861 :           remove_phi_node (&psi, true);
    2111              :         }
    2112              :     }
    2113              : 
    2114              :   /* Ensure that B follows A.  */
    2115     17881878 :   move_block_after (b, a);
    2116              : 
    2117     17881878 :   gcc_assert (single_succ_edge (a)->flags & EDGE_FALLTHRU);
    2118     35763756 :   gcc_assert (!*gsi_last_bb (a)
    2119              :               || !stmt_ends_bb_p (*gsi_last_bb (a)));
    2120              : 
    2121              :   /* Remove labels from B and set gimple_bb to A for other statements.  */
    2122    139764527 :   for (gsi = gsi_start_bb (b); !gsi_end_p (gsi);)
    2123              :     {
    2124    104000771 :       gimple *stmt = gsi_stmt (gsi);
    2125    104000771 :       if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
    2126              :         {
    2127       117674 :           tree label = gimple_label_label (label_stmt);
    2128       117674 :           int lp_nr;
    2129              : 
    2130       117674 :           gsi_remove (&gsi, false);
    2131              : 
    2132              :           /* Now that we can thread computed gotos, we might have
    2133              :              a situation where we have a forced label in block B
    2134              :              However, the label at the start of block B might still be
    2135              :              used in other ways (think about the runtime checking for
    2136              :              Fortran assigned gotos).  So we cannot just delete the
    2137              :              label.  Instead we move the label to the start of block A.  */
    2138       117674 :           if (FORCED_LABEL (label))
    2139              :             {
    2140         5253 :               gimple_stmt_iterator dest_gsi = gsi_start_bb (a);
    2141         5253 :               tree first_label = NULL_TREE;
    2142         5253 :               if (!gsi_end_p (dest_gsi))
    2143         5231 :                 if (glabel *first_label_stmt
    2144         5231 :                     = dyn_cast <glabel *> (gsi_stmt (dest_gsi)))
    2145         4469 :                   first_label = gimple_label_label (first_label_stmt);
    2146         4469 :               if (first_label
    2147         4469 :                   && (DECL_NONLOCAL (first_label)
    2148         4469 :                       || EH_LANDING_PAD_NR (first_label) != 0))
    2149           12 :                 gsi_insert_after (&dest_gsi, stmt, GSI_NEW_STMT);
    2150              :               else
    2151         5241 :                 gsi_insert_before (&dest_gsi, stmt, GSI_NEW_STMT);
    2152              :             }
    2153              :           /* Other user labels keep around in a form of a debug stmt.  */
    2154       112421 :           else if (!DECL_ARTIFICIAL (label) && MAY_HAVE_DEBUG_BIND_STMTS)
    2155              :             {
    2156         9105 :               gimple *dbg = gimple_build_debug_bind (label,
    2157              :                                                      integer_zero_node,
    2158              :                                                      stmt);
    2159         9105 :               gimple_debug_bind_reset_value (dbg);
    2160         9105 :               gsi_insert_before (&gsi, dbg, GSI_SAME_STMT);
    2161              :             }
    2162              : 
    2163       117674 :           lp_nr = EH_LANDING_PAD_NR (label);
    2164       117674 :           if (lp_nr)
    2165              :             {
    2166         9252 :               eh_landing_pad lp = get_eh_landing_pad_from_number (lp_nr);
    2167         9252 :               lp->post_landing_pad = NULL;
    2168              :             }
    2169              :         }
    2170              :       else
    2171              :         {
    2172    103883097 :           gimple_set_bb (stmt, a);
    2173    103883097 :           gsi_next (&gsi);
    2174              :         }
    2175              :     }
    2176              : 
    2177              :   /* When merging two BBs, if their counts are different, the larger count
    2178              :      is selected as the new bb count. This is to handle inconsistent
    2179              :      profiles.  */
    2180     17881878 :   if (a->loop_father == b->loop_father)
    2181              :     {
    2182     17789462 :       a->count = a->count.merge (b->count);
    2183              :     }
    2184              : 
    2185              :   /* Merge the sequences.  */
    2186     17881878 :   last = gsi_last_bb (a);
    2187     35763756 :   gsi_insert_seq_after (&last, bb_seq (b), GSI_NEW_STMT);
    2188     17881878 :   set_bb_seq (b, NULL);
    2189              : 
    2190     17881878 :   if (cfgcleanup_altered_bbs)
    2191     17856166 :     bitmap_set_bit (cfgcleanup_altered_bbs, a->index);
    2192     17881878 : }
    2193              : 
    2194              : 
    2195              : /* Return the one of two successors of BB that is not reachable by a
    2196              :    complex edge, if there is one.  Else, return BB.  We use
    2197              :    this in optimizations that use post-dominators for their heuristics,
    2198              :    to catch the cases in C++ where function calls are involved.  */
    2199              : 
    2200              : basic_block
    2201            6 : single_noncomplex_succ (basic_block bb)
    2202              : {
    2203            6 :   edge e0, e1;
    2204            6 :   if (EDGE_COUNT (bb->succs) != 2)
    2205              :     return bb;
    2206              : 
    2207            6 :   e0 = EDGE_SUCC (bb, 0);
    2208            6 :   e1 = EDGE_SUCC (bb, 1);
    2209            6 :   if (e0->flags & EDGE_COMPLEX)
    2210            6 :     return e1->dest;
    2211            0 :   if (e1->flags & EDGE_COMPLEX)
    2212            0 :     return e0->dest;
    2213              : 
    2214              :   return bb;
    2215              : }
    2216              : 
    2217              : /* T is CALL_EXPR.  Set current_function_calls_* flags.  */
    2218              : 
    2219              : void
    2220     55243652 : notice_special_calls (gcall *call)
    2221              : {
    2222     55243652 :   int flags = gimple_call_flags (call);
    2223              : 
    2224     55243652 :   if (flags & ECF_MAY_BE_ALLOCA)
    2225       160671 :     cfun->calls_alloca = true;
    2226     55243652 :   if (flags & ECF_RETURNS_TWICE)
    2227         8210 :     cfun->calls_setjmp = true;
    2228     55243652 :   if (gimple_call_must_tail_p (call))
    2229         3634 :     cfun->has_musttail = true;
    2230     55243652 : }
    2231              : 
    2232              : 
    2233              : /* Clear flags set by notice_special_calls.  Used by dead code removal
    2234              :    to update the flags.  */
    2235              : 
    2236              : void
    2237      8302670 : clear_special_calls (void)
    2238              : {
    2239      8302670 :   cfun->calls_alloca = false;
    2240      8302670 :   cfun->calls_setjmp = false;
    2241      8302670 :   cfun->has_musttail = false;
    2242      8302670 : }
    2243              : 
    2244              : /* Remove PHI nodes associated with basic block BB and all edges out of BB.  */
    2245              : 
    2246              : static void
    2247     37419675 : remove_phi_nodes_and_edges_for_unreachable_block (basic_block bb)
    2248              : {
    2249              :   /* Since this block is no longer reachable, we can just delete all
    2250              :      of its PHI nodes.  */
    2251     37419675 :   remove_phi_nodes (bb);
    2252              : 
    2253              :   /* Remove edges to BB's successors.  */
    2254    110189800 :   while (EDGE_COUNT (bb->succs) > 0)
    2255     35350450 :     remove_edge (EDGE_SUCC (bb, 0));
    2256     37419675 : }
    2257              : 
    2258              : 
    2259              : /* Remove statements of basic block BB.  */
    2260              : 
    2261              : static void
    2262     37419675 : remove_bb (basic_block bb)
    2263              : {
    2264     37419675 :   gimple_stmt_iterator i;
    2265              : 
    2266     37419675 :   if (dump_file)
    2267              :     {
    2268        49277 :       fprintf (dump_file, "Removing basic block %d\n", bb->index);
    2269        49277 :       if (dump_flags & TDF_DETAILS)
    2270              :         {
    2271        34630 :           dump_bb (dump_file, bb, 0, TDF_BLOCKS);
    2272        34630 :           fprintf (dump_file, "\n");
    2273              :         }
    2274              :     }
    2275              : 
    2276     37419675 :   if (current_loops)
    2277              :     {
    2278     35456990 :       class loop *loop = bb->loop_father;
    2279              : 
    2280              :       /* If a loop gets removed, clean up the information associated
    2281              :          with it.  */
    2282     35456990 :       if (loop->latch == bb
    2283     35244930 :           || loop->header == bb)
    2284       256040 :         free_numbers_of_iterations_estimates (loop);
    2285              :     }
    2286              : 
    2287              :   /* Remove all the instructions in the block.  */
    2288     37419675 :   if (bb_seq (bb) != NULL)
    2289              :     {
    2290              :       /* Walk backwards so as to get a chance to substitute all
    2291              :          released DEFs into debug stmts.  See
    2292              :          eliminate_unnecessary_stmts() in tree-ssa-dce.cc for more
    2293              :          details.  */
    2294     29840284 :       for (i = gsi_last_bb (bb); !gsi_end_p (i);)
    2295              :         {
    2296     22682939 :           gimple *stmt = gsi_stmt (i);
    2297     22682939 :           glabel *label_stmt = dyn_cast <glabel *> (stmt);
    2298      2041339 :           if (label_stmt
    2299      2041339 :               && (FORCED_LABEL (gimple_label_label (label_stmt))
    2300      2038359 :                   || DECL_NONLOCAL (gimple_label_label (label_stmt))))
    2301              :             {
    2302         2982 :               basic_block new_bb;
    2303         2982 :               gimple_stmt_iterator new_gsi;
    2304              : 
    2305              :               /* A non-reachable non-local label may still be referenced.
    2306              :                  But it no longer needs to carry the extra semantics of
    2307              :                  non-locality.  */
    2308         2982 :               if (DECL_NONLOCAL (gimple_label_label (label_stmt)))
    2309              :                 {
    2310            2 :                   DECL_NONLOCAL (gimple_label_label (label_stmt)) = 0;
    2311            2 :                   FORCED_LABEL (gimple_label_label (label_stmt)) = 1;
    2312              :                 }
    2313              : 
    2314         2982 :               new_bb = bb->prev_bb;
    2315              :               /* Don't move any labels into ENTRY block.  */
    2316         2982 :               if (new_bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
    2317              :                 {
    2318            0 :                   new_bb = single_succ (new_bb);
    2319            0 :                   gcc_assert (new_bb != bb);
    2320              :                 }
    2321         2982 :               if ((unsigned) bb->index < bb_to_omp_idx.length ()
    2322            8 :                   && ((unsigned) new_bb->index >= bb_to_omp_idx.length ()
    2323            8 :                       || (bb_to_omp_idx[bb->index]
    2324            8 :                           != bb_to_omp_idx[new_bb->index])))
    2325              :                 {
    2326              :                   /* During cfg pass make sure to put orphaned labels
    2327              :                      into the right OMP region.  */
    2328              :                   unsigned int i;
    2329              :                   int idx;
    2330           24 :                   new_bb = NULL;
    2331           24 :                   FOR_EACH_VEC_ELT (bb_to_omp_idx, i, idx)
    2332           24 :                     if (i >= NUM_FIXED_BLOCKS
    2333            8 :                         && idx == bb_to_omp_idx[bb->index]
    2334           32 :                         && i != (unsigned) bb->index)
    2335              :                       {
    2336            8 :                         new_bb = BASIC_BLOCK_FOR_FN (cfun, i);
    2337            8 :                         break;
    2338              :                       }
    2339            8 :                   if (new_bb == NULL)
    2340              :                     {
    2341            0 :                       new_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
    2342            0 :                       gcc_assert (new_bb != bb);
    2343              :                     }
    2344              :                 }
    2345         2982 :               new_gsi = gsi_after_labels (new_bb);
    2346         2982 :               gsi_remove (&i, false);
    2347         2982 :               gsi_insert_before (&new_gsi, stmt, GSI_NEW_STMT);
    2348              :             }
    2349              :           else
    2350              :             {
    2351              :               /* Release SSA definitions.  */
    2352     22679957 :               release_defs (stmt);
    2353     22679957 :               gsi_remove (&i, true);
    2354              :             }
    2355              : 
    2356     22682939 :           if (gsi_end_p (i))
    2357     45365878 :             i = gsi_last_bb (bb);
    2358              :           else
    2359            0 :             gsi_prev (&i);
    2360              :         }
    2361              :     }
    2362              : 
    2363     37419675 :   if ((unsigned) bb->index < bb_to_omp_idx.length ())
    2364        13752 :     bb_to_omp_idx[bb->index] = -1;
    2365     37419675 :   remove_phi_nodes_and_edges_for_unreachable_block (bb);
    2366     37419675 :   bb->il.gimple.seq = NULL;
    2367     37419675 :   bb->il.gimple.phi_nodes = NULL;
    2368     37419675 : }
    2369              : 
    2370              : 
    2371              : /* Given a basic block BB and a value VAL for use in the final statement
    2372              :    of the block (if a GIMPLE_COND, GIMPLE_SWITCH, or computed goto), return
    2373              :    the edge that will be taken out of the block.
    2374              :    If VAL is NULL_TREE, then the current value of the final statement's
    2375              :    predicate or index is used.
    2376              :    If the value does not match a unique edge, NULL is returned.  */
    2377              : 
    2378              : edge
    2379    254383483 : find_taken_edge (basic_block bb, tree val)
    2380              : {
    2381    254383483 :   gimple *stmt;
    2382              : 
    2383    254383483 :   stmt = *gsi_last_bb (bb);
    2384              : 
    2385              :   /* Handle ENTRY and EXIT.  */
    2386    254383483 :   if (!stmt)
    2387              :     ;
    2388              : 
    2389    250649715 :   else if (gimple_code (stmt) == GIMPLE_COND)
    2390    203788494 :     return find_taken_edge_cond_expr (as_a <gcond *> (stmt), val);
    2391              : 
    2392     46861221 :   else if (gimple_code (stmt) == GIMPLE_SWITCH)
    2393      1113895 :     return find_taken_edge_switch_expr (as_a <gswitch *> (stmt), val);
    2394              : 
    2395     45747326 :   else if (computed_goto_p (stmt))
    2396              :     {
    2397              :       /* Only optimize if the argument is a label, if the argument is
    2398              :          not a label then we cannot construct a proper CFG.
    2399              : 
    2400              :          It may be the case that we only need to allow the LABEL_REF to
    2401              :          appear inside an ADDR_EXPR, but we also allow the LABEL_REF to
    2402              :          appear inside a LABEL_EXPR just to be safe.  */
    2403         3121 :       if (val
    2404         1764 :           && (TREE_CODE (val) == ADDR_EXPR || TREE_CODE (val) == LABEL_EXPR)
    2405         3539 :           && TREE_CODE (TREE_OPERAND (val, 0)) == LABEL_DECL)
    2406          329 :         return find_taken_edge_computed_goto (bb, TREE_OPERAND (val, 0));
    2407              :     }
    2408              : 
    2409              :   /* Otherwise we only know the taken successor edge if it's unique.  */
    2410     49480765 :   return single_succ_p (bb) ? single_succ_edge (bb) : NULL;
    2411              : }
    2412              : 
    2413              : /* Given a constant value VAL and the entry block BB to a GOTO_EXPR
    2414              :    statement, determine which of the outgoing edges will be taken out of the
    2415              :    block.  Return NULL if either edge may be taken.  */
    2416              : 
    2417              : static edge
    2418          329 : find_taken_edge_computed_goto (basic_block bb, tree val)
    2419              : {
    2420          329 :   basic_block dest;
    2421          329 :   edge e = NULL;
    2422              : 
    2423          329 :   dest = label_to_block (cfun, val);
    2424          329 :   if (dest)
    2425          325 :     e = find_edge (bb, dest);
    2426              : 
    2427              :   /* It's possible for find_edge to return NULL here on invalid code
    2428              :      that abuses the labels-as-values extension (e.g. code that attempts to
    2429              :      jump *between* functions via stored labels-as-values; PR 84136).
    2430              :      If so, then we simply return that NULL for the edge.
    2431              :      We don't currently have a way of detecting such invalid code, so we
    2432              :      can't assert that it was the case when a NULL edge occurs here.  */
    2433              : 
    2434          329 :   return e;
    2435              : }
    2436              : 
    2437              : /* Given COND_STMT and a constant value VAL for use as the predicate,
    2438              :    determine which of the two edges will be taken out of
    2439              :    the statement's block.  Return NULL if either edge may be taken.
    2440              :    If VAL is NULL_TREE, then the current value of COND_STMT's predicate
    2441              :    is used.  */
    2442              : 
    2443              : static edge
    2444    203788494 : find_taken_edge_cond_expr (const gcond *cond_stmt, tree val)
    2445              : {
    2446    203788494 :   edge true_edge, false_edge;
    2447              : 
    2448    203788494 :   if (val == NULL_TREE)
    2449              :     {
    2450              :       /* Use the current value of the predicate.  */
    2451    189283973 :       if (gimple_cond_true_p (cond_stmt))
    2452        77618 :         val = integer_one_node;
    2453    189206355 :       else if (gimple_cond_false_p (cond_stmt))
    2454       171969 :         val = integer_zero_node;
    2455              :       else
    2456              :         return NULL;
    2457              :     }
    2458     14504521 :   else if (TREE_CODE (val) != INTEGER_CST)
    2459              :     return NULL;
    2460              : 
    2461     13502051 :   extract_true_false_edges_from_block (gimple_bb (cond_stmt),
    2462              :                                        &true_edge, &false_edge);
    2463              : 
    2464     13502051 :   return (integer_zerop (val) ? false_edge : true_edge);
    2465              : }
    2466              : 
    2467              : /* Given SWITCH_STMT and an INTEGER_CST VAL for use as the index, determine
    2468              :    which edge will be taken out of the statement's block.  Return NULL if any
    2469              :    edge may be taken.
    2470              :    If VAL is NULL_TREE, then the current value of SWITCH_STMT's index
    2471              :    is used.  */
    2472              : 
    2473              : edge
    2474      1114045 : find_taken_edge_switch_expr (const gswitch *switch_stmt, tree val)
    2475              : {
    2476      1114045 :   basic_block dest_bb;
    2477      1114045 :   edge e;
    2478      1114045 :   tree taken_case;
    2479              : 
    2480      1114045 :   if (gimple_switch_num_labels (switch_stmt) == 1)
    2481            0 :     taken_case = gimple_switch_default_label (switch_stmt);
    2482              :   else
    2483              :     {
    2484      1114045 :       if (val == NULL_TREE)
    2485       155519 :         val = gimple_switch_index (switch_stmt);
    2486      1114045 :       if (TREE_CODE (val) != INTEGER_CST)
    2487              :         return NULL;
    2488              :       else
    2489        23941 :         taken_case = find_case_label_for_value (switch_stmt, val);
    2490              :     }
    2491        23941 :   dest_bb = label_to_block (cfun, CASE_LABEL (taken_case));
    2492              : 
    2493        23941 :   e = find_edge (gimple_bb (switch_stmt), dest_bb);
    2494        23941 :   gcc_assert (e);
    2495              :   return e;
    2496              : }
    2497              : 
    2498              : 
    2499              : /* Return the CASE_LABEL_EXPR that SWITCH_STMT will take for VAL.
    2500              :    We can make optimal use here of the fact that the case labels are
    2501              :    sorted: We can do a binary search for a case matching VAL.  */
    2502              : 
    2503              : tree
    2504        23941 : find_case_label_for_value (const gswitch *switch_stmt, tree val)
    2505              : {
    2506        23941 :   size_t low, high, n = gimple_switch_num_labels (switch_stmt);
    2507        23941 :   tree default_case = gimple_switch_default_label (switch_stmt);
    2508              : 
    2509        74023 :   for (low = 0, high = n; high - low > 1; )
    2510              :     {
    2511        46304 :       size_t i = (high + low) / 2;
    2512        46304 :       tree t = gimple_switch_label (switch_stmt, i);
    2513        46304 :       int cmp;
    2514              : 
    2515              :       /* Cache the result of comparing CASE_LOW and val.  */
    2516        46304 :       cmp = tree_int_cst_compare (CASE_LOW (t), val);
    2517              : 
    2518        46304 :       if (cmp > 0)
    2519              :         high = i;
    2520              :       else
    2521        34482 :         low = i;
    2522              : 
    2523        46304 :       if (CASE_HIGH (t) == NULL)
    2524              :         {
    2525              :           /* A single-valued case label.  */
    2526        45237 :           if (cmp == 0)
    2527              :             return t;
    2528              :         }
    2529              :       else
    2530              :         {
    2531              :           /* A case range.  We can only handle integer ranges.  */
    2532         1067 :           if (cmp <= 0 && tree_int_cst_compare (CASE_HIGH (t), val) >= 0)
    2533              :             return t;
    2534              :         }
    2535              :     }
    2536              : 
    2537              :   return default_case;
    2538              : }
    2539              : 
    2540              : 
    2541              : /* Dump a basic block on stderr.  */
    2542              : 
    2543              : void
    2544            0 : gimple_debug_bb (basic_block bb)
    2545              : {
    2546            0 :   dump_bb (stderr, bb, 0, TDF_VOPS|TDF_MEMSYMS|TDF_BLOCKS);
    2547            0 : }
    2548              : 
    2549              : 
    2550              : /* Dump basic block with index N on stderr.  */
    2551              : 
    2552              : basic_block
    2553            0 : gimple_debug_bb_n (int n)
    2554              : {
    2555            0 :   gimple_debug_bb (BASIC_BLOCK_FOR_FN (cfun, n));
    2556            0 :   return BASIC_BLOCK_FOR_FN (cfun, n);
    2557              : }
    2558              : 
    2559              : 
    2560              : /* Dump the CFG on stderr.
    2561              : 
    2562              :    FLAGS are the same used by the tree dumping functions
    2563              :    (see TDF_* in dumpfile.h).  */
    2564              : 
    2565              : void
    2566            0 : gimple_debug_cfg (dump_flags_t flags)
    2567              : {
    2568            0 :   gimple_dump_cfg (stderr, flags);
    2569            0 : }
    2570              : 
    2571              : 
    2572              : /* Dump the program showing basic block boundaries on the given FILE.
    2573              : 
    2574              :    FLAGS are the same used by the tree dumping functions (see TDF_* in
    2575              :    tree.h).  */
    2576              : 
    2577              : void
    2578          203 : gimple_dump_cfg (FILE *file, dump_flags_t flags)
    2579              : {
    2580          203 :   if (flags & TDF_DETAILS)
    2581              :     {
    2582            2 :       dump_function_header (file, current_function_decl, flags);
    2583            2 :       fprintf (file, ";; \n%d basic blocks, %d edges, last basic block %d.\n\n",
    2584              :                n_basic_blocks_for_fn (cfun), n_edges_for_fn (cfun),
    2585            2 :                last_basic_block_for_fn (cfun));
    2586              : 
    2587            2 :       brief_dump_cfg (file, flags);
    2588            2 :       fprintf (file, "\n");
    2589              :     }
    2590              : 
    2591          203 :   if (flags & TDF_STATS)
    2592           46 :     dump_cfg_stats (file);
    2593              : 
    2594          203 :   dump_function_to_file (current_function_decl, file, flags | TDF_BLOCKS);
    2595          203 : }
    2596              : 
    2597              : 
    2598              : /* Dump CFG statistics on FILE.  */
    2599              : 
    2600              : void
    2601           46 : dump_cfg_stats (FILE *file)
    2602              : {
    2603           46 :   static long max_num_merged_labels = 0;
    2604           46 :   unsigned long size, total = 0;
    2605           46 :   long num_edges;
    2606           46 :   basic_block bb;
    2607           46 :   const char * const fmt_str   = "%-30s%-13s%12s\n";
    2608           46 :   const char * const fmt_str_1 = "%-30s%13d" PRsa (11) "\n";
    2609           46 :   const char * const fmt_str_2 = "%-30s%13ld" PRsa (11) "\n";
    2610           46 :   const char * const fmt_str_3 = "%-43s" PRsa (11) "\n";
    2611           46 :   const char *funcname = current_function_name ();
    2612              : 
    2613           46 :   fprintf (file, "\nCFG Statistics for %s\n\n", funcname);
    2614              : 
    2615           46 :   fprintf (file, "---------------------------------------------------------\n");
    2616           46 :   fprintf (file, fmt_str, "", "  Number of  ", "Memory");
    2617           46 :   fprintf (file, fmt_str, "", "  instances  ", "used ");
    2618           46 :   fprintf (file, "---------------------------------------------------------\n");
    2619              : 
    2620           46 :   size = n_basic_blocks_for_fn (cfun) * sizeof (struct basic_block_def);
    2621           46 :   total += size;
    2622           46 :   fprintf (file, fmt_str_1, "Basic blocks", n_basic_blocks_for_fn (cfun),
    2623            0 :            SIZE_AMOUNT (size));
    2624              : 
    2625           46 :   num_edges = 0;
    2626          129 :   FOR_EACH_BB_FN (bb, cfun)
    2627          166 :     num_edges += EDGE_COUNT (bb->succs);
    2628           46 :   size = num_edges * sizeof (class edge_def);
    2629           46 :   total += size;
    2630           46 :   fprintf (file, fmt_str_2, "Edges", num_edges, SIZE_AMOUNT (size));
    2631              : 
    2632           46 :   fprintf (file, "---------------------------------------------------------\n");
    2633           46 :   fprintf (file, fmt_str_3, "Total memory used by CFG data",
    2634            0 :            SIZE_AMOUNT (total));
    2635           46 :   fprintf (file, "---------------------------------------------------------\n");
    2636           46 :   fprintf (file, "\n");
    2637              : 
    2638           46 :   if (cfg_stats.num_merged_labels > max_num_merged_labels)
    2639            0 :     max_num_merged_labels = cfg_stats.num_merged_labels;
    2640              : 
    2641           46 :   fprintf (file, "Coalesced label blocks: %ld (Max so far: %ld)\n",
    2642              :            cfg_stats.num_merged_labels, max_num_merged_labels);
    2643              : 
    2644           46 :   fprintf (file, "\n");
    2645           46 : }
    2646              : 
    2647              : 
    2648              : /* Dump CFG statistics on stderr.  Keep extern so that it's always
    2649              :    linked in the final executable.  */
    2650              : 
    2651              : DEBUG_FUNCTION void
    2652            0 : debug_cfg_stats (void)
    2653              : {
    2654            0 :   dump_cfg_stats (stderr);
    2655            0 : }
    2656              : 
    2657              : /*---------------------------------------------------------------------------
    2658              :                              Miscellaneous helpers
    2659              : ---------------------------------------------------------------------------*/
    2660              : 
    2661              : /* Return true if T, a GIMPLE_CALL, can make an abnormal transfer of control
    2662              :    flow.  Transfers of control flow associated with EH are excluded.  */
    2663              : 
    2664              : static bool
    2665    148752257 : call_can_make_abnormal_goto (gimple *t)
    2666              : {
    2667              :   /* If the function has no non-local labels, then a call cannot make an
    2668              :      abnormal transfer of control.  */
    2669    148752257 :   if (!cfun->has_nonlocal_label
    2670    148638091 :       && !cfun->calls_setjmp)
    2671              :    return false;
    2672              : 
    2673              :   /* Likewise if the call has no side effects.  */
    2674       229536 :   if (!gimple_has_side_effects (t))
    2675              :     return false;
    2676              : 
    2677              :   /* Likewise if the called function is leaf.  */
    2678       221346 :   if (gimple_call_flags (t) & ECF_LEAF)
    2679        51942 :     return false;
    2680              : 
    2681              :   return true;
    2682              : }
    2683              : 
    2684              : 
    2685              : /* Return true if T can make an abnormal transfer of control flow.
    2686              :    Transfers of control flow associated with EH are excluded.  */
    2687              : 
    2688              : bool
    2689    636605029 : stmt_can_make_abnormal_goto (gimple *t)
    2690              : {
    2691    636605029 :   if (computed_goto_p (t))
    2692              :     return true;
    2693    636600924 :   if (is_gimple_call (t))
    2694    137305245 :     return call_can_make_abnormal_goto (t);
    2695              :   return false;
    2696              : }
    2697              : 
    2698              : 
    2699              : /* Return true if T represents a stmt that always transfers control.  */
    2700              : 
    2701              : bool
    2702  17702093775 : is_ctrl_stmt (gimple *t)
    2703              : {
    2704  17702093775 :   switch (gimple_code (t))
    2705              :     {
    2706              :     case GIMPLE_COND:
    2707              :     case GIMPLE_SWITCH:
    2708              :     case GIMPLE_GOTO:
    2709              :     case GIMPLE_RETURN:
    2710              :     case GIMPLE_RESX:
    2711              :       return true;
    2712  15432055197 :     default:
    2713  15432055197 :       return false;
    2714              :     }
    2715              : }
    2716              : 
    2717              : 
    2718              : /* Return true if T is a statement that may alter the flow of control
    2719              :    (e.g., a call to a non-returning function).  */
    2720              : 
    2721              : bool
    2722  14701821237 : is_ctrl_altering_stmt (gimple *t)
    2723              : {
    2724  14701821237 :   gcc_assert (t);
    2725              : 
    2726  14701821237 :   switch (gimple_code (t))
    2727              :     {
    2728   1127333357 :     case GIMPLE_CALL:
    2729              :       /* Per stmt call flag indicates whether the call could alter
    2730              :          controlflow.  */
    2731   1127333357 :       if (gimple_call_ctrl_altering_p (t))
    2732              :         return true;
    2733              :       break;
    2734              : 
    2735              :     case GIMPLE_EH_DISPATCH:
    2736              :       /* EH_DISPATCH branches to the individual catch handlers at
    2737              :          this level of a try or allowed-exceptions region.  It can
    2738              :          fallthru to the next statement as well.  */
    2739              :       return true;
    2740              : 
    2741     12855957 :     case GIMPLE_ASM:
    2742     12855957 :       if (gimple_asm_nlabels (as_a <gasm *> (t)) > 0)
    2743              :         return true;
    2744              :       break;
    2745              : 
    2746              :     CASE_GIMPLE_OMP:
    2747              :       /* OpenMP directives alter control flow.  */
    2748              :       return true;
    2749              : 
    2750              :     case GIMPLE_TRANSACTION:
    2751              :       /* A transaction start alters control flow.  */
    2752              :       return true;
    2753              : 
    2754              :     default:
    2755              :       break;
    2756              :     }
    2757              : 
    2758              :   /* If a statement can throw, it alters control flow.  */
    2759  14526654268 :   return stmt_can_throw_internal (cfun, t);
    2760              : }
    2761              : 
    2762              : 
    2763              : /* Return true if T is a simple local goto.  */
    2764              : 
    2765              : bool
    2766      6383873 : simple_goto_p (gimple *t)
    2767              : {
    2768      6383873 :   return (gimple_code (t) == GIMPLE_GOTO
    2769      6383873 :           && TREE_CODE (gimple_goto_dest (t)) == LABEL_DECL);
    2770              : }
    2771              : 
    2772              : 
    2773              : /* Return true if STMT should start a new basic block.  PREV_STMT is
    2774              :    the statement preceding STMT.  It is used when STMT is a label or a
    2775              :    case label.  Labels should only start a new basic block if their
    2776              :    previous statement wasn't a label.  Otherwise, sequence of labels
    2777              :    would generate unnecessary basic blocks that only contain a single
    2778              :    label.  */
    2779              : 
    2780              : static inline bool
    2781     73537096 : stmt_starts_bb_p (gimple *stmt, gimple *prev_stmt)
    2782              : {
    2783     73537096 :   if (stmt == NULL)
    2784              :     return false;
    2785              : 
    2786              :   /* PREV_STMT is only set to a debug stmt if the debug stmt is before
    2787              :      any nondebug stmts in the block.  We don't want to start another
    2788              :      block in this case: the debug stmt will already have started the
    2789              :      one STMT would start if we weren't outputting debug stmts.  */
    2790     73537096 :   if (prev_stmt && is_gimple_debug (prev_stmt))
    2791              :     return false;
    2792              : 
    2793              :   /* Labels start a new basic block only if the preceding statement
    2794              :      wasn't a label of the same type.  This prevents the creation of
    2795              :      consecutive blocks that have nothing but a single label.  */
    2796     73090035 :   if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
    2797              :     {
    2798              :       /* Nonlocal and computed GOTO targets always start a new block.  */
    2799      4187563 :       if (DECL_NONLOCAL (gimple_label_label (label_stmt))
    2800      4187563 :           || FORCED_LABEL (gimple_label_label (label_stmt)))
    2801              :         return true;
    2802              : 
    2803      4173743 :       if (glabel *plabel = safe_dyn_cast <glabel *> (prev_stmt))
    2804              :         {
    2805      2268314 :           if (DECL_NONLOCAL (gimple_label_label (plabel))
    2806      2268314 :               || !DECL_ARTIFICIAL (gimple_label_label (plabel)))
    2807              :             return true;
    2808              : 
    2809      2257161 :           cfg_stats.num_merged_labels++;
    2810      2257161 :           return false;
    2811              :         }
    2812              :       else
    2813              :         return true;
    2814              :     }
    2815     68902472 :   else if (gimple_code (stmt) == GIMPLE_CALL)
    2816              :     {
    2817      9650576 :       if (gimple_call_flags (stmt) & ECF_RETURNS_TWICE)
    2818              :         /* setjmp acts similar to a nonlocal GOTO target and thus should
    2819              :            start a new block.  */
    2820              :         return true;
    2821      9649366 :       if (gimple_call_internal_p (stmt, IFN_PHI)
    2822            0 :           && prev_stmt
    2823            0 :           && gimple_code (prev_stmt) != GIMPLE_LABEL
    2824      9649366 :           && (gimple_code (prev_stmt) != GIMPLE_CALL
    2825            0 :               || ! gimple_call_internal_p (prev_stmt, IFN_PHI)))
    2826              :         /* PHI nodes start a new block unless preceded by a label
    2827              :            or another PHI.  */
    2828            0 :         return true;
    2829              :     }
    2830              : 
    2831              :   return false;
    2832              : }
    2833              : 
    2834              : 
    2835              : /* Return true if T should end a basic block.  */
    2836              : 
    2837              : bool
    2838  15672160211 : stmt_ends_bb_p (gimple *t)
    2839              : {
    2840  15672160211 :   return is_ctrl_stmt (t) || is_ctrl_altering_stmt (t);
    2841              : }
    2842              : 
    2843              : /* Remove block annotations and other data structures.  */
    2844              : 
    2845              : void
    2846      3304528 : delete_tree_cfg_annotations (struct function *fn)
    2847              : {
    2848      3304528 :   vec_free (label_to_block_map_for_fn (fn));
    2849      3304528 : }
    2850              : 
    2851              : /* Return the virtual phi in BB.  */
    2852              : 
    2853              : gphi *
    2854   2852775384 : get_virtual_phi (basic_block bb)
    2855              : {
    2856   2852775384 :   for (gphi_iterator gsi = gsi_start_phis (bb);
    2857   3530350691 :        !gsi_end_p (gsi);
    2858    677575307 :        gsi_next (&gsi))
    2859              :     {
    2860   1520408569 :       gphi *phi = gsi.phi ();
    2861              : 
    2862   3040817138 :       if (virtual_operand_p (PHI_RESULT (phi)))
    2863    842833262 :         return phi;
    2864              :     }
    2865              : 
    2866   2009942122 :   return NULL;
    2867              : }
    2868              : 
    2869              : /* Return the first statement in basic block BB.  */
    2870              : 
    2871              : gimple *
    2872    154184271 : first_stmt (basic_block bb)
    2873              : {
    2874    154184271 :   gimple_stmt_iterator i = gsi_start_bb (bb);
    2875    154184271 :   gimple *stmt = NULL;
    2876              : 
    2877    562866123 :   while (!gsi_end_p (i) && is_gimple_debug ((stmt = gsi_stmt (i))))
    2878              :     {
    2879    408681852 :       gsi_next (&i);
    2880    408681852 :       stmt = NULL;
    2881              :     }
    2882    154184271 :   return stmt;
    2883              : }
    2884              : 
    2885              : /* Return the last statement in basic block BB.  */
    2886              : 
    2887              : gimple *
    2888    188980752 : last_nondebug_stmt (basic_block bb)
    2889              : {
    2890    188980752 :   gimple_stmt_iterator i = gsi_last_bb (bb);
    2891    188980752 :   gimple *stmt = NULL;
    2892              : 
    2893    239422407 :   while (!gsi_end_p (i) && is_gimple_debug ((stmt = gsi_stmt (i))))
    2894              :     {
    2895     50441655 :       gsi_prev (&i);
    2896     50441655 :       stmt = NULL;
    2897              :     }
    2898    188980752 :   return stmt;
    2899              : }
    2900              : 
    2901              : /* Return the last statement of an otherwise empty block.  Return NULL
    2902              :    if the block is totally empty, or if it contains more than one
    2903              :    statement.  */
    2904              : 
    2905              : gimple *
    2906     17117654 : last_and_only_stmt (basic_block bb)
    2907              : {
    2908     17117654 :   gimple_stmt_iterator i = gsi_last_nondebug_bb (bb);
    2909     17117654 :   gimple *last, *prev;
    2910              : 
    2911     17117654 :   if (gsi_end_p (i))
    2912              :     return NULL;
    2913              : 
    2914     16653959 :   last = gsi_stmt (i);
    2915     16653959 :   gsi_prev_nondebug (&i);
    2916     16653959 :   if (gsi_end_p (i))
    2917              :     return last;
    2918              : 
    2919              :   /* Empty statements should no longer appear in the instruction stream.
    2920              :      Everything that might have appeared before should be deleted by
    2921              :      remove_useless_stmts, and the optimizers should just gsi_remove
    2922              :      instead of smashing with build_empty_stmt.
    2923              : 
    2924              :      Thus the only thing that should appear here in a block containing
    2925              :      one executable statement is a label.  */
    2926     12679458 :   prev = gsi_stmt (i);
    2927     12679458 :   if (gimple_code (prev) == GIMPLE_LABEL)
    2928              :     return last;
    2929              :   else
    2930     12335254 :     return NULL;
    2931              : }
    2932              : 
    2933              : /* Returns the basic block after which the new basic block created
    2934              :    by splitting edge EDGE_IN should be placed.  Tries to keep the new block
    2935              :    near its "logical" location.  This is of most help to humans looking
    2936              :    at debugging dumps.  */
    2937              : 
    2938              : basic_block
    2939     28167685 : split_edge_bb_loc (edge edge_in)
    2940              : {
    2941     28167685 :   basic_block dest = edge_in->dest;
    2942     28167685 :   basic_block dest_prev = dest->prev_bb;
    2943              : 
    2944     28167685 :   if (dest_prev)
    2945              :     {
    2946     28167685 :       edge e = find_edge (dest_prev, dest);
    2947     28167685 :       if (e && !(e->flags & EDGE_COMPLEX))
    2948     23777743 :         return edge_in->src;
    2949              :     }
    2950              :   return dest_prev;
    2951              : }
    2952              : 
    2953              : /* Split a (typically critical) edge EDGE_IN.  Return the new block.
    2954              :    Abort on abnormal edges.  */
    2955              : 
    2956              : static basic_block
    2957     26270817 : gimple_split_edge (edge edge_in)
    2958              : {
    2959     26270817 :   basic_block new_bb, after_bb, dest;
    2960     26270817 :   edge new_edge, e;
    2961              : 
    2962              :   /* Abnormal edges cannot be split.  */
    2963     26270817 :   gcc_assert (!(edge_in->flags & EDGE_ABNORMAL));
    2964              : 
    2965     26270817 :   dest = edge_in->dest;
    2966              : 
    2967     26270817 :   after_bb = split_edge_bb_loc (edge_in);
    2968              : 
    2969     26270817 :   new_bb = create_empty_bb (after_bb);
    2970     26270817 :   new_bb->count = edge_in->count ();
    2971     26270817 :   if (coverage_suppressed_p (after_bb))
    2972           73 :     suppress_coverage (new_bb);
    2973              : 
    2974              :   /* We want to avoid re-allocating PHIs when we first
    2975              :      add the fallthru edge from new_bb to dest but we also
    2976              :      want to avoid changing PHI argument order when
    2977              :      first redirecting edge_in away from dest.  The former
    2978              :      avoids changing PHI argument order by adding them
    2979              :      last and then the redirection swapping it back into
    2980              :      place by means of unordered remove.
    2981              :      So hack around things by temporarily removing all PHIs
    2982              :      from the destination during the edge redirection and then
    2983              :      making sure the edges stay in order.  */
    2984     26270817 :   gimple_seq saved_phis = phi_nodes (dest);
    2985     26270817 :   unsigned old_dest_idx = edge_in->dest_idx;
    2986     26270817 :   set_phi_nodes (dest, NULL);
    2987     26270817 :   new_edge = make_single_succ_edge (new_bb, dest, EDGE_FALLTHRU);
    2988     26270817 :   e = redirect_edge_and_branch (edge_in, new_bb);
    2989     26270817 :   gcc_assert (e == edge_in && new_edge->dest_idx == old_dest_idx);
    2990              :   /* set_phi_nodes sets the BB of the PHI nodes, so do it manually here.  */
    2991     26270817 :   dest->il.gimple.phi_nodes = saved_phis;
    2992              : 
    2993     26270817 :   return new_bb;
    2994              : }
    2995              : 
    2996              : 
    2997              : /* Verify properties of the address expression T whose base should be
    2998              :    TREE_ADDRESSABLE if VERIFY_ADDRESSABLE is true.  */
    2999              : 
    3000              : static bool
    3001    627428459 : verify_address (tree t, bool verify_addressable)
    3002              : {
    3003    627428459 :   bool old_constant;
    3004    627428459 :   bool old_side_effects;
    3005    627428459 :   bool new_constant;
    3006    627428459 :   bool new_side_effects;
    3007              : 
    3008    627428459 :   old_constant = TREE_CONSTANT (t);
    3009    627428459 :   old_side_effects = TREE_SIDE_EFFECTS (t);
    3010              : 
    3011    627428459 :   recompute_tree_invariant_for_addr_expr (t);
    3012    627428459 :   new_side_effects = TREE_SIDE_EFFECTS (t);
    3013    627428459 :   new_constant = TREE_CONSTANT (t);
    3014              : 
    3015    627428459 :   if (old_constant != new_constant)
    3016              :     {
    3017            0 :       error ("constant not recomputed when %<ADDR_EXPR%> changed");
    3018            0 :       return true;
    3019              :     }
    3020    627428459 :   if (old_side_effects != new_side_effects)
    3021              :     {
    3022            0 :       error ("side effects not recomputed when %<ADDR_EXPR%> changed");
    3023            0 :       return true;
    3024              :     }
    3025              : 
    3026    627428459 :   tree base = TREE_OPERAND (t, 0);
    3027    776863128 :   while (handled_component_p (base))
    3028    149434669 :     base = TREE_OPERAND (base, 0);
    3029              : 
    3030    627428459 :   if (!(VAR_P (base)
    3031              :         || TREE_CODE (base) == PARM_DECL
    3032              :         || TREE_CODE (base) == RESULT_DECL))
    3033              :     return false;
    3034              : 
    3035    457753298 :   if (verify_addressable && !TREE_ADDRESSABLE (base))
    3036              :     {
    3037            0 :       error ("address taken but %<TREE_ADDRESSABLE%> bit not set");
    3038            0 :       return true;
    3039              :     }
    3040              : 
    3041              :   return false;
    3042              : }
    3043              : 
    3044              : 
    3045              : /* Verify if EXPR is a valid GIMPLE reference expression.  If
    3046              :    REQUIRE_LVALUE is true verifies it is an lvalue.  Returns true
    3047              :    if there is an error, otherwise false.  */
    3048              : 
    3049              : static bool
    3050   3825996097 : verify_types_in_gimple_reference (tree expr, bool require_lvalue)
    3051              : {
    3052   3825996097 :   const char *code_name = get_tree_code_name (TREE_CODE (expr));
    3053              : 
    3054   3825996097 :   if (TREE_CODE (expr) == REALPART_EXPR
    3055   3825996097 :       || TREE_CODE (expr) == IMAGPART_EXPR
    3056   3788166982 :       || TREE_CODE (expr) == BIT_FIELD_REF
    3057   3773098868 :       || TREE_CODE (expr) == VIEW_CONVERT_EXPR)
    3058              :     {
    3059     97588089 :       tree op = TREE_OPERAND (expr, 0);
    3060     97588089 :       if (TREE_CODE (expr) != VIEW_CONVERT_EXPR
    3061     97588089 :           && !is_gimple_reg_type (TREE_TYPE (expr)))
    3062              :         {
    3063            0 :           error ("non-scalar %qs", code_name);
    3064            0 :           return true;
    3065              :         }
    3066              : 
    3067     97588089 :       if (TREE_CODE (expr) == BIT_FIELD_REF)
    3068              :         {
    3069     15068114 :           tree t1 = TREE_OPERAND (expr, 1);
    3070     15068114 :           tree t2 = TREE_OPERAND (expr, 2);
    3071     15068114 :           poly_uint64 size, bitpos;
    3072     15068114 :           if (!poly_int_tree_p (t1, &size)
    3073     15068114 :               || !poly_int_tree_p (t2, &bitpos)
    3074     15068114 :               || !types_compatible_p (bitsizetype, TREE_TYPE (t1))
    3075     30136228 :               || !types_compatible_p (bitsizetype, TREE_TYPE (t2)))
    3076              :             {
    3077            0 :               error ("invalid position or size operand to %qs", code_name);
    3078            0 :               return true;
    3079              :             }
    3080     30097808 :           if (INTEGRAL_TYPE_P (TREE_TYPE (expr))
    3081     27605019 :               && maybe_ne (TYPE_PRECISION (TREE_TYPE (expr)), size))
    3082              :             {
    3083            0 :               error ("integral result type precision does not match "
    3084              :                      "field size of %qs", code_name);
    3085            0 :               return true;
    3086              :             }
    3087     15068114 :           else if (!INTEGRAL_TYPE_P (TREE_TYPE (expr))
    3088      2492789 :                    && TYPE_MODE (TREE_TYPE (expr)) != BLKmode
    3089      4982770 :                    && maybe_ne (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (expr))),
    3090              :                                 size))
    3091              :             {
    3092            0 :               error ("mode size of non-integral result does not "
    3093              :                      "match field size of %qs",
    3094              :                      code_name);
    3095            0 :               return true;
    3096              :             }
    3097     30135528 :           if (INTEGRAL_TYPE_P (TREE_TYPE (op))
    3098     15298604 :               && !type_has_mode_precision_p (TREE_TYPE (op)))
    3099              :             {
    3100            0 :               error ("%qs of non-mode-precision operand", code_name);
    3101            0 :               return true;
    3102              :             }
    3103     15068114 :           if (!AGGREGATE_TYPE_P (TREE_TYPE (op))
    3104     15068114 :               && known_gt (size + bitpos,
    3105              :                            tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (op)))))
    3106              :             {
    3107            0 :               error ("position plus size exceeds size of referenced object in "
    3108              :                      "%qs", code_name);
    3109            0 :               return true;
    3110              :             }
    3111              :         }
    3112              : 
    3113     97588089 :       if ((TREE_CODE (expr) == REALPART_EXPR
    3114     97588089 :            || TREE_CODE (expr) == IMAGPART_EXPR)
    3115    135417204 :           && !useless_type_conversion_p (TREE_TYPE (expr),
    3116     37829115 :                                          TREE_TYPE (TREE_TYPE (op))))
    3117              :         {
    3118            0 :           error ("type mismatch in %qs reference", code_name);
    3119            0 :           debug_generic_stmt (TREE_TYPE (expr));
    3120            0 :           debug_generic_stmt (TREE_TYPE (TREE_TYPE (op)));
    3121            0 :           return true;
    3122              :         }
    3123              : 
    3124     97588089 :       if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
    3125              :         {
    3126              :           /* For VIEW_CONVERT_EXPRs which are allowed here too, we only check
    3127              :              that their operand is not a register an invariant when
    3128              :              requiring an lvalue (this usually means there is a SRA or IPA-SRA
    3129              :              bug).  Otherwise there is nothing to verify, gross mismatches at
    3130              :              most invoke undefined behavior.  */
    3131     44690860 :           if (require_lvalue
    3132     44690860 :               && (is_gimple_reg (op) || is_gimple_min_invariant (op)))
    3133              :             {
    3134            0 :               error ("conversion of %qs on the left hand side of %qs",
    3135            0 :                      get_tree_code_name (TREE_CODE (op)), code_name);
    3136            0 :               debug_generic_stmt (expr);
    3137            0 :               return true;
    3138              :             }
    3139     44690860 :           else if (is_gimple_reg (op)
    3140     44690860 :                    && TYPE_SIZE (TREE_TYPE (expr)) != TYPE_SIZE (TREE_TYPE (op)))
    3141              :             {
    3142            0 :               error ("conversion of register to a different size in %qs",
    3143              :                      code_name);
    3144            0 :               debug_generic_stmt (expr);
    3145            0 :               return true;
    3146              :             }
    3147              :         }
    3148              : 
    3149              :       expr = op;
    3150              :     }
    3151              : 
    3152   3825996097 :   bool require_non_reg = false;
    3153   6247653302 :   while (handled_component_p (expr))
    3154              :     {
    3155   2421657205 :       require_non_reg = true;
    3156   2421657205 :       code_name = get_tree_code_name (TREE_CODE (expr));
    3157              : 
    3158   2421657205 :       if (TREE_CODE (expr) == REALPART_EXPR
    3159   2421657205 :           || TREE_CODE (expr) == IMAGPART_EXPR
    3160   2421657205 :           || TREE_CODE (expr) == BIT_FIELD_REF)
    3161              :         {
    3162            0 :           error ("non-top-level %qs", code_name);
    3163            0 :           return true;
    3164              :         }
    3165              : 
    3166   2421657205 :       tree op = TREE_OPERAND (expr, 0);
    3167              : 
    3168   2421657205 :       if (TREE_CODE (expr) == ARRAY_REF
    3169   2421657205 :           || TREE_CODE (expr) == ARRAY_RANGE_REF)
    3170              :         {
    3171    455970901 :           if (!is_gimple_val (TREE_OPERAND (expr, 1))
    3172    455970901 :               || (TREE_OPERAND (expr, 2)
    3173      3622958 :                   && !is_gimple_val (TREE_OPERAND (expr, 2)))
    3174    911941802 :               || (TREE_OPERAND (expr, 3)
    3175       688649 :                   && !is_gimple_val (TREE_OPERAND (expr, 3))))
    3176              :             {
    3177            0 :               error ("invalid operands to %qs", code_name);
    3178            0 :               debug_generic_stmt (expr);
    3179            0 :               return true;
    3180              :             }
    3181              :         }
    3182              : 
    3183              :       /* Verify if the reference array element types are compatible.  */
    3184   2421657205 :       if (TREE_CODE (expr) == ARRAY_REF
    3185   2877319876 :           && !useless_type_conversion_p (TREE_TYPE (expr),
    3186    455662671 :                                          TREE_TYPE (TREE_TYPE (op))))
    3187              :         {
    3188            0 :           error ("type mismatch in %qs", code_name);
    3189            0 :           debug_generic_stmt (TREE_TYPE (expr));
    3190            0 :           debug_generic_stmt (TREE_TYPE (TREE_TYPE (op)));
    3191            0 :           return true;
    3192              :         }
    3193   2421657205 :       if (TREE_CODE (expr) == ARRAY_RANGE_REF
    3194   2421965435 :           && !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (expr)),
    3195       308230 :                                          TREE_TYPE (TREE_TYPE (op))))
    3196              :         {
    3197            0 :           error ("type mismatch in %qs", code_name);
    3198            0 :           debug_generic_stmt (TREE_TYPE (TREE_TYPE (expr)));
    3199            0 :           debug_generic_stmt (TREE_TYPE (TREE_TYPE (op)));
    3200            0 :           return true;
    3201              :         }
    3202              : 
    3203   2421657205 :       if (TREE_CODE (expr) == COMPONENT_REF)
    3204              :         {
    3205   1962905538 :           if (TREE_OPERAND (expr, 2)
    3206   1962905538 :               && !is_gimple_val (TREE_OPERAND (expr, 2)))
    3207              :             {
    3208            0 :               error ("invalid %qs offset operator", code_name);
    3209            0 :               return true;
    3210              :             }
    3211   1962905538 :           if (!useless_type_conversion_p (TREE_TYPE (expr),
    3212   1962905538 :                                           TREE_TYPE (TREE_OPERAND (expr, 1))))
    3213              :             {
    3214            0 :               error ("type mismatch in %qs", code_name);
    3215            0 :               debug_generic_stmt (TREE_TYPE (expr));
    3216            0 :               debug_generic_stmt (TREE_TYPE (TREE_OPERAND (expr, 1)));
    3217            0 :               return true;
    3218              :             }
    3219              :         }
    3220              : 
    3221   2421657205 :       expr = op;
    3222              :     }
    3223              : 
    3224   3825996097 :   code_name = get_tree_code_name (TREE_CODE (expr));
    3225              : 
    3226   3825996097 :   if (TREE_CODE (expr) == MEM_REF)
    3227              :     {
    3228   1295300318 :       if (!is_gimple_mem_ref_addr (TREE_OPERAND (expr, 0))
    3229   1295300318 :           || (TREE_CODE (TREE_OPERAND (expr, 0)) == ADDR_EXPR
    3230    353543818 :               && verify_address (TREE_OPERAND (expr, 0), false)))
    3231              :         {
    3232            0 :           error ("invalid address operand in %qs", code_name);
    3233            0 :           debug_generic_stmt (expr);
    3234            0 :           return true;
    3235              :         }
    3236   1295300318 :       if (!poly_int_tree_p (TREE_OPERAND (expr, 1))
    3237   1295300318 :           || !POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (expr, 1))))
    3238              :         {
    3239            0 :           error ("invalid offset operand in %qs", code_name);
    3240            0 :           debug_generic_stmt (expr);
    3241            0 :           return true;
    3242              :         }
    3243   1295300318 :       if (MR_DEPENDENCE_CLIQUE (expr) != 0
    3244   1295300318 :           && MR_DEPENDENCE_CLIQUE (expr) > cfun->last_clique)
    3245              :         {
    3246            0 :           error ("invalid clique in %qs", code_name);
    3247            0 :           debug_generic_stmt (expr);
    3248            0 :           return true;
    3249              :         }
    3250              :     }
    3251   2530695779 :   else if (TREE_CODE (expr) == TARGET_MEM_REF)
    3252              :     {
    3253     29521302 :       if (!TMR_BASE (expr)
    3254     29521302 :           || !is_gimple_mem_ref_addr (TMR_BASE (expr))
    3255     59042604 :           || (TREE_CODE (TMR_BASE (expr)) == ADDR_EXPR
    3256      6229896 :               && verify_address (TMR_BASE (expr), false)))
    3257              :         {
    3258            0 :           error ("invalid address operand in %qs", code_name);
    3259            0 :           return true;
    3260              :         }
    3261     29521302 :       if (!TMR_OFFSET (expr)
    3262     29521302 :           || !poly_int_tree_p (TMR_OFFSET (expr))
    3263     59042604 :           || !POINTER_TYPE_P (TREE_TYPE (TMR_OFFSET (expr))))
    3264              :         {
    3265            0 :           error ("invalid offset operand in %qs", code_name);
    3266            0 :           debug_generic_stmt (expr);
    3267            0 :           return true;
    3268              :         }
    3269     29521302 :       if (MR_DEPENDENCE_CLIQUE (expr) != 0
    3270     29521302 :           && MR_DEPENDENCE_CLIQUE (expr) > cfun->last_clique)
    3271              :         {
    3272            0 :           error ("invalid clique in %qs", code_name);
    3273            0 :           debug_generic_stmt (expr);
    3274            0 :           return true;
    3275              :         }
    3276              :     }
    3277   2501174477 :   else if (INDIRECT_REF_P (expr))
    3278              :     {
    3279            0 :       error ("%qs in gimple IL", code_name);
    3280            0 :       debug_generic_stmt (expr);
    3281            0 :       return true;
    3282              :     }
    3283   2501174477 :   else if (require_non_reg
    3284   2501174477 :            && (is_gimple_reg (expr)
    3285    927640628 :                || (is_gimple_min_invariant (expr)
    3286              :                    /* STRING_CSTs are representatives of the string table
    3287              :                       entry which lives in memory.  */
    3288      9057739 :                    && TREE_CODE (expr) != STRING_CST)))
    3289              :     {
    3290            0 :       error ("%qs as base where non-register is required", code_name);
    3291            0 :       debug_generic_stmt (expr);
    3292            0 :       return true;
    3293              :     }
    3294              : 
    3295   3825996097 :   if (!require_lvalue
    3296   3825996097 :       && (is_gimple_reg (expr) || is_gimple_min_invariant (expr)))
    3297              :     return false;
    3298              : 
    3299   2589388895 :   if (TREE_CODE (expr) != SSA_NAME && is_gimple_id (expr))
    3300              :     return false;
    3301              : 
    3302   1324821620 :   if (TREE_CODE (expr) != TARGET_MEM_REF
    3303   1324821620 :       && TREE_CODE (expr) != MEM_REF)
    3304              :     {
    3305            0 :       error ("invalid expression for min lvalue");
    3306            0 :       return true;
    3307              :     }
    3308              : 
    3309              :   return false;
    3310              : }
    3311              : 
    3312              : /* Returns true if there is one pointer type in TYPE_POINTER_TO (SRC_OBJ)
    3313              :    list of pointer-to types that is trivially convertible to DEST.  */
    3314              : 
    3315              : static bool
    3316          362 : one_pointer_to_useless_type_conversion_p (tree dest, tree src_obj)
    3317              : {
    3318          362 :   tree src;
    3319              : 
    3320          362 :   if (!TYPE_POINTER_TO (src_obj))
    3321              :     return true;
    3322              : 
    3323          362 :   for (src = TYPE_POINTER_TO (src_obj); src; src = TYPE_NEXT_PTR_TO (src))
    3324          362 :     if (useless_type_conversion_p (dest, src))
    3325              :       return true;
    3326              : 
    3327              :   return false;
    3328              : }
    3329              : 
    3330              : /* Return true if TYPE1 is a fixed-point type and if conversions to and
    3331              :    from TYPE2 can be handled by FIXED_CONVERT_EXPR.  */
    3332              : 
    3333              : static bool
    3334            0 : valid_fixed_convert_types_p (tree type1, tree type2)
    3335              : {
    3336            0 :   return (FIXED_POINT_TYPE_P (type1)
    3337            0 :           && (INTEGRAL_TYPE_P (type2)
    3338            0 :               || SCALAR_FLOAT_TYPE_P (type2)
    3339            0 :               || FIXED_POINT_TYPE_P (type2)));
    3340              : }
    3341              : 
    3342              : /* Verify the contents of a GIMPLE_CALL STMT.  Returns true when there
    3343              :    is a problem, otherwise false.  */
    3344              : 
    3345              : static bool
    3346   1101289116 : verify_gimple_call (gcall *stmt)
    3347              : {
    3348   1101289116 :   tree fn = gimple_call_fn (stmt);
    3349   1101289116 :   tree fntype, fndecl;
    3350   1101289116 :   unsigned i;
    3351              : 
    3352   1101289116 :   if (gimple_call_internal_p (stmt))
    3353              :     {
    3354     42057505 :       if (fn)
    3355              :         {
    3356            0 :           error ("gimple call has two targets");
    3357            0 :           debug_generic_stmt (fn);
    3358            0 :           return true;
    3359              :         }
    3360              :     }
    3361              :   else
    3362              :     {
    3363   1059231611 :       if (!fn)
    3364              :         {
    3365            0 :           error ("gimple call has no target");
    3366            0 :           return true;
    3367              :         }
    3368              :     }
    3369              : 
    3370   1059231611 :   if (fn && !is_gimple_call_addr (fn))
    3371              :     {
    3372            0 :       error ("invalid function in gimple call");
    3373            0 :       debug_generic_stmt (fn);
    3374            0 :       return true;
    3375              :     }
    3376              : 
    3377   1101289116 :   if (fn
    3378   1101289116 :       && (!POINTER_TYPE_P (TREE_TYPE (fn))
    3379   1059231611 :           || (TREE_CODE (TREE_TYPE (TREE_TYPE (fn))) != FUNCTION_TYPE
    3380    177450342 :               && TREE_CODE (TREE_TYPE (TREE_TYPE (fn))) != METHOD_TYPE)))
    3381              :     {
    3382            0 :       error ("non-function in gimple call");
    3383            0 :       return true;
    3384              :     }
    3385              : 
    3386   1101289116 :    fndecl = gimple_call_fndecl (stmt);
    3387   1101289116 :    if (fndecl
    3388   1032904656 :        && TREE_CODE (fndecl) == FUNCTION_DECL
    3389   1032904656 :        && DECL_LOOPING_CONST_OR_PURE_P (fndecl)
    3390      6104634 :        && !DECL_PURE_P (fndecl)
    3391   1101660495 :        && !TREE_READONLY (fndecl))
    3392              :      {
    3393            0 :        error ("invalid pure const state for function");
    3394            0 :        return true;
    3395              :      }
    3396              : 
    3397   1101289116 :   tree lhs = gimple_call_lhs (stmt);
    3398   1101289116 :   if (lhs
    3399   1101289116 :       && (!is_gimple_reg (lhs)
    3400     73038727 :           && (!is_gimple_lvalue (lhs)
    3401     73038727 :               || verify_types_in_gimple_reference
    3402     73038727 :                    (TREE_CODE (lhs) == WITH_SIZE_EXPR
    3403            0 :                     ? TREE_OPERAND (lhs, 0) : lhs, true))))
    3404              :     {
    3405            0 :       error ("invalid LHS in gimple call");
    3406            0 :       return true;
    3407              :     }
    3408              : 
    3409   1101289116 :   if (gimple_call_ctrl_altering_p (stmt)
    3410    152149677 :       && gimple_call_noreturn_p (stmt)
    3411   1248324573 :       && should_remove_lhs_p (lhs))
    3412              :     {
    3413            0 :       error ("LHS in %<noreturn%> call");
    3414            0 :       return true;
    3415              :     }
    3416              : 
    3417   1101289116 :   fntype = gimple_call_fntype (stmt);
    3418   1101289116 :   if (fntype
    3419   1101289116 :       && lhs
    3420    401495452 :       && !useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (fntype))
    3421              :       /* ???  At least C++ misses conversions at assignments from
    3422              :          void * call results.
    3423              :          For now simply allow arbitrary pointer type conversions.  */
    3424   1101289116 :       && !(POINTER_TYPE_P (TREE_TYPE (lhs))
    3425            0 :            && POINTER_TYPE_P (TREE_TYPE (fntype))))
    3426              :     {
    3427            0 :       error ("invalid conversion in gimple call");
    3428            0 :       debug_generic_stmt (TREE_TYPE (lhs));
    3429            0 :       debug_generic_stmt (TREE_TYPE (fntype));
    3430            0 :       return true;
    3431              :     }
    3432              : 
    3433   1101289116 :   if (gimple_call_chain (stmt)
    3434   1101289116 :       && !is_gimple_val (gimple_call_chain (stmt)))
    3435              :     {
    3436            0 :       error ("invalid static chain in gimple call");
    3437            0 :       debug_generic_stmt (gimple_call_chain (stmt));
    3438            0 :       return true;
    3439              :     }
    3440              : 
    3441              :   /* If there is a static chain argument, the call should either be
    3442              :      indirect, or the decl should have DECL_STATIC_CHAIN set.  */
    3443   1101289116 :   if (gimple_call_chain (stmt)
    3444      5005422 :       && fndecl
    3445   1102817983 :       && !DECL_STATIC_CHAIN (fndecl))
    3446              :     {
    3447            0 :       error ("static chain with function that doesn%'t use one");
    3448            0 :       return true;
    3449              :     }
    3450              : 
    3451   1101289116 :   if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
    3452              :     {
    3453    259101888 :       switch (DECL_FUNCTION_CODE (fndecl))
    3454              :         {
    3455     26562846 :         case BUILT_IN_UNREACHABLE:
    3456     26562846 :         case BUILT_IN_UNREACHABLE_TRAP:
    3457     26562846 :         case BUILT_IN_TRAP:
    3458     26562846 :           if (gimple_call_num_args (stmt) > 0)
    3459              :             {
    3460              :               /* Built-in unreachable with parameters might not be caught by
    3461              :                  undefined behavior sanitizer.  Front-ends do check users do not
    3462              :                  call them that way but we also produce calls to
    3463              :                  __builtin_unreachable internally, for example when IPA figures
    3464              :                  out a call cannot happen in a legal program.  In such cases,
    3465              :                  we must make sure arguments are stripped off.  */
    3466            0 :               error ("%<__builtin_unreachable%> or %<__builtin_trap%> call "
    3467              :                      "with arguments");
    3468            0 :               return true;
    3469              :             }
    3470              :           break;
    3471              :         default:
    3472              :           break;
    3473              :         }
    3474              :     }
    3475              : 
    3476              :   /* For a call to .DEFERRED_INIT,
    3477              :      LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL)
    3478              :      we should guarantee that when the 1st argument is a constant, it should
    3479              :      be the same as the size of the LHS.  */
    3480              : 
    3481   1101289116 :   if (gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
    3482              :     {
    3483      9940317 :       tree size_of_arg0 = gimple_call_arg (stmt, 0);
    3484      9940317 :       tree size_of_lhs = TYPE_SIZE_UNIT (TREE_TYPE (lhs));
    3485              : 
    3486      9940317 :       if (TREE_CODE (lhs) == SSA_NAME)
    3487      9940317 :         lhs = SSA_NAME_VAR (lhs);
    3488              : 
    3489      9940317 :       poly_uint64 size_from_arg0, size_from_lhs;
    3490      9940317 :       bool is_constant_size_arg0 = poly_int_tree_p (size_of_arg0,
    3491              :                                                     &size_from_arg0);
    3492      9940317 :       bool is_constant_size_lhs = poly_int_tree_p (size_of_lhs,
    3493              :                                                    &size_from_lhs);
    3494      9940317 :       if (is_constant_size_arg0 && is_constant_size_lhs)
    3495      9929758 :         if (maybe_ne (size_from_arg0, size_from_lhs))
    3496              :           {
    3497            0 :             error ("%<DEFERRED_INIT%> calls should have same "
    3498              :                    "constant size for the first argument and LHS");
    3499            0 :             return true;
    3500              :           }
    3501              :     }
    3502              : 
    3503              :   /* IFN_VARING is not allowed to be present after the completion of any pass
    3504              :      as it should have been replaced.  */
    3505   1101289116 :   if (gimple_call_internal_p (stmt, IFN_VARYING))
    3506              :     {
    3507            0 :       error ("%<.VARYING%> calls should have been replaced and are not allowed "
    3508              :              "outside of the pass that introduced them");
    3509            0 :       return true;
    3510              :     }
    3511              : 
    3512   1101289116 :   if (gimple_call_internal_p (stmt))
    3513              :     {
    3514     42057505 :       unsigned nargs = gimple_call_num_args (stmt);
    3515     84029521 :       tree *args = nargs ? gimple_call_arg_ptr (stmt, 0) : NULL;
    3516     42057505 :       if (!check_internal_fn_args (gimple_location (stmt),
    3517              :                                    gimple_call_internal_fn (stmt), nargs, args))
    3518              :         return true;
    3519              :     }
    3520              : 
    3521              :   /* ???  The C frontend passes unpromoted arguments in case it
    3522              :      didn't see a function declaration before the call.  So for now
    3523              :      leave the call arguments mostly unverified.  Once we gimplify
    3524              :      unit-at-a-time we have a chance to fix this.  */
    3525   3286416411 :   for (i = 0; i < gimple_call_num_args (stmt); ++i)
    3526              :     {
    3527   2185127296 :       tree arg = gimple_call_arg (stmt, i);
    3528   2185127296 :       if ((is_gimple_reg_type (TREE_TYPE (arg))
    3529   2064680698 :            && !is_gimple_val (arg))
    3530   4249807993 :           || (!is_gimple_reg_type (TREE_TYPE (arg))
    3531    120446598 :               && !is_gimple_lvalue (arg)))
    3532              :         {
    3533            1 :           error ("invalid argument to gimple call");
    3534            1 :           debug_generic_expr (arg);
    3535            1 :           return true;
    3536              :         }
    3537   2185127295 :       if (!is_gimple_reg (arg))
    3538              :         {
    3539   1295166204 :           if (TREE_CODE (arg) == WITH_SIZE_EXPR)
    3540        22351 :             arg = TREE_OPERAND (arg, 0);
    3541   1295166204 :           if (verify_types_in_gimple_reference (arg, false))
    3542              :             return true;
    3543              :         }
    3544              :     }
    3545              : 
    3546              :   return false;
    3547              : }
    3548              : 
    3549              : /* Verifies the gimple comparison with the result type TYPE and
    3550              :    the operands OP0 and OP1, comparison code is CODE.  */
    3551              : 
    3552              : static bool
    3553    861869670 : verify_gimple_comparison (tree type, tree op0, tree op1, enum tree_code code)
    3554              : {
    3555    861869670 :   tree op0_type = TREE_TYPE (op0);
    3556    861869670 :   tree op1_type = TREE_TYPE (op1);
    3557              : 
    3558    861869670 :   if (!is_gimple_val (op0) || !is_gimple_val (op1))
    3559              :     {
    3560            0 :       error ("invalid operands in gimple comparison");
    3561            0 :       return true;
    3562              :     }
    3563              : 
    3564              :   /* For comparisons we do not have the operations type as the
    3565              :      effective type the comparison is carried out in.  Instead
    3566              :      we require that either the first operand is trivially
    3567              :      convertible into the second, or the other way around.  */
    3568    861869670 :   if (!useless_type_conversion_p (op0_type, op1_type)
    3569    861869670 :       && !useless_type_conversion_p (op1_type, op0_type))
    3570              :     {
    3571            0 :       error ("mismatching comparison operand types");
    3572            0 :       debug_generic_expr (op0_type);
    3573            0 :       debug_generic_expr (op1_type);
    3574            0 :       return true;
    3575              :     }
    3576              : 
    3577              :   /* The resulting type of a comparison may be an effective boolean type.  */
    3578    861869670 :   if (INTEGRAL_TYPE_P (type)
    3579    861869670 :       && (TREE_CODE (type) == BOOLEAN_TYPE
    3580         1849 :           || TYPE_PRECISION (type) == 1))
    3581              :     {
    3582    859890613 :       if ((VECTOR_TYPE_P (op0_type)
    3583    859786194 :            || VECTOR_TYPE_P (op1_type))
    3584       104419 :           && code != EQ_EXPR && code != NE_EXPR
    3585            0 :           && !VECTOR_BOOLEAN_TYPE_P (op0_type)
    3586    859890613 :           && !VECTOR_INTEGER_TYPE_P (op0_type))
    3587              :         {
    3588            0 :           error ("unsupported operation or type for vector comparison"
    3589              :                  " returning a boolean");
    3590            0 :           debug_generic_expr (op0_type);
    3591            0 :           debug_generic_expr (op1_type);
    3592            0 :           return true;
    3593              :         }
    3594              :     }
    3595              :   /* Or a boolean vector type with the same element count
    3596              :      as the comparison operand types.  */
    3597      1979057 :   else if (VECTOR_TYPE_P (type)
    3598      1979057 :            && TREE_CODE (TREE_TYPE (type)) == BOOLEAN_TYPE)
    3599              :     {
    3600      1979057 :       if (TREE_CODE (op0_type) != VECTOR_TYPE
    3601      1979057 :           || TREE_CODE (op1_type) != VECTOR_TYPE)
    3602              :         {
    3603            0 :           error ("non-vector operands in vector comparison");
    3604            0 :           debug_generic_expr (op0_type);
    3605            0 :           debug_generic_expr (op1_type);
    3606            0 :           return true;
    3607              :         }
    3608              : 
    3609      1979057 :       if (maybe_ne (TYPE_VECTOR_SUBPARTS (type),
    3610      3958114 :                     TYPE_VECTOR_SUBPARTS (op0_type)))
    3611              :         {
    3612            0 :           error ("invalid vector comparison resulting type");
    3613            0 :           debug_generic_expr (type);
    3614            0 :           return true;
    3615              :         }
    3616              :     }
    3617              :   else
    3618              :     {
    3619            0 :       error ("bogus comparison result type");
    3620            0 :       debug_generic_expr (type);
    3621            0 :       return true;
    3622              :     }
    3623              : 
    3624              :   return false;
    3625              : }
    3626              : 
    3627              : /* Verify a gimple assignment statement STMT with an unary rhs.
    3628              :    Returns true if anything is wrong.  */
    3629              : 
    3630              : static bool
    3631    427863773 : verify_gimple_assign_unary (gassign *stmt)
    3632              : {
    3633    427863773 :   enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
    3634    427863773 :   tree lhs = gimple_assign_lhs (stmt);
    3635    427863773 :   tree lhs_type = TREE_TYPE (lhs);
    3636    427863773 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    3637    427863773 :   tree rhs1_type = TREE_TYPE (rhs1);
    3638              : 
    3639    427863773 :   if (!is_gimple_reg (lhs))
    3640              :     {
    3641            0 :       error ("non-register as LHS of unary operation");
    3642            0 :       return true;
    3643              :     }
    3644              : 
    3645    427863773 :   if (!is_gimple_val (rhs1))
    3646              :     {
    3647            0 :       error ("invalid operand in unary operation");
    3648            0 :       return true;
    3649              :     }
    3650              : 
    3651    427863773 :   const char* const code_name = get_tree_code_name (rhs_code);
    3652              : 
    3653              :   /* First handle conversions.  */
    3654    427863773 :   switch (rhs_code)
    3655              :     {
    3656    379006826 :     CASE_CONVERT:
    3657    379006826 :       {
    3658              :         /* Allow conversions between vectors with the same number of elements,
    3659              :            provided that the conversion is OK for the element types too.  */
    3660    379006826 :         if (VECTOR_TYPE_P (lhs_type)
    3661       143574 :             && VECTOR_TYPE_P (rhs1_type)
    3662    379150400 :             && known_eq (TYPE_VECTOR_SUBPARTS (lhs_type),
    3663              :                          TYPE_VECTOR_SUBPARTS (rhs1_type)))
    3664              :           {
    3665       143574 :             lhs_type = TREE_TYPE (lhs_type);
    3666       143574 :             rhs1_type = TREE_TYPE (rhs1_type);
    3667              :           }
    3668    378863252 :         else if (VECTOR_TYPE_P (lhs_type) || VECTOR_TYPE_P (rhs1_type))
    3669              :           {
    3670            0 :             error ("invalid vector types in nop conversion");
    3671            0 :             debug_generic_expr (lhs_type);
    3672            0 :             debug_generic_expr (rhs1_type);
    3673            0 :             return true;
    3674              :           }
    3675              : 
    3676              :         /* Allow conversions from pointer type to integral type only if
    3677              :            there is no sign or zero extension involved.
    3678              :            For targets were the precision of ptrofftype doesn't match that
    3679              :            of pointers we allow conversions to types where
    3680              :            POINTERS_EXTEND_UNSIGNED specifies how that works.  */
    3681    379006826 :         if ((POINTER_TYPE_P (lhs_type)
    3682     23574111 :              && INTEGRAL_TYPE_P (rhs1_type))
    3683    383114112 :             || (POINTER_TYPE_P (rhs1_type)
    3684     48148835 :                 && INTEGRAL_TYPE_P (lhs_type)
    3685     44041549 :                 && (TYPE_PRECISION (rhs1_type) >= TYPE_PRECISION (lhs_type)
    3686              : #if defined(POINTERS_EXTEND_UNSIGNED)
    3687            0 :                     || (TYPE_MODE (rhs1_type) == ptr_mode
    3688            0 :                         && (TYPE_PRECISION (lhs_type)
    3689            0 :                               == BITS_PER_WORD /* word_mode */
    3690            0 :                             || (TYPE_PRECISION (lhs_type)
    3691            0 :                                   == GET_MODE_PRECISION (Pmode))))
    3692              : #endif
    3693              :                    )))
    3694              :           return false;
    3695              : 
    3696              :         /* Allow conversion from integral to offset type and vice versa.  */
    3697    315498452 :         if ((TREE_CODE (lhs_type) == OFFSET_TYPE
    3698         7586 :              && INTEGRAL_TYPE_P (rhs1_type))
    3699    315497296 :             || (INTEGRAL_TYPE_P (lhs_type)
    3700    291314305 :                 && TREE_CODE (rhs1_type) == OFFSET_TYPE))
    3701              :           return false;
    3702              : 
    3703              :         /* Otherwise assert we are converting between types of the
    3704              :            same kind.  */
    3705    315464684 :         if (INTEGRAL_TYPE_P (lhs_type) != INTEGRAL_TYPE_P (rhs1_type))
    3706              :           {
    3707            0 :             error ("invalid types in nop conversion");
    3708            0 :             debug_generic_expr (lhs_type);
    3709            0 :             debug_generic_expr (rhs1_type);
    3710            0 :             return true;
    3711              :           }
    3712              : 
    3713              :         return false;
    3714              :       }
    3715              : 
    3716            0 :     case ADDR_SPACE_CONVERT_EXPR:
    3717            0 :       {
    3718            0 :         if (!POINTER_TYPE_P (rhs1_type) || !POINTER_TYPE_P (lhs_type)
    3719            0 :             || (TYPE_ADDR_SPACE (TREE_TYPE (rhs1_type))
    3720            0 :                 == TYPE_ADDR_SPACE (TREE_TYPE (lhs_type))))
    3721              :           {
    3722            0 :             error ("invalid types in address space conversion");
    3723            0 :             debug_generic_expr (lhs_type);
    3724            0 :             debug_generic_expr (rhs1_type);
    3725            0 :             return true;
    3726              :           }
    3727              : 
    3728              :         return false;
    3729              :       }
    3730              : 
    3731            0 :     case FIXED_CONVERT_EXPR:
    3732            0 :       {
    3733            0 :         if (!valid_fixed_convert_types_p (lhs_type, rhs1_type)
    3734    427863773 :             && !valid_fixed_convert_types_p (rhs1_type, lhs_type))
    3735              :           {
    3736            0 :             error ("invalid types in fixed-point conversion");
    3737            0 :             debug_generic_expr (lhs_type);
    3738            0 :             debug_generic_expr (rhs1_type);
    3739            0 :             return true;
    3740              :           }
    3741              : 
    3742              :         return false;
    3743              :       }
    3744              : 
    3745     16710074 :     case FLOAT_EXPR:
    3746     16710074 :       {
    3747     16613778 :         if ((!INTEGRAL_TYPE_P (rhs1_type) || !SCALAR_FLOAT_TYPE_P (lhs_type))
    3748     16710074 :             && (!VECTOR_INTEGER_TYPE_P (rhs1_type)
    3749        96296 :                 || !VECTOR_FLOAT_TYPE_P (lhs_type)))
    3750              :           {
    3751            0 :             error ("invalid types in conversion to floating-point");
    3752            0 :             debug_generic_expr (lhs_type);
    3753            0 :             debug_generic_expr (rhs1_type);
    3754            0 :             return true;
    3755              :           }
    3756              : 
    3757              :         return false;
    3758              :       }
    3759              : 
    3760      5233097 :     case FIX_TRUNC_EXPR:
    3761      5233097 :       {
    3762      5207921 :         if ((!INTEGRAL_TYPE_P (lhs_type) || !SCALAR_FLOAT_TYPE_P (rhs1_type))
    3763      5233097 :             && (!VECTOR_INTEGER_TYPE_P (lhs_type)
    3764        25176 :                 || !VECTOR_FLOAT_TYPE_P (rhs1_type)))
    3765              :           {
    3766            0 :             error ("invalid types in conversion to integer");
    3767            0 :             debug_generic_expr (lhs_type);
    3768            0 :             debug_generic_expr (rhs1_type);
    3769            0 :             return true;
    3770              :           }
    3771              : 
    3772              :         return false;
    3773              :       }
    3774              : 
    3775       852470 :     case VEC_UNPACK_HI_EXPR:
    3776       852470 :     case VEC_UNPACK_LO_EXPR:
    3777       852470 :     case VEC_UNPACK_FLOAT_HI_EXPR:
    3778       852470 :     case VEC_UNPACK_FLOAT_LO_EXPR:
    3779       852470 :     case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
    3780       852470 :     case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
    3781       852470 :       if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    3782       852470 :           || TREE_CODE (lhs_type) != VECTOR_TYPE
    3783       852470 :           || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
    3784       126992 :               && !SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs_type)))
    3785       852470 :           || (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
    3786        39410 :               && !SCALAR_FLOAT_TYPE_P (TREE_TYPE (rhs1_type)))
    3787       852470 :           || ((rhs_code == VEC_UNPACK_HI_EXPR
    3788       852470 :                || rhs_code == VEC_UNPACK_LO_EXPR)
    3789      1524032 :               && (INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
    3790       762016 :                   != INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))))
    3791       852470 :           || ((rhs_code == VEC_UNPACK_FLOAT_HI_EXPR
    3792       852470 :                || rhs_code == VEC_UNPACK_FLOAT_LO_EXPR)
    3793        89018 :               && (INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
    3794        89018 :                   || SCALAR_FLOAT_TYPE_P (TREE_TYPE (rhs1_type))))
    3795       852470 :           || ((rhs_code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
    3796       852470 :                || rhs_code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
    3797         1436 :               && (INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
    3798         1436 :                   || SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs_type))))
    3799      1704940 :           || (maybe_ne (GET_MODE_SIZE (element_mode (lhs_type)),
    3800      1704940 :                         2 * GET_MODE_SIZE (element_mode (rhs1_type)))
    3801        88718 :               && (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
    3802        88718 :                   || !VECTOR_BOOLEAN_TYPE_P (rhs1_type)))
    3803      1704940 :           || maybe_ne (2 * TYPE_VECTOR_SUBPARTS (lhs_type),
    3804      1704940 :                        TYPE_VECTOR_SUBPARTS (rhs1_type)))
    3805              :         {
    3806            0 :           error ("type mismatch in %qs expression", code_name);
    3807            0 :           debug_generic_expr (lhs_type);
    3808            0 :           debug_generic_expr (rhs1_type);
    3809            0 :           return true;
    3810              :         }
    3811              : 
    3812              :       return false;
    3813              : 
    3814       142373 :     case ABSU_EXPR:
    3815        32218 :       if (!ANY_INTEGRAL_TYPE_P (lhs_type)
    3816       142373 :           || !TYPE_UNSIGNED (lhs_type)
    3817       142373 :           || !ANY_INTEGRAL_TYPE_P (rhs1_type)
    3818       142373 :           || TYPE_UNSIGNED (rhs1_type)
    3819       284746 :           || element_precision (lhs_type) != element_precision (rhs1_type))
    3820              :         {
    3821            0 :           error ("invalid types for %qs", code_name);
    3822            0 :           debug_generic_expr (lhs_type);
    3823            0 :           debug_generic_expr (rhs1_type);
    3824            0 :           return true;
    3825              :         }
    3826              :       return false;
    3827              : 
    3828            0 :     case VEC_DUPLICATE_EXPR:
    3829            0 :       if (TREE_CODE (lhs_type) != VECTOR_TYPE
    3830            0 :           || !useless_type_conversion_p (TREE_TYPE (lhs_type), rhs1_type))
    3831              :         {
    3832            0 :           error ("%qs should be from a scalar to a like vector", code_name);
    3833            0 :           debug_generic_expr (lhs_type);
    3834            0 :           debug_generic_expr (rhs1_type);
    3835            0 :           return true;
    3836              :         }
    3837              :       return false;
    3838              : 
    3839        39290 :     case CONJ_EXPR:
    3840        39290 :       if (TREE_CODE (lhs_type) != COMPLEX_TYPE)
    3841              :         {
    3842            0 : diagnose_unary_lhs:
    3843            0 :           error ("invalid type for %qs", code_name);
    3844            0 :           debug_generic_expr (lhs_type);
    3845            0 :           return true;
    3846              :         }
    3847              :       break;
    3848              : 
    3849     25407059 :     case NEGATE_EXPR:
    3850     25407059 :     case ABS_EXPR:
    3851     25407059 :     case BIT_NOT_EXPR:
    3852     25407059 :       if (POINTER_TYPE_P (lhs_type) || TREE_CODE (lhs_type) == OFFSET_TYPE)
    3853            0 :         goto diagnose_unary_lhs;
    3854              :       /* FALLTHRU */
    3855     25879643 :     case PAREN_EXPR:
    3856     25879643 :       if (AGGREGATE_TYPE_P (lhs_type))
    3857            0 :         goto diagnose_unary_lhs;
    3858              :       break;
    3859              : 
    3860            0 :     default:
    3861            0 :       gcc_unreachable ();
    3862              :     }
    3863              : 
    3864              :   /* For the remaining codes assert there is no conversion involved.  */
    3865     25918933 :   if (!useless_type_conversion_p (lhs_type, rhs1_type))
    3866              :     {
    3867            0 :       error ("non-trivial conversion in unary operation");
    3868            0 :       debug_generic_expr (lhs_type);
    3869            0 :       debug_generic_expr (rhs1_type);
    3870            0 :       return true;
    3871              :     }
    3872              : 
    3873              :   return false;
    3874              : }
    3875              : 
    3876              : /* Verify a gimple assignment statement STMT with a binary rhs.
    3877              :    Returns true if anything is wrong.  */
    3878              : 
    3879              : static bool
    3880   1061910239 : verify_gimple_assign_binary (gassign *stmt)
    3881              : {
    3882   1061910239 :   enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
    3883   1061910239 :   tree lhs = gimple_assign_lhs (stmt);
    3884   1061910239 :   tree lhs_type = TREE_TYPE (lhs);
    3885   1061910239 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    3886   1061910239 :   tree rhs1_type = TREE_TYPE (rhs1);
    3887   1061910239 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    3888   1061910239 :   tree rhs2_type = TREE_TYPE (rhs2);
    3889              : 
    3890   1061910239 :   if (!is_gimple_reg (lhs))
    3891              :     {
    3892            0 :       error ("non-register as LHS of binary operation");
    3893            0 :       return true;
    3894              :     }
    3895              : 
    3896   1061910239 :   if (!is_gimple_val (rhs1)
    3897   1061910239 :       || !is_gimple_val (rhs2))
    3898              :     {
    3899            0 :       error ("invalid operands in binary operation");
    3900            0 :       return true;
    3901              :     }
    3902              : 
    3903   1061910239 :   const char* const code_name = get_tree_code_name (rhs_code);
    3904              : 
    3905              :   /* First handle operations that involve different types.  */
    3906   1061910239 :   switch (rhs_code)
    3907              :     {
    3908      2467184 :     case COMPLEX_EXPR:
    3909      2467184 :       {
    3910      2467184 :         if (TREE_CODE (lhs_type) != COMPLEX_TYPE
    3911      2467184 :             || !(INTEGRAL_TYPE_P (rhs1_type)
    3912              :                  || SCALAR_FLOAT_TYPE_P (rhs1_type))
    3913      2467184 :             || !(INTEGRAL_TYPE_P (rhs2_type)
    3914              :                  || SCALAR_FLOAT_TYPE_P (rhs2_type)))
    3915              :           {
    3916            0 :             error ("type mismatch in %qs", code_name);
    3917            0 :             debug_generic_expr (lhs_type);
    3918            0 :             debug_generic_expr (rhs1_type);
    3919            0 :             debug_generic_expr (rhs2_type);
    3920            0 :             return true;
    3921              :           }
    3922              : 
    3923              :         return false;
    3924              :       }
    3925              : 
    3926     34079692 :     case LSHIFT_EXPR:
    3927     34079692 :     case RSHIFT_EXPR:
    3928     34079692 :     case LROTATE_EXPR:
    3929     34079692 :     case RROTATE_EXPR:
    3930     34079692 :       {
    3931              :         /* Shifts and rotates are ok on integral types, fixed point
    3932              :            types and integer vector types.  */
    3933     34079692 :         if ((!INTEGRAL_TYPE_P (rhs1_type)
    3934       764274 :              && !FIXED_POINT_TYPE_P (rhs1_type)
    3935       764274 :              && ! (VECTOR_TYPE_P (rhs1_type)
    3936       764274 :                   && INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))))
    3937     34079692 :             || (!INTEGRAL_TYPE_P (rhs2_type)
    3938              :                 /* Vector shifts of vectors are also ok.  */
    3939       204117 :                 && ! (VECTOR_TYPE_P (rhs1_type)
    3940       204117 :                      && INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
    3941       204117 :                      && VECTOR_TYPE_P (rhs2_type)
    3942       204117 :                      && INTEGRAL_TYPE_P (TREE_TYPE (rhs2_type))))
    3943     68159384 :             || !useless_type_conversion_p (lhs_type, rhs1_type))
    3944              :           {
    3945            0 :             error ("type mismatch in %qs", code_name);
    3946            0 :             debug_generic_expr (lhs_type);
    3947            0 :             debug_generic_expr (rhs1_type);
    3948            0 :             debug_generic_expr (rhs2_type);
    3949            0 :             return true;
    3950              :           }
    3951              : 
    3952              :         return false;
    3953              :       }
    3954              : 
    3955            0 :     case WIDEN_LSHIFT_EXPR:
    3956            0 :       {
    3957            0 :         if (!INTEGRAL_TYPE_P (lhs_type)
    3958            0 :             || !INTEGRAL_TYPE_P (rhs1_type)
    3959            0 :             || TREE_CODE (rhs2) != INTEGER_CST
    3960            0 :             || (2 * TYPE_PRECISION (rhs1_type) > TYPE_PRECISION (lhs_type)))
    3961              :           {
    3962            0 :             error ("type mismatch in %qs", code_name);
    3963            0 :             debug_generic_expr (lhs_type);
    3964            0 :             debug_generic_expr (rhs1_type);
    3965            0 :             debug_generic_expr (rhs2_type);
    3966            0 :             return true;
    3967              :           }
    3968              : 
    3969              :         return false;
    3970              :       }
    3971              : 
    3972            0 :     case VEC_WIDEN_LSHIFT_HI_EXPR:
    3973            0 :     case VEC_WIDEN_LSHIFT_LO_EXPR:
    3974            0 :       {
    3975            0 :         if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    3976            0 :             || TREE_CODE (lhs_type) != VECTOR_TYPE
    3977            0 :             || !INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
    3978            0 :             || !INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
    3979            0 :             || TREE_CODE (rhs2) != INTEGER_CST
    3980            0 :             || (2 * TYPE_PRECISION (TREE_TYPE (rhs1_type))
    3981            0 :                 > TYPE_PRECISION (TREE_TYPE (lhs_type))))
    3982              :           {
    3983            0 :             error ("type mismatch in %qs", code_name);
    3984            0 :             debug_generic_expr (lhs_type);
    3985            0 :             debug_generic_expr (rhs1_type);
    3986            0 :             debug_generic_expr (rhs2_type);
    3987            0 :             return true;
    3988              :           }
    3989              : 
    3990              :         return false;
    3991              :       }
    3992              : 
    3993    470854204 :     case PLUS_EXPR:
    3994    470854204 :     case MINUS_EXPR:
    3995    470854204 :       {
    3996    470854204 :         tree lhs_etype = lhs_type;
    3997    470854204 :         tree rhs1_etype = rhs1_type;
    3998    470854204 :         tree rhs2_etype = rhs2_type;
    3999    470854204 :         if (VECTOR_TYPE_P (lhs_type))
    4000              :           {
    4001      4653859 :             if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    4002      4653859 :                 || TREE_CODE (rhs2_type) != VECTOR_TYPE)
    4003              :               {
    4004            0 :                 error ("invalid non-vector operands to %qs", code_name);
    4005            0 :                 return true;
    4006              :               }
    4007      4653859 :             lhs_etype = TREE_TYPE (lhs_type);
    4008      4653859 :             rhs1_etype = TREE_TYPE (rhs1_type);
    4009      4653859 :             rhs2_etype = TREE_TYPE (rhs2_type);
    4010              :           }
    4011    470854204 :         if (POINTER_TYPE_P (lhs_etype)
    4012    470854204 :             || POINTER_TYPE_P (rhs1_etype)
    4013    470854204 :             || POINTER_TYPE_P (rhs2_etype))
    4014              :           {
    4015            0 :             error ("invalid (pointer) operands %qs", code_name);
    4016            0 :             return true;
    4017              :           }
    4018              : 
    4019              :         /* Continue with generic binary expression handling.  */
    4020              :         break;
    4021              :       }
    4022              : 
    4023    158391116 :     case POINTER_PLUS_EXPR:
    4024    158391116 :       {
    4025    158391116 :         if (!POINTER_TYPE_P (rhs1_type)
    4026    158391116 :             || !useless_type_conversion_p (lhs_type, rhs1_type)
    4027    316782232 :             || !ptrofftype_p (rhs2_type))
    4028              :           {
    4029            0 :             error ("type mismatch in %qs", code_name);
    4030            0 :             debug_generic_stmt (lhs_type);
    4031            0 :             debug_generic_stmt (rhs1_type);
    4032            0 :             debug_generic_stmt (rhs2_type);
    4033            0 :             return true;
    4034              :           }
    4035              : 
    4036              :         return false;
    4037              :       }
    4038              : 
    4039     16441123 :     case POINTER_DIFF_EXPR:
    4040     16441123 :       {
    4041     16441123 :         if (!POINTER_TYPE_P (rhs1_type)
    4042     16441123 :             || !POINTER_TYPE_P (rhs2_type)
    4043              :             /* Because we special-case pointers to void we allow difference
    4044              :                of arbitrary pointers with the same mode.  */
    4045     16441123 :             || TYPE_MODE (rhs1_type) != TYPE_MODE (rhs2_type)
    4046     16441123 :             || !INTEGRAL_TYPE_P (lhs_type)
    4047     16441123 :             || TYPE_UNSIGNED (lhs_type)
    4048     32882246 :             || TYPE_PRECISION (lhs_type) != TYPE_PRECISION (rhs1_type))
    4049              :           {
    4050            0 :             error ("type mismatch in %qs", code_name);
    4051            0 :             debug_generic_stmt (lhs_type);
    4052            0 :             debug_generic_stmt (rhs1_type);
    4053            0 :             debug_generic_stmt (rhs2_type);
    4054            0 :             return true;
    4055              :           }
    4056              : 
    4057              :         return false;
    4058              :       }
    4059              : 
    4060            0 :     case TRUTH_ANDIF_EXPR:
    4061            0 :     case TRUTH_ORIF_EXPR:
    4062            0 :     case TRUTH_AND_EXPR:
    4063            0 :     case TRUTH_OR_EXPR:
    4064            0 :     case TRUTH_XOR_EXPR:
    4065              : 
    4066            0 :       gcc_unreachable ();
    4067              : 
    4068     89239877 :     case LT_EXPR:
    4069     89239877 :     case LE_EXPR:
    4070     89239877 :     case GT_EXPR:
    4071     89239877 :     case GE_EXPR:
    4072     89239877 :     case EQ_EXPR:
    4073     89239877 :     case NE_EXPR:
    4074     89239877 :     case UNORDERED_EXPR:
    4075     89239877 :     case ORDERED_EXPR:
    4076     89239877 :     case UNLT_EXPR:
    4077     89239877 :     case UNLE_EXPR:
    4078     89239877 :     case UNGT_EXPR:
    4079     89239877 :     case UNGE_EXPR:
    4080     89239877 :     case UNEQ_EXPR:
    4081     89239877 :     case LTGT_EXPR:
    4082              :       /* Comparisons are also binary, but the result type is not
    4083              :          connected to the operand types.  */
    4084     89239877 :       return verify_gimple_comparison (lhs_type, rhs1, rhs2, rhs_code);
    4085              : 
    4086       206778 :     case WIDEN_MULT_EXPR:
    4087       206778 :       if (TREE_CODE (lhs_type) != INTEGER_TYPE)
    4088              :         return true;
    4089       206778 :       return ((2 * TYPE_PRECISION (rhs1_type) > TYPE_PRECISION (lhs_type))
    4090       206778 :               || (TYPE_PRECISION (rhs1_type) != TYPE_PRECISION (rhs2_type)));
    4091              : 
    4092            0 :     case WIDEN_SUM_EXPR:
    4093            0 :       {
    4094            0 :         if (((TREE_CODE (rhs1_type) != VECTOR_TYPE
    4095            0 :               || TREE_CODE (lhs_type) != VECTOR_TYPE)
    4096            0 :              && ((!INTEGRAL_TYPE_P (rhs1_type)
    4097            0 :                   && !SCALAR_FLOAT_TYPE_P (rhs1_type))
    4098            0 :                  || (!INTEGRAL_TYPE_P (lhs_type)
    4099            0 :                      && !SCALAR_FLOAT_TYPE_P (lhs_type))))
    4100            0 :             || !useless_type_conversion_p (lhs_type, rhs2_type)
    4101            0 :             || maybe_lt (GET_MODE_SIZE (element_mode (rhs2_type)),
    4102            0 :                          2 * GET_MODE_SIZE (element_mode (rhs1_type))))
    4103              :           {
    4104            0 :             error ("type mismatch in %qs", code_name);
    4105            0 :             debug_generic_expr (lhs_type);
    4106            0 :             debug_generic_expr (rhs1_type);
    4107            0 :             debug_generic_expr (rhs2_type);
    4108            0 :             return true;
    4109              :           }
    4110              :         return false;
    4111              :       }
    4112              : 
    4113        43691 :     case VEC_WIDEN_MULT_HI_EXPR:
    4114        43691 :     case VEC_WIDEN_MULT_LO_EXPR:
    4115        43691 :     case VEC_WIDEN_MULT_EVEN_EXPR:
    4116        43691 :     case VEC_WIDEN_MULT_ODD_EXPR:
    4117        43691 :       {
    4118        43691 :         if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    4119        43691 :             || TREE_CODE (lhs_type) != VECTOR_TYPE
    4120        43691 :             || !types_compatible_p (rhs1_type, rhs2_type)
    4121       131073 :             || maybe_ne (GET_MODE_SIZE (element_mode (lhs_type)),
    4122        87382 :                          2 * GET_MODE_SIZE (element_mode (rhs1_type))))
    4123              :           {
    4124            0 :             error ("type mismatch in %qs", code_name);
    4125            0 :             debug_generic_expr (lhs_type);
    4126            0 :             debug_generic_expr (rhs1_type);
    4127            0 :             debug_generic_expr (rhs2_type);
    4128            0 :             return true;
    4129              :           }
    4130              :         return false;
    4131              :       }
    4132              : 
    4133       490882 :     case VEC_PACK_TRUNC_EXPR:
    4134              :       /* ???  We currently use VEC_PACK_TRUNC_EXPR to simply concat
    4135              :          vector boolean types.  */
    4136       490882 :       if (VECTOR_BOOLEAN_TYPE_P (lhs_type)
    4137       117707 :           && VECTOR_BOOLEAN_TYPE_P (rhs1_type)
    4138       117707 :           && types_compatible_p (rhs1_type, rhs2_type)
    4139       981764 :           && known_eq (TYPE_VECTOR_SUBPARTS (lhs_type),
    4140              :                        2 * TYPE_VECTOR_SUBPARTS (rhs1_type)))
    4141       117707 :         return false;
    4142              : 
    4143              :       /* Fallthru.  */
    4144       387450 :     case VEC_PACK_SAT_EXPR:
    4145       387450 :     case VEC_PACK_FIX_TRUNC_EXPR:
    4146       387450 :       {
    4147       387450 :         if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    4148       387450 :             || TREE_CODE (lhs_type) != VECTOR_TYPE
    4149       760625 :             || !((rhs_code == VEC_PACK_FIX_TRUNC_EXPR
    4150        14275 :                   && SCALAR_FLOAT_TYPE_P (TREE_TYPE (rhs1_type))
    4151        14275 :                   && INTEGRAL_TYPE_P (TREE_TYPE (lhs_type)))
    4152       373175 :                  || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
    4153       373175 :                      == INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))))
    4154       387450 :             || !types_compatible_p (rhs1_type, rhs2_type)
    4155       774900 :             || maybe_ne (GET_MODE_SIZE (element_mode (rhs1_type)),
    4156       774900 :                          2 * GET_MODE_SIZE (element_mode (lhs_type)))
    4157       774900 :             || maybe_ne (2 * TYPE_VECTOR_SUBPARTS (rhs1_type),
    4158       774900 :                          TYPE_VECTOR_SUBPARTS (lhs_type)))
    4159              :           {
    4160            0 :             error ("type mismatch in %qs", code_name);
    4161            0 :             debug_generic_expr (lhs_type);
    4162            0 :             debug_generic_expr (rhs1_type);
    4163            0 :             debug_generic_expr (rhs2_type);
    4164            0 :             return true;
    4165              :           }
    4166              : 
    4167              :         return false;
    4168              :       }
    4169              : 
    4170         1186 :     case VEC_PACK_FLOAT_EXPR:
    4171         1186 :       if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    4172         1186 :           || TREE_CODE (lhs_type) != VECTOR_TYPE
    4173         1186 :           || !INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
    4174         1186 :           || !SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs_type))
    4175         1186 :           || !types_compatible_p (rhs1_type, rhs2_type)
    4176         2372 :           || maybe_ne (GET_MODE_SIZE (element_mode (rhs1_type)),
    4177         2372 :                        2 * GET_MODE_SIZE (element_mode (lhs_type)))
    4178         2372 :           || maybe_ne (2 * TYPE_VECTOR_SUBPARTS (rhs1_type),
    4179         2372 :                        TYPE_VECTOR_SUBPARTS (lhs_type)))
    4180              :         {
    4181            0 :           error ("type mismatch in %qs", code_name);
    4182            0 :           debug_generic_expr (lhs_type);
    4183            0 :           debug_generic_expr (rhs1_type);
    4184            0 :           debug_generic_expr (rhs2_type);
    4185            0 :           return true;
    4186              :         }
    4187              : 
    4188              :       return false;
    4189              : 
    4190    219321091 :     case MULT_EXPR:
    4191    219321091 :     case MULT_HIGHPART_EXPR:
    4192    219321091 :     case TRUNC_DIV_EXPR:
    4193    219321091 :     case CEIL_DIV_EXPR:
    4194    219321091 :     case FLOOR_DIV_EXPR:
    4195    219321091 :     case ROUND_DIV_EXPR:
    4196    219321091 :     case TRUNC_MOD_EXPR:
    4197    219321091 :     case CEIL_MOD_EXPR:
    4198    219321091 :     case FLOOR_MOD_EXPR:
    4199    219321091 :     case ROUND_MOD_EXPR:
    4200    219321091 :     case RDIV_EXPR:
    4201    219321091 :     case EXACT_DIV_EXPR:
    4202    219321091 :     case BIT_IOR_EXPR:
    4203    219321091 :     case BIT_XOR_EXPR:
    4204              :       /* Disallow pointer and offset types for many of the binary gimple. */
    4205    219321091 :       if (POINTER_TYPE_P (lhs_type)
    4206    219321091 :           || TREE_CODE (lhs_type) == OFFSET_TYPE)
    4207              :         {
    4208            0 :           error ("invalid types for %qs", code_name);
    4209            0 :           debug_generic_expr (lhs_type);
    4210            0 :           debug_generic_expr (rhs1_type);
    4211            0 :           debug_generic_expr (rhs2_type);
    4212            0 :           return true;
    4213              :         }
    4214              :       /* Continue with generic binary expression handling.  */
    4215              :       break;
    4216              : 
    4217              :     case MIN_EXPR:
    4218              :     case MAX_EXPR:
    4219              :       /* Continue with generic binary expression handling.  */
    4220              :       break;
    4221              : 
    4222     52570287 :     case BIT_AND_EXPR:
    4223     52570287 :       if (POINTER_TYPE_P (lhs_type)
    4224       174787 :           && TREE_CODE (rhs2) == INTEGER_CST)
    4225              :         break;
    4226              :       /* Disallow pointer and offset types for many of the binary gimple. */
    4227     52395500 :       if (POINTER_TYPE_P (lhs_type)
    4228     52395500 :           || TREE_CODE (lhs_type) == OFFSET_TYPE)
    4229              :         {
    4230            0 :           error ("invalid types for %qs", code_name);
    4231            0 :           debug_generic_expr (lhs_type);
    4232            0 :           debug_generic_expr (rhs1_type);
    4233            0 :           debug_generic_expr (rhs2_type);
    4234            0 :           return true;
    4235              :         }
    4236              :       /* Continue with generic binary expression handling.  */
    4237              :       break;
    4238              : 
    4239            0 :     case VEC_SERIES_EXPR:
    4240            0 :       if (!useless_type_conversion_p (rhs1_type, rhs2_type))
    4241              :         {
    4242            0 :           error ("type mismatch in %qs", code_name);
    4243            0 :           debug_generic_expr (rhs1_type);
    4244            0 :           debug_generic_expr (rhs2_type);
    4245            0 :           return true;
    4246              :         }
    4247            0 :       if (TREE_CODE (lhs_type) != VECTOR_TYPE
    4248            0 :           || !useless_type_conversion_p (TREE_TYPE (lhs_type), rhs1_type))
    4249              :         {
    4250            0 :           error ("vector type expected in %qs", code_name);
    4251            0 :           debug_generic_expr (lhs_type);
    4252            0 :           return true;
    4253              :         }
    4254              :       return false;
    4255              : 
    4256            0 :     default:
    4257            0 :       gcc_unreachable ();
    4258              :     }
    4259              : 
    4260    760534435 :   if (!useless_type_conversion_p (lhs_type, rhs1_type)
    4261    760534435 :       || !useless_type_conversion_p (lhs_type, rhs2_type))
    4262              :     {
    4263            0 :       error ("type mismatch in binary expression");
    4264            0 :       debug_generic_stmt (lhs_type);
    4265            0 :       debug_generic_stmt (rhs1_type);
    4266            0 :       debug_generic_stmt (rhs2_type);
    4267            0 :       return true;
    4268              :     }
    4269              : 
    4270              :   return false;
    4271              : }
    4272              : 
    4273              : /* Verify a gimple assignment statement STMT with a ternary rhs.
    4274              :    Returns true if anything is wrong.  */
    4275              : 
    4276              : static bool
    4277     10049925 : verify_gimple_assign_ternary (gassign *stmt)
    4278              : {
    4279     10049925 :   enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
    4280     10049925 :   tree lhs = gimple_assign_lhs (stmt);
    4281     10049925 :   tree lhs_type = TREE_TYPE (lhs);
    4282     10049925 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    4283     10049925 :   tree rhs1_type = TREE_TYPE (rhs1);
    4284     10049925 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    4285     10049925 :   tree rhs2_type = TREE_TYPE (rhs2);
    4286     10049925 :   tree rhs3 = gimple_assign_rhs3 (stmt);
    4287     10049925 :   tree rhs3_type = TREE_TYPE (rhs3);
    4288              : 
    4289     10049925 :   if (!is_gimple_reg (lhs))
    4290              :     {
    4291            0 :       error ("non-register as LHS of ternary operation");
    4292            0 :       return true;
    4293              :     }
    4294              : 
    4295     10049925 :   if (!is_gimple_val (rhs1)
    4296     10049925 :       || !is_gimple_val (rhs2)
    4297     20099850 :       || !is_gimple_val (rhs3))
    4298              :     {
    4299            0 :       error ("invalid operands in ternary operation");
    4300            0 :       return true;
    4301              :     }
    4302              : 
    4303     10049925 :   const char* const code_name = get_tree_code_name (rhs_code);
    4304              : 
    4305              :   /* First handle operations that involve different types.  */
    4306     10049925 :   switch (rhs_code)
    4307              :     {
    4308            0 :     case WIDEN_MULT_PLUS_EXPR:
    4309            0 :     case WIDEN_MULT_MINUS_EXPR:
    4310            0 :       if ((!INTEGRAL_TYPE_P (rhs1_type)
    4311            0 :            && !FIXED_POINT_TYPE_P (rhs1_type))
    4312            0 :           || !useless_type_conversion_p (rhs1_type, rhs2_type)
    4313            0 :           || !useless_type_conversion_p (lhs_type, rhs3_type)
    4314            0 :           || 2 * TYPE_PRECISION (rhs1_type) > TYPE_PRECISION (lhs_type)
    4315            0 :           || TYPE_PRECISION (rhs1_type) != TYPE_PRECISION (rhs2_type))
    4316              :         {
    4317            0 :           error ("type mismatch in %qs", code_name);
    4318            0 :           debug_generic_expr (lhs_type);
    4319            0 :           debug_generic_expr (rhs1_type);
    4320            0 :           debug_generic_expr (rhs2_type);
    4321            0 :           debug_generic_expr (rhs3_type);
    4322            0 :           return true;
    4323              :         }
    4324              :       break;
    4325              : 
    4326      1834754 :     case VEC_COND_EXPR:
    4327      1834754 :       if (!VECTOR_BOOLEAN_TYPE_P (rhs1_type)
    4328      3669508 :           || maybe_ne (TYPE_VECTOR_SUBPARTS (rhs1_type),
    4329      3669508 :                        TYPE_VECTOR_SUBPARTS (lhs_type)))
    4330              :         {
    4331            0 :           error ("the first argument of a %qs must be of a "
    4332              :                  "boolean vector type of the same number of elements "
    4333              :                  "as the result", code_name);
    4334            0 :           debug_generic_expr (lhs_type);
    4335            0 :           debug_generic_expr (rhs1_type);
    4336            0 :           return true;
    4337              :         }
    4338              :       /* Fallthrough.  */
    4339      2974870 :     case COND_EXPR:
    4340      2974870 :       if (!useless_type_conversion_p (lhs_type, rhs2_type)
    4341      2974870 :           || !useless_type_conversion_p (lhs_type, rhs3_type))
    4342              :         {
    4343            0 :           error ("type mismatch in %qs", code_name);
    4344            0 :           debug_generic_expr (lhs_type);
    4345            0 :           debug_generic_expr (rhs2_type);
    4346            0 :           debug_generic_expr (rhs3_type);
    4347            0 :           return true;
    4348              :         }
    4349              :       break;
    4350              : 
    4351      6952257 :     case VEC_PERM_EXPR:
    4352              :       /* If permute is constant, then we allow for lhs and rhs
    4353              :          to have different vector types, provided:
    4354              :          (1) lhs, rhs1, rhs2 have same element type.
    4355              :          (2) rhs3 vector is constant and has integer element type.
    4356              :          (3) len(lhs) == len(rhs3) && len(rhs1) == len(rhs2).  */
    4357              : 
    4358      6952257 :       if (TREE_CODE (lhs_type) != VECTOR_TYPE
    4359      6952257 :           || TREE_CODE (rhs1_type) != VECTOR_TYPE
    4360      6952257 :           || TREE_CODE (rhs2_type) != VECTOR_TYPE
    4361      6952257 :           || TREE_CODE (rhs3_type) != VECTOR_TYPE)
    4362              :         {
    4363            0 :           error ("vector types expected in %qs", code_name);
    4364            0 :           debug_generic_expr (lhs_type);
    4365            0 :           debug_generic_expr (rhs1_type);
    4366            0 :           debug_generic_expr (rhs2_type);
    4367            0 :           debug_generic_expr (rhs3_type);
    4368            0 :           return true;
    4369              :         }
    4370              : 
    4371              :       /* If rhs3 is constant, we allow lhs, rhs1 and rhs2 to be different vector types,
    4372              :          as long as lhs, rhs1 and rhs2 have same element type.  */
    4373      6952257 :       if (TREE_CONSTANT (rhs3)
    4374      6952257 :           ? (!useless_type_conversion_p (TREE_TYPE (lhs_type), TREE_TYPE (rhs1_type))
    4375      6831119 :              || !useless_type_conversion_p (TREE_TYPE (lhs_type), TREE_TYPE (rhs2_type)))
    4376       121138 :           : (!useless_type_conversion_p (lhs_type, rhs1_type)
    4377       121138 :              || !useless_type_conversion_p (lhs_type, rhs2_type)))
    4378              :         {
    4379            0 :             error ("type mismatch in %qs", code_name);
    4380            0 :             debug_generic_expr (lhs_type);
    4381            0 :             debug_generic_expr (rhs1_type);
    4382            0 :             debug_generic_expr (rhs2_type);
    4383            0 :             debug_generic_expr (rhs3_type);
    4384            0 :             return true;
    4385              :         }
    4386              : 
    4387              :       /* If rhs3 is constant, relax the check len(rhs2) == len(rhs3).  */
    4388      6952257 :       if (maybe_ne (TYPE_VECTOR_SUBPARTS (rhs1_type),
    4389      6952257 :                     TYPE_VECTOR_SUBPARTS (rhs2_type))
    4390      6952257 :           || (!TREE_CONSTANT(rhs3)
    4391       121138 :               && maybe_ne (TYPE_VECTOR_SUBPARTS (rhs2_type),
    4392       121138 :                            TYPE_VECTOR_SUBPARTS (rhs3_type)))
    4393     13904514 :           || maybe_ne (TYPE_VECTOR_SUBPARTS (rhs3_type),
    4394      6952257 :                        TYPE_VECTOR_SUBPARTS (lhs_type)))
    4395              :         {
    4396            0 :           error ("vectors with different element number found in %qs",
    4397              :                  code_name);
    4398            0 :           debug_generic_expr (lhs_type);
    4399            0 :           debug_generic_expr (rhs1_type);
    4400            0 :           debug_generic_expr (rhs2_type);
    4401            0 :           debug_generic_expr (rhs3_type);
    4402            0 :           return true;
    4403              :         }
    4404              : 
    4405      6952257 :       if (TREE_CODE (TREE_TYPE (rhs3_type)) != INTEGER_TYPE
    4406      6952257 :           || (TREE_CODE (rhs3) != VECTOR_CST
    4407       121138 :               && (GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE
    4408              :                                     (TREE_TYPE (rhs3_type)))
    4409      7073395 :                   != GET_MODE_BITSIZE (SCALAR_TYPE_MODE
    4410              :                                        (TREE_TYPE (rhs1_type))))))
    4411              :         {
    4412            0 :           error ("invalid mask type in %qs", code_name);
    4413            0 :           debug_generic_expr (lhs_type);
    4414            0 :           debug_generic_expr (rhs1_type);
    4415            0 :           debug_generic_expr (rhs2_type);
    4416            0 :           debug_generic_expr (rhs3_type);
    4417            0 :           return true;
    4418              :         }
    4419              : 
    4420              :       return false;
    4421              : 
    4422         4810 :     case SAD_EXPR:
    4423         4810 :       if (!useless_type_conversion_p (rhs1_type, rhs2_type)
    4424         4810 :           || !useless_type_conversion_p (lhs_type, rhs3_type)
    4425         9620 :           || 2 * GET_MODE_UNIT_BITSIZE (TYPE_MODE (TREE_TYPE (rhs1_type)))
    4426         9620 :                > GET_MODE_UNIT_BITSIZE (TYPE_MODE (TREE_TYPE (lhs_type))))
    4427              :         {
    4428            0 :           error ("type mismatch in %qs", code_name);
    4429            0 :           debug_generic_expr (lhs_type);
    4430            0 :           debug_generic_expr (rhs1_type);
    4431            0 :           debug_generic_expr (rhs2_type);
    4432            0 :           debug_generic_expr (rhs3_type);
    4433            0 :           return true;
    4434              :         }
    4435              : 
    4436         4810 :       if (TREE_CODE (rhs1_type) != VECTOR_TYPE
    4437         4810 :           || TREE_CODE (rhs2_type) != VECTOR_TYPE
    4438         4810 :           || TREE_CODE (rhs3_type) != VECTOR_TYPE)
    4439              :         {
    4440            0 :           error ("vector types expected in %qs", code_name);
    4441            0 :           debug_generic_expr (lhs_type);
    4442            0 :           debug_generic_expr (rhs1_type);
    4443            0 :           debug_generic_expr (rhs2_type);
    4444            0 :           debug_generic_expr (rhs3_type);
    4445            0 :           return true;
    4446              :         }
    4447              : 
    4448              :       return false;
    4449              : 
    4450       100312 :     case BIT_INSERT_EXPR:
    4451       100312 :       if (! useless_type_conversion_p (lhs_type, rhs1_type))
    4452              :         {
    4453            0 :           error ("type mismatch in %qs", code_name);
    4454            0 :           debug_generic_expr (lhs_type);
    4455            0 :           debug_generic_expr (rhs1_type);
    4456            0 :           return true;
    4457              :         }
    4458       100312 :       if (! ((INTEGRAL_TYPE_P (rhs1_type)
    4459         5505 :               && INTEGRAL_TYPE_P (rhs2_type))
    4460              :              /* Vector element insert.  */
    4461        94807 :              || (VECTOR_TYPE_P (rhs1_type)
    4462        94807 :                  && types_compatible_p (TREE_TYPE (rhs1_type), rhs2_type))
    4463              :              /* Aligned sub-vector insert.  */
    4464         4292 :              || (VECTOR_TYPE_P (rhs1_type)
    4465         4292 :                  && VECTOR_TYPE_P (rhs2_type)
    4466         4292 :                  && types_compatible_p (TREE_TYPE (rhs1_type),
    4467         4292 :                                         TREE_TYPE (rhs2_type))
    4468         4292 :                  && multiple_p (TYPE_VECTOR_SUBPARTS (rhs1_type),
    4469         4292 :                                 TYPE_VECTOR_SUBPARTS (rhs2_type))
    4470         4292 :                  && multiple_p (wi::to_poly_offset (rhs3),
    4471         4292 :                                 wi::to_poly_offset (TYPE_SIZE (rhs2_type))))))
    4472              :         {
    4473            0 :           error ("not allowed type combination in %qs", code_name);
    4474            0 :           debug_generic_expr (rhs1_type);
    4475            0 :           debug_generic_expr (rhs2_type);
    4476            0 :           return true;
    4477              :         }
    4478       100312 :       if (! tree_fits_uhwi_p (rhs3)
    4479       100312 :           || ! types_compatible_p (bitsizetype, TREE_TYPE (rhs3))
    4480       200624 :           || ! tree_fits_uhwi_p (TYPE_SIZE (rhs2_type)))
    4481              :         {
    4482            0 :           error ("invalid position or size in %qs", code_name);
    4483            0 :           return true;
    4484              :         }
    4485       100312 :       if (INTEGRAL_TYPE_P (rhs1_type)
    4486       100312 :           && !type_has_mode_precision_p (rhs1_type))
    4487              :         {
    4488            0 :           error ("%qs into non-mode-precision operand", code_name);
    4489            0 :           return true;
    4490              :         }
    4491       100312 :       if (INTEGRAL_TYPE_P (rhs1_type))
    4492              :         {
    4493         5505 :           unsigned HOST_WIDE_INT bitpos = tree_to_uhwi (rhs3);
    4494         5505 :           if (bitpos >= TYPE_PRECISION (rhs1_type)
    4495         5505 :               || (bitpos + TYPE_PRECISION (rhs2_type)
    4496         5505 :                   > TYPE_PRECISION (rhs1_type)))
    4497              :             {
    4498            0 :               error ("insertion out of range in %qs", code_name);
    4499            0 :               return true;
    4500              :             }
    4501              :         }
    4502        94807 :       else if (VECTOR_TYPE_P (rhs1_type))
    4503              :         {
    4504        94807 :           unsigned HOST_WIDE_INT bitpos = tree_to_uhwi (rhs3);
    4505        94807 :           unsigned HOST_WIDE_INT bitsize = tree_to_uhwi (TYPE_SIZE (rhs2_type));
    4506        94807 :           if (bitpos % bitsize != 0)
    4507              :             {
    4508            0 :               error ("%qs not at element boundary", code_name);
    4509            0 :               return true;
    4510              :             }
    4511              :         }
    4512              :       return false;
    4513              : 
    4514        17676 :     case DOT_PROD_EXPR:
    4515        17676 :       {
    4516        17676 :         if (((TREE_CODE (rhs1_type) != VECTOR_TYPE
    4517        17676 :               || TREE_CODE (lhs_type) != VECTOR_TYPE)
    4518            0 :              && ((!INTEGRAL_TYPE_P (rhs1_type)
    4519            0 :                   && !SCALAR_FLOAT_TYPE_P (rhs1_type))
    4520            0 :                  || (!INTEGRAL_TYPE_P (lhs_type)
    4521            0 :                      && !SCALAR_FLOAT_TYPE_P (lhs_type))))
    4522              :             /* rhs1_type and rhs2_type may differ in sign.  */
    4523        17676 :             || !tree_nop_conversion_p (rhs1_type, rhs2_type)
    4524        17676 :             || !useless_type_conversion_p (lhs_type, rhs3_type)
    4525        53028 :             || maybe_lt (GET_MODE_SIZE (element_mode (rhs3_type)),
    4526        35352 :                          2 * GET_MODE_SIZE (element_mode (rhs1_type))))
    4527              :           {
    4528            0 :             error ("type mismatch in %qs", code_name);
    4529            0 :             debug_generic_expr (lhs_type);
    4530            0 :             debug_generic_expr (rhs1_type);
    4531            0 :             debug_generic_expr (rhs2_type);
    4532            0 :             return true;
    4533              :           }
    4534              :         return false;
    4535              :       }
    4536              : 
    4537              :     case REALIGN_LOAD_EXPR:
    4538              :       /* FIXME.  */
    4539              :       return false;
    4540              : 
    4541            0 :     default:
    4542            0 :       gcc_unreachable ();
    4543              :     }
    4544              :   return false;
    4545              : }
    4546              : 
    4547              : /* Verify a gimple assignment statement STMT with a single rhs.
    4548              :    Returns true if anything is wrong.  */
    4549              : 
    4550              : static bool
    4551   3202169013 : verify_gimple_assign_single (gassign *stmt)
    4552              : {
    4553   3202169013 :   enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
    4554   3202169013 :   tree lhs = gimple_assign_lhs (stmt);
    4555   3202169013 :   tree lhs_type = TREE_TYPE (lhs);
    4556   3202169013 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    4557   3202169013 :   tree rhs1_type = TREE_TYPE (rhs1);
    4558   3202169013 :   bool res = false;
    4559              : 
    4560   3202169013 :   const char* const code_name = get_tree_code_name (rhs_code);
    4561              : 
    4562   3202169013 :   if (!useless_type_conversion_p (lhs_type, rhs1_type))
    4563              :     {
    4564            0 :       error ("non-trivial conversion in %qs", code_name);
    4565            0 :       debug_generic_expr (lhs_type);
    4566            0 :       debug_generic_expr (rhs1_type);
    4567            0 :       return true;
    4568              :     }
    4569              : 
    4570              :   /* LHS can't be a constant or an address expression. */
    4571   3202169013 :   if (CONSTANT_CLASS_P (lhs)|| TREE_CODE (lhs) == ADDR_EXPR)
    4572              :     {
    4573            0 :       error ("invalid LHS (%qs) for assignment: %qs",
    4574              :              get_tree_code_name (TREE_CODE (lhs)), code_name);
    4575            0 :       return true;
    4576              :     }
    4577              : 
    4578   3202169013 :   if (gimple_clobber_p (stmt)
    4579   3202169013 :       && !(DECL_P (lhs) || TREE_CODE (lhs) == MEM_REF))
    4580              :     {
    4581            0 :       error ("%qs LHS in clobber statement",
    4582              :              get_tree_code_name (TREE_CODE (lhs)));
    4583            0 :       debug_generic_expr (lhs);
    4584            0 :       return true;
    4585              :     }
    4586              : 
    4587   3202169013 :   if (TREE_CODE (lhs) == WITH_SIZE_EXPR)
    4588              :     {
    4589            0 :       error ("%qs LHS in assignment statement",
    4590              :              get_tree_code_name (TREE_CODE (lhs)));
    4591            0 :       debug_generic_expr (lhs);
    4592            0 :       return true;
    4593              :     }
    4594              : 
    4595   3202169013 :   if (handled_component_p (lhs)
    4596   2368816998 :       || TREE_CODE (lhs) == MEM_REF
    4597   2113485376 :       || TREE_CODE (lhs) == TARGET_MEM_REF)
    4598   1097851964 :     res |= verify_types_in_gimple_reference (lhs, true);
    4599              : 
    4600              :   /* Special codes we cannot handle via their class.  */
    4601   3202169013 :   switch (rhs_code)
    4602              :     {
    4603    267654745 :     case ADDR_EXPR:
    4604    267654745 :       {
    4605    267654745 :         tree op = TREE_OPERAND (rhs1, 0);
    4606    267654745 :         if (!is_gimple_addressable (op))
    4607              :           {
    4608            0 :             error ("invalid operand in %qs", code_name);
    4609            0 :             return true;
    4610              :           }
    4611              : 
    4612              :         /* Technically there is no longer a need for matching types, but
    4613              :            gimple hygiene asks for this check.  In LTO we can end up
    4614              :            combining incompatible units and thus end up with addresses
    4615              :            of globals that change their type to a common one.  */
    4616    267654745 :         if (!in_lto_p
    4617    267141949 :             && !types_compatible_p (TREE_TYPE (op),
    4618    267141949 :                                     TREE_TYPE (TREE_TYPE (rhs1)))
    4619    267655107 :             && !one_pointer_to_useless_type_conversion_p (TREE_TYPE (rhs1),
    4620          362 :                                                           TREE_TYPE (op)))
    4621              :           {
    4622            0 :             error ("type mismatch in %qs", code_name);
    4623            0 :             debug_generic_stmt (TREE_TYPE (rhs1));
    4624            0 :             debug_generic_stmt (TREE_TYPE (op));
    4625            0 :             return true;
    4626              :           }
    4627              : 
    4628    267654745 :         return (verify_address (rhs1, true)
    4629    267654745 :                 || verify_types_in_gimple_reference (op, true));
    4630              :       }
    4631              : 
    4632              :     /* tcc_reference  */
    4633            0 :     case INDIRECT_REF:
    4634            0 :       error ("%qs in gimple IL", code_name);
    4635            0 :       return true;
    4636              : 
    4637           46 :     case WITH_SIZE_EXPR:
    4638           46 :       if (!is_gimple_val (TREE_OPERAND (rhs1, 1)))
    4639              :         {
    4640            0 :           error ("invalid %qs size argument in load", code_name);
    4641            0 :           debug_generic_stmt (lhs);
    4642            0 :           debug_generic_stmt (rhs1);
    4643            0 :           return true;
    4644              :         }
    4645           46 :       rhs1 = TREE_OPERAND (rhs1, 0);
    4646              :       /* Fallthru.  */
    4647   1092284457 :     case COMPONENT_REF:
    4648   1092284457 :     case BIT_FIELD_REF:
    4649   1092284457 :     case ARRAY_REF:
    4650   1092284457 :     case ARRAY_RANGE_REF:
    4651   1092284457 :     case VIEW_CONVERT_EXPR:
    4652   1092284457 :     case REALPART_EXPR:
    4653   1092284457 :     case IMAGPART_EXPR:
    4654   1092284457 :     case TARGET_MEM_REF:
    4655   1092284457 :     case MEM_REF:
    4656   1092284457 :       if (!is_gimple_reg (lhs)
    4657   1092284457 :           && is_gimple_reg_type (TREE_TYPE (lhs)))
    4658              :         {
    4659            0 :           error ("invalid RHS for gimple memory store: %qs", code_name);
    4660            0 :           debug_generic_stmt (lhs);
    4661            0 :           debug_generic_stmt (rhs1);
    4662            0 :           return true;
    4663              :         }
    4664   1092284457 :       return res || verify_types_in_gimple_reference (rhs1, false);
    4665              : 
    4666              :     /* tcc_constant  */
    4667              :     case SSA_NAME:
    4668              :     case INTEGER_CST:
    4669              :     case REAL_CST:
    4670              :     case FIXED_CST:
    4671              :     case COMPLEX_CST:
    4672              :     case VECTOR_CST:
    4673              :     case STRING_CST:
    4674              :       return res;
    4675              : 
    4676              :     /* tcc_declaration  */
    4677              :     case CONST_DECL:
    4678              :       return res;
    4679    330711106 :     case VAR_DECL:
    4680    330711106 :     case PARM_DECL:
    4681    330711106 :       if (!is_gimple_reg (lhs)
    4682     90438733 :           && !is_gimple_reg (rhs1)
    4683    409120109 :           && is_gimple_reg_type (TREE_TYPE (lhs)))
    4684              :         {
    4685            0 :           error ("invalid RHS for gimple memory store: %qs", code_name);
    4686            0 :           debug_generic_stmt (lhs);
    4687            0 :           debug_generic_stmt (rhs1);
    4688            0 :           return true;
    4689              :         }
    4690              :       return res;
    4691              : 
    4692    331573843 :     case CONSTRUCTOR:
    4693    331573843 :       if (VECTOR_TYPE_P (rhs1_type))
    4694              :         {
    4695      6900126 :           unsigned int i;
    4696      6900126 :           tree elt_i, elt_v, elt_t = NULL_TREE;
    4697              : 
    4698      6900126 :           if (CONSTRUCTOR_NELTS (rhs1) == 0)
    4699              :             return res;
    4700      6182051 :           if (!is_gimple_reg (lhs))
    4701              :             {
    4702            0 :               error ("non-register as LHS with vector constructor");
    4703            0 :               return true;
    4704              :             }
    4705              :           /* For vector CONSTRUCTORs we require that either it is empty
    4706              :              CONSTRUCTOR, or it is a CONSTRUCTOR of smaller vector elements
    4707              :              (then the element count must be correct to cover the whole
    4708              :              outer vector and index must be NULL on all elements, or it is
    4709              :              a CONSTRUCTOR of scalar elements, where we as an exception allow
    4710              :              smaller number of elements (assuming zero filling) and
    4711              :              consecutive indexes as compared to NULL indexes (such
    4712              :              CONSTRUCTORs can appear in the IL from FEs).  */
    4713     28914066 :           FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (rhs1), i, elt_i, elt_v)
    4714              :             {
    4715     22732015 :               if (elt_t == NULL_TREE)
    4716              :                 {
    4717      6182051 :                   elt_t = TREE_TYPE (elt_v);
    4718      6182051 :                   if (VECTOR_TYPE_P (elt_t))
    4719              :                     {
    4720       206250 :                       tree elt_t = TREE_TYPE (elt_v);
    4721       206250 :                       if (!useless_type_conversion_p (TREE_TYPE (rhs1_type),
    4722       206250 :                                                       TREE_TYPE (elt_t)))
    4723              :                         {
    4724            0 :                           error ("incorrect type of vector %qs elements",
    4725              :                                  code_name);
    4726            0 :                           debug_generic_stmt (rhs1);
    4727            0 :                           return true;
    4728              :                         }
    4729       412500 :                       else if (maybe_ne (CONSTRUCTOR_NELTS (rhs1)
    4730       412500 :                                          * TYPE_VECTOR_SUBPARTS (elt_t),
    4731       412500 :                                          TYPE_VECTOR_SUBPARTS (rhs1_type)))
    4732              :                         {
    4733            0 :                           error ("incorrect number of vector %qs elements",
    4734              :                                  code_name);
    4735            0 :                           debug_generic_stmt (rhs1);
    4736            0 :                           return true;
    4737              :                         }
    4738              :                     }
    4739      5975801 :                   else if (!useless_type_conversion_p (TREE_TYPE (rhs1_type),
    4740              :                                                        elt_t))
    4741              :                     {
    4742            0 :                       error ("incorrect type of vector %qs elements",
    4743              :                              code_name);
    4744            0 :                       debug_generic_stmt (rhs1);
    4745            0 :                       return true;
    4746              :                     }
    4747     11951602 :                   else if (maybe_gt (CONSTRUCTOR_NELTS (rhs1),
    4748              :                                      TYPE_VECTOR_SUBPARTS (rhs1_type)))
    4749              :                     {
    4750            0 :                       error ("incorrect number of vector %qs elements",
    4751              :                              code_name);
    4752            0 :                       debug_generic_stmt (rhs1);
    4753            0 :                       return true;
    4754              :                     }
    4755              :                 }
    4756     16549964 :               else if (!useless_type_conversion_p (elt_t, TREE_TYPE (elt_v)))
    4757              :                 {
    4758            0 :                   error ("incorrect type of vector CONSTRUCTOR elements");
    4759            0 :                   debug_generic_stmt (rhs1);
    4760            0 :                   return true;
    4761              :                 }
    4762     22732015 :               if (elt_i != NULL_TREE
    4763     22732015 :                   && (VECTOR_TYPE_P (elt_t)
    4764      7996051 :                       || TREE_CODE (elt_i) != INTEGER_CST
    4765      7996051 :                       || compare_tree_int (elt_i, i) != 0))
    4766              :                 {
    4767            0 :                   error ("vector %qs with non-NULL element index",
    4768              :                          code_name);
    4769            0 :                   debug_generic_stmt (rhs1);
    4770            0 :                   return true;
    4771              :                 }
    4772     22732015 :               if (!is_gimple_val (elt_v))
    4773              :                 {
    4774            0 :                   error ("vector %qs element is not a GIMPLE value",
    4775              :                          code_name);
    4776            0 :                   debug_generic_stmt (rhs1);
    4777            0 :                   return true;
    4778              :                 }
    4779              :             }
    4780              :         }
    4781    324673717 :       else if (CONSTRUCTOR_NELTS (rhs1) != 0)
    4782              :         {
    4783            0 :           error ("non-vector %qs with elements", code_name);
    4784            0 :           debug_generic_stmt (rhs1);
    4785            0 :           return true;
    4786              :         }
    4787              :       return res;
    4788              : 
    4789              :     case OBJ_TYPE_REF:
    4790              :       /* FIXME.  */
    4791              :       return res;
    4792              : 
    4793              :     default:;
    4794              :     }
    4795              : 
    4796              :   return res;
    4797              : }
    4798              : 
    4799              : /* Verify the contents of a GIMPLE_ASSIGN STMT.  Returns true when there
    4800              :    is a problem, otherwise false.  */
    4801              : 
    4802              : static bool
    4803   4701992950 : verify_gimple_assign (gassign *stmt)
    4804              : {
    4805   4701992950 :   if (gimple_assign_nontemporal_move_p (stmt))
    4806              :     {
    4807         1155 :       tree lhs = gimple_assign_lhs (stmt);
    4808         1155 :       if (is_gimple_reg (lhs))
    4809              :         {
    4810            0 :           error ("nontemporal store lhs cannot be a gimple register");
    4811            0 :           debug_generic_stmt (lhs);
    4812            0 :           return true;
    4813              :         }
    4814              :     }
    4815              : 
    4816   4701992950 :   switch (gimple_assign_rhs_class (stmt))
    4817              :     {
    4818   3202169013 :     case GIMPLE_SINGLE_RHS:
    4819   3202169013 :       return verify_gimple_assign_single (stmt);
    4820              : 
    4821    427863773 :     case GIMPLE_UNARY_RHS:
    4822    427863773 :       return verify_gimple_assign_unary (stmt);
    4823              : 
    4824   1061910239 :     case GIMPLE_BINARY_RHS:
    4825   1061910239 :       return verify_gimple_assign_binary (stmt);
    4826              : 
    4827     10049925 :     case GIMPLE_TERNARY_RHS:
    4828     10049925 :       return verify_gimple_assign_ternary (stmt);
    4829              : 
    4830            0 :     default:
    4831            0 :       gcc_unreachable ();
    4832              :     }
    4833              : }
    4834              : 
    4835              : /* Verify the contents of a GIMPLE_RETURN STMT.  Returns true when there
    4836              :    is a problem, otherwise false.  */
    4837              : 
    4838              : static bool
    4839    263759581 : verify_gimple_return (greturn *stmt)
    4840              : {
    4841    263759581 :   tree op = gimple_return_retval (stmt);
    4842    263759581 :   tree restype = TREE_TYPE (TREE_TYPE (cfun->decl));
    4843    263759581 :   tree resdecl = DECL_RESULT (cfun->decl);
    4844              : 
    4845              :   /* We cannot test for present return values as we do not fix up missing
    4846              :      return values from the original source.  */
    4847    263759581 :   if (op == NULL)
    4848              :     return false;
    4849              : 
    4850    147699750 :   if (!is_gimple_val (op)
    4851    147699750 :       && TREE_CODE (op) != RESULT_DECL)
    4852              :     {
    4853            0 :       error ("invalid operand in return statement");
    4854            0 :       debug_generic_stmt (op);
    4855            0 :       return true;
    4856              :     }
    4857              : 
    4858    147699750 :   if (resdecl && DECL_BY_REFERENCE (resdecl))
    4859      3380072 :     op = TREE_TYPE (op);
    4860              : 
    4861    147699750 :   if (!useless_type_conversion_p (restype, TREE_TYPE (op)))
    4862              :     {
    4863            0 :       error ("invalid conversion in return statement");
    4864            0 :       debug_generic_stmt (restype);
    4865            0 :       debug_generic_stmt (TREE_TYPE (op));
    4866            0 :       return true;
    4867              :     }
    4868              : 
    4869              :   return false;
    4870              : }
    4871              : 
    4872              : 
    4873              : /* Verify the contents of a GIMPLE_GOTO STMT.  Returns true when there
    4874              :    is a problem, otherwise false.  */
    4875              : 
    4876              : static bool
    4877     26301079 : verify_gimple_goto (ggoto *stmt)
    4878              : {
    4879     26301079 :   tree dest = gimple_goto_dest (stmt);
    4880              : 
    4881              :   /* ???  We have two canonical forms of direct goto destinations, a
    4882              :      bare LABEL_DECL and an ADDR_EXPR of a LABEL_DECL.  */
    4883     26301079 :   if (TREE_CODE (dest) != LABEL_DECL
    4884     26301079 :       && (!is_gimple_val (dest)
    4885        62671 :           || !POINTER_TYPE_P (TREE_TYPE (dest))))
    4886              :     {
    4887            0 :       error ("goto destination is neither a label nor a pointer");
    4888            0 :       return true;
    4889              :     }
    4890              : 
    4891              :   return false;
    4892              : }
    4893              : 
    4894              : /* Verify the contents of a GIMPLE_SWITCH STMT.  Returns true when there
    4895              :    is a problem, otherwise false.  */
    4896              : 
    4897              : static bool
    4898      4802757 : verify_gimple_switch (gswitch *stmt)
    4899              : {
    4900      4802757 :   unsigned int i, n;
    4901      4802757 :   tree elt, prev_upper_bound = NULL_TREE;
    4902      4802757 :   tree index_type, elt_type = NULL_TREE;
    4903              : 
    4904      4802757 :   if (!is_gimple_val (gimple_switch_index (stmt)))
    4905              :     {
    4906            0 :       error ("invalid operand to switch statement");
    4907            0 :       debug_generic_stmt (gimple_switch_index (stmt));
    4908            0 :       return true;
    4909              :     }
    4910              : 
    4911      4802757 :   index_type = TREE_TYPE (gimple_switch_index (stmt));
    4912      4802757 :   if (! INTEGRAL_TYPE_P (index_type))
    4913              :     {
    4914            0 :       error ("non-integral type switch statement");
    4915            0 :       debug_generic_expr (index_type);
    4916            0 :       return true;
    4917              :     }
    4918              : 
    4919      4802757 :   elt = gimple_switch_label (stmt, 0);
    4920      4802757 :   if (CASE_LOW (elt) != NULL_TREE
    4921      4802757 :       || CASE_HIGH (elt) != NULL_TREE
    4922      9605514 :       || CASE_CHAIN (elt) != NULL_TREE)
    4923              :     {
    4924            0 :       error ("invalid default case label in switch statement");
    4925            0 :       debug_generic_expr (elt);
    4926            0 :       return true;
    4927              :     }
    4928              : 
    4929      4802757 :   n = gimple_switch_num_labels (stmt);
    4930     41747898 :   for (i = 1; i < n; i++)
    4931              :     {
    4932     32142384 :       elt = gimple_switch_label (stmt, i);
    4933              : 
    4934     32142384 :       if (CASE_CHAIN (elt))
    4935              :         {
    4936            0 :           error ("invalid %<CASE_CHAIN%>");
    4937            0 :           debug_generic_expr (elt);
    4938            0 :           return true;
    4939              :         }
    4940     32142384 :       if (! CASE_LOW (elt))
    4941              :         {
    4942            0 :           error ("invalid case label in switch statement");
    4943            0 :           debug_generic_expr (elt);
    4944            0 :           return true;
    4945              :         }
    4946     32142384 :       if (CASE_HIGH (elt)
    4947     32142384 :           && ! tree_int_cst_lt (CASE_LOW (elt), CASE_HIGH (elt)))
    4948              :         {
    4949            0 :           error ("invalid case range in switch statement");
    4950            0 :           debug_generic_expr (elt);
    4951            0 :           return true;
    4952              :         }
    4953              : 
    4954     32142384 :       if (! elt_type)
    4955              :         {
    4956      4799616 :           elt_type = TREE_TYPE (CASE_LOW (elt));
    4957      4799616 :           if (TYPE_PRECISION (index_type) < TYPE_PRECISION (elt_type))
    4958              :             {
    4959            0 :               error ("type precision mismatch in switch statement");
    4960            0 :               return true;
    4961              :             }
    4962              :         }
    4963     32142384 :       if (TREE_TYPE (CASE_LOW (elt)) != elt_type
    4964     32142384 :           || (CASE_HIGH (elt) && TREE_TYPE (CASE_HIGH (elt)) != elt_type))
    4965              :         {
    4966            0 :           error ("type mismatch for case label in switch statement");
    4967            0 :           debug_generic_expr (elt);
    4968            0 :           return true;
    4969              :         }
    4970              : 
    4971     32142384 :       if (prev_upper_bound)
    4972              :         {
    4973     27342768 :           if (! tree_int_cst_lt (prev_upper_bound, CASE_LOW (elt)))
    4974              :             {
    4975            0 :               error ("case labels not sorted in switch statement");
    4976            0 :               return true;
    4977              :             }
    4978              :         }
    4979              : 
    4980     32142384 :       prev_upper_bound = CASE_HIGH (elt);
    4981     32142384 :       if (! prev_upper_bound)
    4982     30139630 :         prev_upper_bound = CASE_LOW (elt);
    4983              :     }
    4984              : 
    4985              :   return false;
    4986              : }
    4987              : 
    4988              : /* Verify a gimple debug statement STMT.
    4989              :    Returns true if anything is wrong.  */
    4990              : 
    4991              : static bool
    4992            0 : verify_gimple_debug (gimple *stmt ATTRIBUTE_UNUSED)
    4993              : {
    4994              :   /* There isn't much that could be wrong in a gimple debug stmt.  A
    4995              :      gimple debug bind stmt, for example, maps a tree, that's usually
    4996              :      a VAR_DECL or a PARM_DECL, but that could also be some scalarized
    4997              :      component or member of an aggregate type, to another tree, that
    4998              :      can be an arbitrary expression.  These stmts expand into debug
    4999              :      insns, and are converted to debug notes by var-tracking.cc.  */
    5000            0 :   return false;
    5001              : }
    5002              : 
    5003              : /* Verify a gimple label statement STMT.
    5004              :    Returns true if anything is wrong.  */
    5005              : 
    5006              : static bool
    5007    232630906 : verify_gimple_label (glabel *stmt)
    5008              : {
    5009    232630906 :   tree decl = gimple_label_label (stmt);
    5010    232630906 :   int uid;
    5011    232630906 :   bool err = false;
    5012              : 
    5013    232630906 :   if (TREE_CODE (decl) != LABEL_DECL)
    5014              :     return true;
    5015    465202148 :   if (!DECL_NONLOCAL (decl) && !FORCED_LABEL (decl)
    5016    461851801 :       && DECL_CONTEXT (decl) != current_function_decl)
    5017              :     {
    5018            0 :       error ("label context is not the current function declaration");
    5019            0 :       err |= true;
    5020              :     }
    5021              : 
    5022    232630906 :   uid = LABEL_DECL_UID (decl);
    5023    232630906 :   if (cfun->cfg
    5024    232630906 :       && (uid == -1
    5025    144578209 :           || (*label_to_block_map_for_fn (cfun))[uid] != gimple_bb (stmt)))
    5026              :     {
    5027            0 :       error ("incorrect entry in %<label_to_block_map%>");
    5028            0 :       err |= true;
    5029              :     }
    5030              : 
    5031    232630906 :   uid = EH_LANDING_PAD_NR (decl);
    5032    232630906 :   if (uid)
    5033              :     {
    5034     80223310 :       eh_landing_pad lp = get_eh_landing_pad_from_number (uid);
    5035     80223310 :       if (decl != lp->post_landing_pad)
    5036              :         {
    5037            0 :           error ("incorrect setting of landing pad number");
    5038            0 :           err |= true;
    5039              :         }
    5040              :     }
    5041              : 
    5042              :   return err;
    5043              : }
    5044              : 
    5045              : /* Verify a gimple cond statement STMT.
    5046              :    Returns true if anything is wrong.  */
    5047              : 
    5048              : static bool
    5049    772629793 : verify_gimple_cond (gcond *stmt)
    5050              : {
    5051    772629793 :   if (TREE_CODE_CLASS (gimple_cond_code (stmt)) != tcc_comparison)
    5052              :     {
    5053            0 :       error ("invalid comparison code in gimple cond");
    5054            0 :       return true;
    5055              :     }
    5056    772629793 :   if (!(!gimple_cond_true_label (stmt)
    5057     29681174 :         || TREE_CODE (gimple_cond_true_label (stmt)) == LABEL_DECL)
    5058    802310967 :       || !(!gimple_cond_false_label (stmt)
    5059     29681174 :            || TREE_CODE (gimple_cond_false_label (stmt)) == LABEL_DECL))
    5060              :     {
    5061            0 :       error ("invalid labels in gimple cond");
    5062            0 :       return true;
    5063              :     }
    5064              : 
    5065    772629793 :   tree lhs = gimple_cond_lhs (stmt);
    5066              : 
    5067              :   /* GIMPLE_CONDs condition may not throw.  */
    5068    772629793 :   if (flag_exceptions
    5069    442725472 :       && cfun->can_throw_non_call_exceptions
    5070   1089235834 :       && operation_could_trap_p (gimple_cond_code (stmt),
    5071    316606041 :                                  FLOAT_TYPE_P (TREE_TYPE (lhs)),
    5072              :                                  false, NULL_TREE))
    5073              :     {
    5074            0 :       error ("gimple cond condition cannot throw");
    5075            0 :       return true;
    5076              :     }
    5077              : 
    5078    772629793 :   return verify_gimple_comparison (boolean_type_node,
    5079              :                                    gimple_cond_lhs (stmt),
    5080              :                                    gimple_cond_rhs (stmt),
    5081    772629793 :                                    gimple_cond_code (stmt));
    5082              : }
    5083              : 
    5084              : /* Verify the GIMPLE statement STMT.  Returns true if there is an
    5085              :    error, otherwise false.  */
    5086              : 
    5087              : static bool
    5088  13949767069 : verify_gimple_stmt (gimple *stmt)
    5089              : {
    5090  13949767069 :   switch (gimple_code (stmt))
    5091              :     {
    5092   4701992950 :     case GIMPLE_ASSIGN:
    5093   4701992950 :       return verify_gimple_assign (as_a <gassign *> (stmt));
    5094              : 
    5095    232630906 :     case GIMPLE_LABEL:
    5096    232630906 :       return verify_gimple_label (as_a <glabel *> (stmt));
    5097              : 
    5098   1101289116 :     case GIMPLE_CALL:
    5099   1101289116 :       return verify_gimple_call (as_a <gcall *> (stmt));
    5100              : 
    5101    772629793 :     case GIMPLE_COND:
    5102    772629793 :       return verify_gimple_cond (as_a <gcond *> (stmt));
    5103              : 
    5104     26301079 :     case GIMPLE_GOTO:
    5105     26301079 :       return verify_gimple_goto (as_a <ggoto *> (stmt));
    5106              : 
    5107      4802757 :     case GIMPLE_SWITCH:
    5108      4802757 :       return verify_gimple_switch (as_a <gswitch *> (stmt));
    5109              : 
    5110    263759581 :     case GIMPLE_RETURN:
    5111    263759581 :       return verify_gimple_return (as_a <greturn *> (stmt));
    5112              : 
    5113              :     case GIMPLE_ASM:
    5114              :       return false;
    5115              : 
    5116        31113 :     case GIMPLE_TRANSACTION:
    5117        31113 :       return verify_gimple_transaction (as_a <gtransaction *> (stmt));
    5118              : 
    5119              :     /* Tuples that do not have tree operands.  */
    5120              :     case GIMPLE_NOP:
    5121              :     case GIMPLE_PREDICT:
    5122              :     case GIMPLE_RESX:
    5123              :     case GIMPLE_EH_DISPATCH:
    5124              :     case GIMPLE_EH_MUST_NOT_THROW:
    5125              :       return false;
    5126              : 
    5127              :     CASE_GIMPLE_OMP:
    5128              :       /* OpenMP directives are validated by the FE and never operated
    5129              :          on by the optimizers.  Furthermore, GIMPLE_OMP_FOR may contain
    5130              :          non-gimple expressions when the main index variable has had
    5131              :          its address taken.  This does not affect the loop itself
    5132              :          because the header of an GIMPLE_OMP_FOR is merely used to determine
    5133              :          how to setup the parallel iteration.  */
    5134              :       return false;
    5135              : 
    5136              :     case GIMPLE_ASSUME:
    5137              :       return false;
    5138              : 
    5139              :     case GIMPLE_DEBUG:
    5140              :       return verify_gimple_debug (stmt);
    5141              : 
    5142            0 :     default:
    5143            0 :       gcc_unreachable ();
    5144              :     }
    5145              : }
    5146              : 
    5147              : /* Verify the contents of a GIMPLE_PHI.  Returns true if there is a problem,
    5148              :    and false otherwise.  */
    5149              : 
    5150              : static bool
    5151    663114980 : verify_gimple_phi (gphi *phi)
    5152              : {
    5153    663114980 :   bool err = false;
    5154    663114980 :   unsigned i;
    5155    663114980 :   tree phi_result = gimple_phi_result (phi);
    5156    663114980 :   bool virtual_p;
    5157              : 
    5158    663114980 :   if (!phi_result)
    5159              :     {
    5160            0 :       error ("invalid %<PHI%> result");
    5161            0 :       return true;
    5162              :     }
    5163              : 
    5164    663114980 :   virtual_p = virtual_operand_p (phi_result);
    5165    663114980 :   if (TREE_CODE (phi_result) != SSA_NAME
    5166    663114980 :       || (virtual_p
    5167    311063140 :           && SSA_NAME_VAR (phi_result) != gimple_vop (cfun)))
    5168              :     {
    5169            0 :       error ("invalid %<PHI%> result");
    5170            0 :       err = true;
    5171              :     }
    5172              : 
    5173   2346377890 :   for (i = 0; i < gimple_phi_num_args (phi); i++)
    5174              :     {
    5175   1683262910 :       tree t = gimple_phi_arg_def (phi, i);
    5176              : 
    5177   1683262910 :       if (!t)
    5178              :         {
    5179            0 :           error ("missing %<PHI%> def");
    5180            0 :           err |= true;
    5181            0 :           continue;
    5182              :         }
    5183              :       /* Addressable variables do have SSA_NAMEs but they
    5184              :          are not considered gimple values.  */
    5185   1683262910 :       else if ((TREE_CODE (t) == SSA_NAME
    5186   1451966558 :                 && virtual_p != virtual_operand_p (t))
    5187   1683262910 :                || (virtual_p
    5188    826665444 :                    && (TREE_CODE (t) != SSA_NAME
    5189    826665444 :                        || SSA_NAME_VAR (t) != gimple_vop (cfun)))
    5190   1683262910 :                || (!virtual_p
    5191    856597466 :                    && !is_gimple_val (t)))
    5192              :         {
    5193            0 :           error ("invalid %<PHI%> argument");
    5194            0 :           debug_generic_expr (t);
    5195            0 :           err |= true;
    5196              :         }
    5197              : #ifdef ENABLE_TYPES_CHECKING
    5198   1683262910 :       if (!useless_type_conversion_p (TREE_TYPE (phi_result), TREE_TYPE (t)))
    5199              :         {
    5200            0 :           error ("incompatible types in %<PHI%> argument %u", i);
    5201            0 :           debug_generic_stmt (TREE_TYPE (phi_result));
    5202            0 :           debug_generic_stmt (TREE_TYPE (t));
    5203            0 :           err |= true;
    5204              :         }
    5205              : #endif
    5206              :     }
    5207              : 
    5208              :   return err;
    5209              : }
    5210              : 
    5211              : /* Verify the GIMPLE statements inside the sequence STMTS.  */
    5212              : 
    5213              : static bool
    5214     57095974 : verify_gimple_in_seq_2 (gimple_seq stmts)
    5215              : {
    5216     57095974 :   gimple_stmt_iterator ittr;
    5217     57095974 :   bool err = false;
    5218              : 
    5219    551094412 :   for (ittr = gsi_start (stmts); !gsi_end_p (ittr); gsi_next (&ittr))
    5220              :     {
    5221    493998438 :       gimple *stmt = gsi_stmt (ittr);
    5222              : 
    5223    493998438 :       switch (gimple_code (stmt))
    5224              :         {
    5225     15209233 :         case GIMPLE_BIND:
    5226     15209233 :           err |= verify_gimple_in_seq_2 (
    5227     15209233 :                    gimple_bind_body (as_a <gbind *> (stmt)));
    5228     15209233 :           break;
    5229              : 
    5230     11725456 :         case GIMPLE_TRY:
    5231     11725456 :           err |= verify_gimple_in_seq_2 (gimple_try_eval (stmt));
    5232     11725456 :           err |= verify_gimple_in_seq_2 (gimple_try_cleanup (stmt));
    5233     11725456 :           break;
    5234              : 
    5235        22082 :         case GIMPLE_EH_FILTER:
    5236        22082 :           err |= verify_gimple_in_seq_2 (gimple_eh_filter_failure (stmt));
    5237        22082 :           break;
    5238              : 
    5239         1878 :         case GIMPLE_EH_ELSE:
    5240         1878 :           {
    5241         1878 :             geh_else *eh_else = as_a <geh_else *> (stmt);
    5242         1878 :             err |= verify_gimple_in_seq_2 (gimple_eh_else_n_body (eh_else));
    5243         1878 :             err |= verify_gimple_in_seq_2 (gimple_eh_else_e_body (eh_else));
    5244              :           }
    5245         1878 :           break;
    5246              : 
    5247       192491 :         case GIMPLE_CATCH:
    5248       192491 :           err |= verify_gimple_in_seq_2 (gimple_catch_handler (
    5249       192491 :                                            as_a <gcatch *> (stmt)));
    5250       192491 :           break;
    5251              : 
    5252          510 :         case GIMPLE_ASSUME:
    5253          510 :           err |= verify_gimple_in_seq_2 (gimple_assume_body (stmt));
    5254          510 :           break;
    5255              : 
    5256         3076 :         case GIMPLE_TRANSACTION:
    5257         3076 :           err |= verify_gimple_transaction (as_a <gtransaction *> (stmt));
    5258         3076 :           break;
    5259              : 
    5260    466843712 :         default:
    5261    466843712 :           {
    5262    466843712 :             bool err2 = verify_gimple_stmt (stmt);
    5263    466843712 :             if (err2)
    5264            1 :               debug_gimple_stmt (stmt);
    5265    466843712 :             err |= err2;
    5266              :           }
    5267              :         }
    5268              :     }
    5269              : 
    5270     57095974 :   return err;
    5271              : }
    5272              : 
    5273              : /* Verify the contents of a GIMPLE_TRANSACTION.  Returns true if there
    5274              :    is a problem, otherwise false.  */
    5275              : 
    5276              : static bool
    5277        34189 : verify_gimple_transaction (gtransaction *stmt)
    5278              : {
    5279        34189 :   tree lab;
    5280              : 
    5281        34189 :   lab = gimple_transaction_label_norm (stmt);
    5282        34189 :   if (lab != NULL && TREE_CODE (lab) != LABEL_DECL)
    5283              :     return true;
    5284        34189 :   lab = gimple_transaction_label_uninst (stmt);
    5285        34189 :   if (lab != NULL && TREE_CODE (lab) != LABEL_DECL)
    5286              :     return true;
    5287        34189 :   lab = gimple_transaction_label_over (stmt);
    5288        34189 :   if (lab != NULL && TREE_CODE (lab) != LABEL_DECL)
    5289              :     return true;
    5290              : 
    5291        34189 :   return verify_gimple_in_seq_2 (gimple_transaction_body (stmt));
    5292              : }
    5293              : 
    5294              : 
    5295              : /* Verify the GIMPLE statements inside the statement list STMTS.  */
    5296              : 
    5297              : DEBUG_FUNCTION bool
    5298     18182801 : verify_gimple_in_seq (gimple_seq stmts, bool ice)
    5299              : {
    5300     18182801 :   timevar_push (TV_TREE_STMT_VERIFY);
    5301     18182801 :   bool res = verify_gimple_in_seq_2 (stmts);
    5302     18182801 :   if (res && ice)
    5303            0 :     internal_error ("%<verify_gimple%> failed");
    5304     18182801 :   timevar_pop (TV_TREE_STMT_VERIFY);
    5305     18182801 :   return res;
    5306              : }
    5307              : 
    5308              : /* Return true when the T can be shared.  */
    5309              : 
    5310              : static bool
    5311  33852248936 : tree_node_can_be_shared (tree t)
    5312              : {
    5313  33852248936 :   if (IS_TYPE_OR_DECL_P (t)
    5314              :       || TREE_CODE (t) == SSA_NAME
    5315              :       || TREE_CODE (t) == IDENTIFIER_NODE
    5316              :       || TREE_CODE (t) == CASE_LABEL_EXPR
    5317              :       || TREE_CODE (t) == OMP_NEXT_VARIANT
    5318  46230473704 :       || is_gimple_min_invariant (t))
    5319              :     return true;
    5320              : 
    5321   4864492111 :   if (t == error_mark_node)
    5322            0 :     return true;
    5323              : 
    5324              :   return false;
    5325              : }
    5326              : 
    5327              : /* Called via walk_tree.  Verify tree sharing.  */
    5328              : 
    5329              : static tree
    5330  33852248936 : verify_node_sharing_1 (tree *tp, int *walk_subtrees, void *data)
    5331              : {
    5332  33852248936 :   hash_set<void *> *visited = (hash_set<void *> *) data;
    5333              : 
    5334  33852248936 :   if (tree_node_can_be_shared (*tp))
    5335              :     {
    5336  28987756825 :       *walk_subtrees = false;
    5337  28987756825 :       return NULL;
    5338              :     }
    5339              : 
    5340   4864492111 :   if (visited->add (*tp))
    5341            0 :     return *tp;
    5342              : 
    5343              :   return NULL;
    5344              : }
    5345              : 
    5346              : /* Called via walk_gimple_stmt.  Verify tree sharing.  */
    5347              : 
    5348              : static tree
    5349  32168986026 : verify_node_sharing (tree *tp, int *walk_subtrees, void *data)
    5350              : {
    5351  32168986026 :   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
    5352  32168986026 :   return verify_node_sharing_1 (tp, walk_subtrees, wi->info);
    5353              : }
    5354              : 
    5355              : static bool eh_error_found;
    5356              : bool
    5357    201618879 : verify_eh_throw_stmt_node (gimple *const &stmt, const int &,
    5358              :                            hash_set<gimple *> *visited)
    5359              : {
    5360    201618879 :   if (!visited->contains (stmt))
    5361              :     {
    5362            0 :       error ("dead statement in EH table");
    5363            0 :       debug_gimple_stmt (stmt);
    5364            0 :       eh_error_found = true;
    5365              :     }
    5366    201618879 :   return true;
    5367              : }
    5368              : 
    5369              : /* Verify if the location LOCs block is in BLOCKS.  */
    5370              : 
    5371              : static bool
    5372  23690768723 : verify_location (hash_set<tree> *blocks, location_t loc)
    5373              : {
    5374  38422055015 :   tree block = LOCATION_BLOCK (loc);
    5375  38422055015 :   if (block != NULL_TREE
    5376  38422055015 :       && !blocks->contains (block))
    5377              :     {
    5378            0 :       error ("location references block not in block tree");
    5379            0 :       return true;
    5380              :     }
    5381  38422055015 :   if (block != NULL_TREE)
    5382  14731286292 :     return verify_location (blocks, BLOCK_SOURCE_LOCATION (block));
    5383              :   return false;
    5384              : }
    5385              : 
    5386              : /* Called via walk_tree.  Verify that expressions have no blocks.  */
    5387              : 
    5388              : static tree
    5389   1666437248 : verify_expr_no_block (tree *tp, int *walk_subtrees, void *)
    5390              : {
    5391   1666437248 :   if (!EXPR_P (*tp))
    5392              :     {
    5393    997823610 :       *walk_subtrees = false;
    5394    997823610 :       return NULL;
    5395              :     }
    5396              : 
    5397    668613638 :   location_t loc = EXPR_LOCATION (*tp);
    5398    668613638 :   if (LOCATION_BLOCK (loc) != NULL)
    5399            0 :     return *tp;
    5400              : 
    5401              :   return NULL;
    5402              : }
    5403              : 
    5404              : /* Called via walk_tree.  Verify locations of expressions.  */
    5405              : 
    5406              : static tree
    5407  37769344956 : verify_expr_location_1 (tree *tp, int *walk_subtrees, void *data)
    5408              : {
    5409  37769344956 :   hash_set<tree> *blocks = (hash_set<tree> *) data;
    5410  37769344956 :   tree t = *tp;
    5411              : 
    5412              :   /* ???  This doesn't really belong here but there's no good place to
    5413              :      stick this remainder of old verify_expr.  */
    5414              :   /* ???  This barfs on debug stmts which contain binds to vars with
    5415              :      different function context.  */
    5416              : #if 0
    5417              :   if (VAR_P (t)
    5418              :       || TREE_CODE (t) == PARM_DECL
    5419              :       || TREE_CODE (t) == RESULT_DECL)
    5420              :     {
    5421              :       tree context = decl_function_context (t);
    5422              :       if (context != cfun->decl
    5423              :           && !SCOPE_FILE_SCOPE_P (context)
    5424              :           && !TREE_STATIC (t)
    5425              :           && !DECL_EXTERNAL (t))
    5426              :         {
    5427              :           error ("local declaration from a different function");
    5428              :           return t;
    5429              :         }
    5430              :     }
    5431              : #endif
    5432              : 
    5433  37769344956 :   if (VAR_P (t) && DECL_HAS_DEBUG_EXPR_P (t))
    5434              :     {
    5435    436125150 :       tree x = DECL_DEBUG_EXPR (t);
    5436    436125150 :       tree addr = walk_tree (&x, verify_expr_no_block, NULL, NULL);
    5437    436125150 :       if (addr)
    5438            0 :         return addr;
    5439              :     }
    5440  37769344956 :   if ((VAR_P (t)
    5441              :        || TREE_CODE (t) == PARM_DECL
    5442              :        || TREE_CODE (t) == RESULT_DECL)
    5443   8476114754 :       && DECL_HAS_VALUE_EXPR_P (t))
    5444              :     {
    5445         1946 :       tree x = DECL_VALUE_EXPR (t);
    5446         1946 :       tree addr = walk_tree (&x, verify_expr_no_block, NULL, NULL);
    5447         1946 :       if (addr)
    5448            0 :         return addr;
    5449              :     }
    5450              : 
    5451  37769344956 :   if (!EXPR_P (t))
    5452              :     {
    5453  29783673197 :       *walk_subtrees = false;
    5454  29783673197 :       return NULL;
    5455              :     }
    5456              : 
    5457   7985671759 :   location_t loc = EXPR_LOCATION (t);
    5458   7985671759 :   if (verify_location (blocks, loc))
    5459            0 :     return t;
    5460              : 
    5461              :   return NULL;
    5462              : }
    5463              : 
    5464              : /* Called via walk_gimple_op.  Verify locations of expressions.  */
    5465              : 
    5466              : static tree
    5467  36056465441 : verify_expr_location (tree *tp, int *walk_subtrees, void *data)
    5468              : {
    5469  36056465441 :   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
    5470  36056465441 :   return verify_expr_location_1 (tp, walk_subtrees, wi->info);
    5471              : }
    5472              : 
    5473              : /* Insert all subblocks of BLOCK into BLOCKS and recurse.  */
    5474              : 
    5475              : static void
    5476   2502882178 : collect_subblocks (hash_set<tree> *blocks, tree block)
    5477              : {
    5478   2502882178 :   tree t;
    5479   4753685997 :   for (t = BLOCK_SUBBLOCKS (block); t; t = BLOCK_CHAIN (t))
    5480              :     {
    5481   2250803819 :       blocks->add (t);
    5482   2250803819 :       collect_subblocks (blocks, t);
    5483              :     }
    5484   2502882178 : }
    5485              : 
    5486              : /* Disable warnings about missing quoting in GCC diagnostics for
    5487              :    the verification errors.  Their format strings don't follow
    5488              :    GCC diagnostic conventions and trigger an ICE in the end.  */
    5489              : #if __GNUC__ >= 10
    5490              : #  pragma GCC diagnostic push
    5491              : #  pragma GCC diagnostic ignored "-Wformat-diag"
    5492              : #endif
    5493              : 
    5494              : /* Verify the GIMPLE statements in the CFG of FN.  */
    5495              : 
    5496              : DEBUG_FUNCTION bool
    5497    252078359 : verify_gimple_in_cfg (struct function *fn, bool verify_nothrow, bool ice)
    5498              : {
    5499    252078359 :   basic_block bb;
    5500    252078359 :   bool err = false;
    5501              : 
    5502    252078359 :   timevar_push (TV_TREE_STMT_VERIFY);
    5503    252078359 :   hash_set<void *> visited;
    5504    252078359 :   hash_set<gimple *> visited_throwing_stmts;
    5505              : 
    5506              :   /* Collect all BLOCKs referenced by the BLOCK tree of FN.  */
    5507    252078359 :   hash_set<tree> blocks;
    5508    252078359 :   if (DECL_INITIAL (fn->decl))
    5509              :     {
    5510    252078359 :       blocks.add (DECL_INITIAL (fn->decl));
    5511    252078359 :       collect_subblocks (&blocks, DECL_INITIAL (fn->decl));
    5512              :     }
    5513              : 
    5514   2261014364 :   FOR_EACH_BB_FN (bb, fn)
    5515              :     {
    5516   2008936005 :       gimple_stmt_iterator gsi;
    5517   2008936005 :       edge_iterator ei;
    5518   2008936005 :       edge e;
    5519              : 
    5520   2008936005 :       for (gphi_iterator gpi = gsi_start_phis (bb);
    5521   2672050985 :            !gsi_end_p (gpi);
    5522    663114980 :            gsi_next (&gpi))
    5523              :         {
    5524    663114980 :           gphi *phi = gpi.phi ();
    5525    663114980 :           bool err2 = false;
    5526    663114980 :           unsigned i;
    5527              : 
    5528    663114980 :           if (gimple_bb (phi) != bb)
    5529              :             {
    5530            0 :               error ("gimple_bb (phi) is set to a wrong basic block");
    5531            0 :               err2 = true;
    5532              :             }
    5533              : 
    5534    663114980 :           err2 |= verify_gimple_phi (phi);
    5535              : 
    5536              :           /* Only PHI arguments have locations.  */
    5537    663114980 :           if (gimple_location (phi) != UNKNOWN_LOCATION)
    5538              :             {
    5539            0 :               error ("PHI node with location");
    5540            0 :               err2 = true;
    5541              :             }
    5542              : 
    5543   2346377890 :           for (i = 0; i < gimple_phi_num_args (phi); i++)
    5544              :             {
    5545   1683262910 :               tree arg = gimple_phi_arg_def (phi, i);
    5546   1683262910 :               tree addr = walk_tree (&arg, verify_node_sharing_1,
    5547              :                                      &visited, NULL);
    5548   1683262910 :               if (addr)
    5549              :                 {
    5550            0 :                   error ("incorrect sharing of tree nodes");
    5551            0 :                   debug_generic_expr (addr);
    5552            0 :                   err2 |= true;
    5553              :                 }
    5554   1683262910 :               location_t loc = gimple_phi_arg_location (phi, i);
    5555   1683262910 :               if (virtual_operand_p (gimple_phi_result (phi))
    5556   1683262910 :                   && loc != UNKNOWN_LOCATION)
    5557              :                 {
    5558            0 :                   error ("virtual PHI with argument locations");
    5559            0 :                   err2 = true;
    5560              :                 }
    5561   1683262910 :               addr = walk_tree (&arg, verify_expr_location_1, &blocks, NULL);
    5562   1683262910 :               if (addr)
    5563              :                 {
    5564            0 :                   debug_generic_expr (addr);
    5565            0 :                   err2 = true;
    5566              :                 }
    5567   1683262910 :               err2 |= verify_location (&blocks, loc);
    5568              :             }
    5569              : 
    5570    663114980 :           if (err2)
    5571            0 :             debug_gimple_stmt (phi);
    5572    663114980 :           err |= err2;
    5573              :         }
    5574              : 
    5575  17500795367 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    5576              :         {
    5577  13482923357 :           gimple *stmt = gsi_stmt (gsi);
    5578  13482923357 :           bool err2 = false;
    5579  13482923357 :           struct walk_stmt_info wi;
    5580  13482923357 :           tree addr;
    5581  13482923357 :           int lp_nr;
    5582              : 
    5583  13482923357 :           if (gimple_bb (stmt) != bb)
    5584              :             {
    5585            0 :               error ("gimple_bb (stmt) is set to a wrong basic block");
    5586            0 :               err2 = true;
    5587              :             }
    5588              : 
    5589  13482923357 :           err2 |= verify_gimple_stmt (stmt);
    5590  13482923357 :           err2 |= verify_location (&blocks, gimple_location (stmt));
    5591              : 
    5592  13482923357 :           memset (&wi, 0, sizeof (wi));
    5593  13482923357 :           wi.info = (void *) &visited;
    5594  13482923357 :           addr = walk_gimple_op (stmt, verify_node_sharing, &wi);
    5595  13482923357 :           if (addr)
    5596              :             {
    5597            0 :               error ("incorrect sharing of tree nodes");
    5598            0 :               debug_generic_expr (addr);
    5599            0 :               err2 |= true;
    5600              :             }
    5601              : 
    5602  13482923357 :           memset (&wi, 0, sizeof (wi));
    5603  13482923357 :           wi.info = (void *) &blocks;
    5604  13482923357 :           addr = walk_gimple_op (stmt, verify_expr_location, &wi);
    5605  13482923357 :           if (addr)
    5606              :             {
    5607            0 :               debug_generic_expr (addr);
    5608            0 :               err2 |= true;
    5609              :             }
    5610              : 
    5611              :           /* If the statement is marked as part of an EH region, then it is
    5612              :              expected that the statement could throw.  Verify that when we
    5613              :              have optimizations that simplify statements such that we prove
    5614              :              that they cannot throw, that we update other data structures
    5615              :              to match.  */
    5616  13482923357 :           lp_nr = lookup_stmt_eh_lp (stmt);
    5617  13482923357 :           if (lp_nr != 0)
    5618    201618879 :             visited_throwing_stmts.add (stmt);
    5619    201618879 :           if (lp_nr > 0)
    5620              :             {
    5621    193613935 :               if (!stmt_could_throw_p (cfun, stmt))
    5622              :                 {
    5623           24 :                   if (verify_nothrow)
    5624              :                     {
    5625            0 :                       error ("statement marked for throw, but doesn%'t");
    5626            0 :                       err2 |= true;
    5627              :                     }
    5628              :                 }
    5629    193613911 :               else if (!gsi_one_before_end_p (gsi))
    5630              :                 {
    5631            0 :                   error ("statement marked for throw in middle of block");
    5632            0 :                   err2 |= true;
    5633              :                 }
    5634              :             }
    5635              : 
    5636  13482923357 :           if (err2)
    5637            0 :             debug_gimple_stmt (stmt);
    5638  13482923357 :           err |= err2;
    5639              :         }
    5640              : 
    5641   4785277657 :       FOR_EACH_EDGE (e, ei, bb->succs)
    5642   2776341652 :         if (e->goto_locus != UNKNOWN_LOCATION)
    5643    538910697 :           err |= verify_location (&blocks, e->goto_locus);
    5644              :     }
    5645              : 
    5646    252078359 :   hash_map<gimple *, int> *eh_table = get_eh_throw_stmt_table (cfun);
    5647    252078359 :   eh_error_found = false;
    5648    252078359 :   if (eh_table)
    5649     37770918 :     eh_table->traverse<hash_set<gimple *> *, verify_eh_throw_stmt_node>
    5650    239389797 :       (&visited_throwing_stmts);
    5651              : 
    5652    252078359 :   if (ice && (err || eh_error_found))
    5653            0 :     internal_error ("verify_gimple failed");
    5654              : 
    5655    252078359 :   verify_histograms ();
    5656    252078359 :   timevar_pop (TV_TREE_STMT_VERIFY);
    5657              : 
    5658    252078359 :   return (err || eh_error_found);
    5659    252078359 : }
    5660              : 
    5661              : 
    5662              : /* Verifies that the flow information is OK.  */
    5663              : 
    5664              : static bool
    5665    235870670 : gimple_verify_flow_info (void)
    5666              : {
    5667    235870670 :   bool err = false;
    5668    235870670 :   basic_block bb;
    5669    235870670 :   gimple_stmt_iterator gsi;
    5670    235870670 :   gimple *stmt;
    5671    235870670 :   edge e;
    5672    235870670 :   edge_iterator ei;
    5673              : 
    5674    235870670 :   if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->il.gimple.seq
    5675    235870670 :       || ENTRY_BLOCK_PTR_FOR_FN (cfun)->il.gimple.phi_nodes)
    5676              :     {
    5677            0 :       error ("ENTRY_BLOCK has IL associated with it");
    5678            0 :       err = true;
    5679              :     }
    5680              : 
    5681    235870670 :   if (EXIT_BLOCK_PTR_FOR_FN (cfun)->il.gimple.seq
    5682    235870670 :       || EXIT_BLOCK_PTR_FOR_FN (cfun)->il.gimple.phi_nodes)
    5683              :     {
    5684            0 :       error ("EXIT_BLOCK has IL associated with it");
    5685            0 :       err = true;
    5686              :     }
    5687              : 
    5688    467725773 :   FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
    5689    231855103 :     if (e->flags & EDGE_FALLTHRU)
    5690              :       {
    5691            0 :         error ("fallthru to exit from bb %d", e->src->index);
    5692            0 :         err = true;
    5693              :       }
    5694    235870670 :   if (cfun->cfg->full_profile
    5695    235870670 :       && !ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ())
    5696              :     {
    5697            0 :       error ("entry block count not initialized");
    5698            0 :       err = true;
    5699              :     }
    5700    235870670 :   if (cfun->cfg->full_profile
    5701    235870670 :       && !EXIT_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ())
    5702              :     {
    5703            0 :       error ("exit block count not initialized");
    5704            0 :       err = true;
    5705              :     }
    5706    235870670 :   if (cfun->cfg->full_profile
    5707    355588038 :       && !single_succ_edge
    5708    119717368 :               (ENTRY_BLOCK_PTR_FOR_FN (cfun))->probability.initialized_p ())
    5709              :     {
    5710            0 :       error ("probability of edge from entry block not initialized");
    5711            0 :       err = true;
    5712              :     }
    5713    235870670 :   if (!EXIT_BLOCK_PTR_FOR_FN (cfun)
    5714    235870670 :         ->count.compatible_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count))
    5715              :     {
    5716            0 :       error ("exit block count is not compatible with entry block count");
    5717            0 :       err = true;
    5718              :     }
    5719              : 
    5720              : 
    5721   2268419061 :   FOR_EACH_BB_FN (bb, cfun)
    5722              :     {
    5723   2032548391 :       bool found_ctrl_stmt = false;
    5724              : 
    5725   2032548391 :       stmt = NULL;
    5726              : 
    5727   2032548391 :       if (cfun->cfg->full_profile)
    5728              :         {
    5729   1332687617 :           if (!bb->count.initialized_p ())
    5730              :             {
    5731            0 :               error ("count of bb %d not initialized", bb->index);
    5732            0 :               err = true;
    5733              :             }
    5734   3211482430 :           FOR_EACH_EDGE (e, ei, bb->succs)
    5735   1878794813 :             if (!e->probability.initialized_p ())
    5736              :               {
    5737            0 :                 error ("probability of edge %d->%d not initialized",
    5738            0 :                        bb->index, e->dest->index);
    5739            0 :                 err = true;
    5740              :               }
    5741              :         }
    5742   2032548391 :       if (!bb->count.compatible_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count))
    5743              :         {
    5744            0 :           error ("count of bb %d is not compatible with entry block count",
    5745              :                  bb->index);
    5746            0 :           err = true;
    5747              :         }
    5748              : 
    5749              :       /* Skip labels on the start of basic block.  */
    5750   4215337008 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    5751              :         {
    5752   2022118290 :           tree label;
    5753   2022118290 :           gimple *prev_stmt = stmt;
    5754              : 
    5755   2022118290 :           stmt = gsi_stmt (gsi);
    5756              : 
    5757   2022118290 :           if (gimple_code (stmt) != GIMPLE_LABEL)
    5758              :             break;
    5759              : 
    5760    150240226 :           label = gimple_label_label (as_a <glabel *> (stmt));
    5761    150240226 :           if (prev_stmt && DECL_NONLOCAL (label))
    5762              :             {
    5763            0 :               error ("nonlocal label %qD is not first in a sequence "
    5764              :                      "of labels in bb %d", label, bb->index);
    5765            0 :               err = true;
    5766              :             }
    5767              : 
    5768    152968575 :           if (prev_stmt && EH_LANDING_PAD_NR (label) != 0)
    5769              :             {
    5770            0 :               error ("EH landing pad label %qD is not first in a sequence "
    5771              :                      "of labels in bb %d", label, bb->index);
    5772            0 :               err = true;
    5773              :             }
    5774              : 
    5775    150240226 :           if (label_to_block (cfun, label) != bb)
    5776              :             {
    5777            0 :               error ("label %qD to block does not match in bb %d",
    5778              :                      label, bb->index);
    5779            0 :               err = true;
    5780              :             }
    5781              : 
    5782    150240226 :           if (decl_function_context (label) != current_function_decl)
    5783              :             {
    5784            0 :               error ("label %qD has incorrect context in bb %d",
    5785              :                      label, bb->index);
    5786            0 :               err = true;
    5787              :             }
    5788              :         }
    5789              : 
    5790              :       /* Verify that body of basic block BB is free of control flow.  */
    5791   2032548391 :       bool seen_nondebug_stmt = false;
    5792  16566502479 :       for (; !gsi_end_p (gsi); gsi_next (&gsi))
    5793              :         {
    5794  14533954088 :           gimple *stmt = gsi_stmt (gsi);
    5795              : 
    5796              :           /* Do NOT disregard debug stmts after found_ctrl_stmt.  */
    5797  14533954088 :           if (found_ctrl_stmt)
    5798              :             {
    5799            0 :               error ("control flow in the middle of basic block %d",
    5800              :                      bb->index);
    5801            0 :               err = true;
    5802              :             }
    5803              : 
    5804  14533954088 :           if (stmt_ends_bb_p (stmt))
    5805   1372847086 :             found_ctrl_stmt = true;
    5806              : 
    5807  14533954088 :           if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
    5808              :             {
    5809            0 :               error ("label %qD in the middle of basic block %d",
    5810              :                      gimple_label_label (label_stmt), bb->index);
    5811            0 :               err = true;
    5812              :             }
    5813              : 
    5814              :           /* Check that no statements appear between a returns_twice call
    5815              :              and its associated abnormal edge.  */
    5816  14533954088 :           if (gimple_code (stmt) == GIMPLE_CALL
    5817  14533954088 :               && gimple_call_flags (stmt) & ECF_RETURNS_TWICE)
    5818              :             {
    5819       153121 :               bool misplaced = false;
    5820              :               /* TM is an exception: it points abnormal edges just after the
    5821              :                  call that starts a transaction, i.e. it must end the BB.  */
    5822       153121 :               if (gimple_call_builtin_p (stmt, BUILT_IN_TM_START))
    5823              :                 {
    5824         3779 :                   if (single_succ_p (bb)
    5825         2989 :                       && bb_has_abnormal_pred (single_succ (bb))
    5826         6121 :                       && !gsi_one_nondebug_before_end_p (gsi))
    5827              :                     {
    5828            0 :                       error ("returns_twice call is not last in basic block "
    5829              :                              "%d", bb->index);
    5830            0 :                       misplaced = true;
    5831              :                     }
    5832              :                 }
    5833              :               else
    5834              :                 {
    5835       149342 :                   if (seen_nondebug_stmt && bb_has_abnormal_pred (bb))
    5836              :                     {
    5837            0 :                       error ("returns_twice call is not first in basic block "
    5838              :                              "%d", bb->index);
    5839            0 :                       misplaced = true;
    5840              :                     }
    5841              :                 }
    5842            0 :               if (misplaced)
    5843              :                 {
    5844            0 :                   print_gimple_stmt (stderr, stmt, 0, TDF_SLIM);
    5845            0 :                   err = true;
    5846              :                 }
    5847              :             }
    5848  14533954088 :           if (!is_gimple_debug (stmt))
    5849   6693854293 :             seen_nondebug_stmt = true;
    5850              :         }
    5851              : 
    5852   2032548391 :       gsi = gsi_last_nondebug_bb (bb);
    5853   2032548391 :       if (gsi_end_p (gsi))
    5854    130731310 :         continue;
    5855              : 
    5856   1901817081 :       stmt = gsi_stmt (gsi);
    5857              : 
    5858   1901817081 :       if (gimple_code (stmt) == GIMPLE_LABEL)
    5859     42665866 :         continue;
    5860              : 
    5861   1859151215 :       if (verify_eh_edges (stmt))
    5862            0 :         err = true;
    5863              : 
    5864   1859151215 :       if (is_ctrl_stmt (stmt))
    5865              :         {
    5866   2893438826 :           FOR_EACH_EDGE (e, ei, bb->succs)
    5867   1834578423 :             if (e->flags & EDGE_FALLTHRU)
    5868              :               {
    5869            0 :                 error ("fallthru edge after a control statement in bb %d",
    5870              :                        bb->index);
    5871            0 :                 err = true;
    5872              :               }
    5873              :         }
    5874              : 
    5875   1859151215 :       if (gimple_code (stmt) != GIMPLE_COND)
    5876              :         {
    5877              :           /* Verify that there are no edges with EDGE_TRUE/FALSE_FLAG set
    5878              :              after anything else but if statement.  */
    5879   2201384079 :           FOR_EACH_EDGE (e, ei, bb->succs)
    5880   1115856808 :             if (e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE))
    5881              :               {
    5882            0 :                 error ("true/false edge after a non-GIMPLE_COND in bb %d",
    5883              :                        bb->index);
    5884            0 :                 err = true;
    5885              :               }
    5886              :         }
    5887              : 
    5888   1859151215 :       switch (gimple_code (stmt))
    5889              :         {
    5890    773623944 :         case GIMPLE_COND:
    5891    773623944 :           {
    5892    773623944 :             edge true_edge;
    5893    773623944 :             edge false_edge;
    5894              : 
    5895    773623944 :             extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
    5896              : 
    5897    773623944 :             if (!true_edge
    5898    773623944 :                 || !false_edge
    5899    773623944 :                 || !(true_edge->flags & EDGE_TRUE_VALUE)
    5900    773623944 :                 || !(false_edge->flags & EDGE_FALSE_VALUE)
    5901    773623944 :                 || (true_edge->flags & (EDGE_FALLTHRU | EDGE_ABNORMAL))
    5902    773623944 :                 || (false_edge->flags & (EDGE_FALLTHRU | EDGE_ABNORMAL))
    5903   1547247888 :                 || EDGE_COUNT (bb->succs) >= 3)
    5904              :               {
    5905            0 :                 error ("wrong outgoing edge flags at end of bb %d",
    5906              :                        bb->index);
    5907            0 :                 err = true;
    5908              :               }
    5909              :           }
    5910    773623944 :           break;
    5911              : 
    5912        51924 :         case GIMPLE_GOTO:
    5913        51924 :           if (simple_goto_p (stmt))
    5914              :             {
    5915            0 :               error ("explicit goto at end of bb %d", bb->index);
    5916            0 :               err = true;
    5917              :             }
    5918              :           else
    5919              :             {
    5920              :               /* FIXME.  We should double check that the labels in the
    5921              :                  destination blocks have their address taken.  */
    5922       198295 :               FOR_EACH_EDGE (e, ei, bb->succs)
    5923       146371 :                 if ((e->flags & (EDGE_FALLTHRU | EDGE_TRUE_VALUE
    5924              :                                  | EDGE_FALSE_VALUE))
    5925       146371 :                     || !(e->flags & EDGE_ABNORMAL))
    5926              :                   {
    5927            0 :                     error ("wrong outgoing edge flags at end of bb %d",
    5928              :                            bb->index);
    5929            0 :                     err = true;
    5930              :                   }
    5931              :             }
    5932              :           break;
    5933              : 
    5934    396992990 :         case GIMPLE_CALL:
    5935    396992990 :           if (!gimple_call_builtin_p (stmt, BUILT_IN_RETURN))
    5936              :             break;
    5937              :           /* fallthru */
    5938    231850653 :         case GIMPLE_RETURN:
    5939    231850653 :           if (!single_succ_p (bb)
    5940    231850653 :               || (single_succ_edge (bb)->flags
    5941    231850653 :                   & (EDGE_FALLTHRU | EDGE_ABNORMAL
    5942              :                      | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
    5943              :             {
    5944            0 :               error ("wrong outgoing edge flags at end of bb %d", bb->index);
    5945            0 :               err = true;
    5946              :             }
    5947    231850653 :           if (single_succ (bb) != EXIT_BLOCK_PTR_FOR_FN (cfun))
    5948              :             {
    5949            0 :               error ("return edge does not point to exit in bb %d",
    5950              :                      bb->index);
    5951            0 :               err = true;
    5952              :             }
    5953              :           break;
    5954              : 
    5955      4346558 :         case GIMPLE_SWITCH:
    5956      4346558 :           {
    5957      4346558 :             gswitch *switch_stmt = as_a <gswitch *> (stmt);
    5958      4346558 :             tree prev;
    5959      4346558 :             edge e;
    5960      4346558 :             size_t i, n;
    5961              : 
    5962      4346558 :             n = gimple_switch_num_labels (switch_stmt);
    5963              : 
    5964              :             /* Mark all the destination basic blocks.  */
    5965     34714079 :             for (i = 0; i < n; ++i)
    5966              :               {
    5967     30367521 :                 basic_block label_bb = gimple_switch_label_bb (cfun, switch_stmt, i);
    5968     30367521 :                 gcc_assert (!label_bb->aux || label_bb->aux == (void *)1);
    5969     30367521 :                 label_bb->aux = (void *)1;
    5970              :               }
    5971              : 
    5972              :             /* Verify that the case labels are sorted.  */
    5973      4346558 :             prev = gimple_switch_label (switch_stmt, 0);
    5974     30367521 :             for (i = 1; i < n; ++i)
    5975              :               {
    5976     26020963 :                 tree c = gimple_switch_label (switch_stmt, i);
    5977     26020963 :                 if (!CASE_LOW (c))
    5978              :                   {
    5979            0 :                     error ("found default case not at the start of "
    5980              :                            "case vector");
    5981            0 :                     err = true;
    5982            0 :                     continue;
    5983              :                   }
    5984     26020963 :                 if (CASE_LOW (prev)
    5985     26020963 :                     && !tree_int_cst_lt (CASE_LOW (prev), CASE_LOW (c)))
    5986              :                   {
    5987            0 :                     error ("case labels not sorted: ");
    5988            0 :                     print_generic_expr (stderr, prev);
    5989            0 :                     fprintf (stderr," is greater than ");
    5990            0 :                     print_generic_expr (stderr, c);
    5991            0 :                     fprintf (stderr," but comes before it.\n");
    5992            0 :                     err = true;
    5993              :                   }
    5994              :                 prev = c;
    5995              :               }
    5996              :             /* VRP will remove the default case if it can prove it will
    5997              :                never be executed.  So do not verify there always exists
    5998              :                a default case here.  */
    5999              : 
    6000     31161588 :             FOR_EACH_EDGE (e, ei, bb->succs)
    6001              :               {
    6002     26815030 :                 if (!e->dest->aux)
    6003              :                   {
    6004            0 :                     error ("extra outgoing edge %d->%d",
    6005              :                            bb->index, e->dest->index);
    6006            0 :                     err = true;
    6007              :                   }
    6008              : 
    6009     26815030 :                 e->dest->aux = (void *)2;
    6010     26815030 :                 if ((e->flags & (EDGE_FALLTHRU | EDGE_ABNORMAL
    6011              :                                  | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
    6012              :                   {
    6013            0 :                     error ("wrong outgoing edge flags at end of bb %d",
    6014              :                            bb->index);
    6015            0 :                     err = true;
    6016              :                   }
    6017              :               }
    6018              : 
    6019              :             /* Check that we have all of them.  */
    6020     34714079 :             for (i = 0; i < n; ++i)
    6021              :               {
    6022     30367521 :                 basic_block label_bb = gimple_switch_label_bb (cfun,
    6023              :                                                                switch_stmt, i);
    6024              : 
    6025     30367521 :                 if (label_bb->aux != (void *)2)
    6026              :                   {
    6027            0 :                     error ("missing edge %i->%i", bb->index, label_bb->index);
    6028            0 :                     err = true;
    6029              :                   }
    6030              :               }
    6031              : 
    6032     31161588 :             FOR_EACH_EDGE (e, ei, bb->succs)
    6033     26815030 :               e->dest->aux = (void *)0;
    6034              :           }
    6035      4346558 :           break;
    6036              : 
    6037      2017260 :         case GIMPLE_EH_DISPATCH:
    6038      2017260 :           if (verify_eh_dispatch_edge (as_a <geh_dispatch *> (stmt)))
    6039            0 :             err = true;
    6040              :           break;
    6041              : 
    6042              :         default:
    6043              :           break;
    6044              :         }
    6045              :     }
    6046              : 
    6047    235870670 :   if (dom_info_state (CDI_DOMINATORS) >= DOM_NO_FAST_QUERY)
    6048    191984509 :     verify_dominators (CDI_DOMINATORS);
    6049              : 
    6050    235870670 :   return err;
    6051              : }
    6052              : 
    6053              : #if __GNUC__ >= 10
    6054              : #  pragma GCC diagnostic pop
    6055              : #endif
    6056              : 
    6057              : /* Updates phi nodes after creating a forwarder block joined
    6058              :    by edge FALLTHRU.  */
    6059              : 
    6060              : static void
    6061        98005 : gimple_make_forwarder_block (edge fallthru)
    6062              : {
    6063        98005 :   edge e;
    6064        98005 :   edge_iterator ei;
    6065        98005 :   basic_block dummy, bb;
    6066        98005 :   tree var;
    6067        98005 :   gphi_iterator gsi;
    6068        98005 :   bool forward_location_p;
    6069              : 
    6070        98005 :   dummy = fallthru->src;
    6071        98005 :   bb = fallthru->dest;
    6072              : 
    6073        98005 :   if (single_pred_p (bb))
    6074          152 :     return;
    6075              : 
    6076              :   /* We can forward location info if we have only one predecessor.  */
    6077        97853 :   forward_location_p = single_pred_p (dummy);
    6078              : 
    6079              :   /* If we redirected a branch we must create new PHI nodes at the
    6080              :      start of BB.  */
    6081       335535 :   for (gsi = gsi_start_phis (dummy); !gsi_end_p (gsi); gsi_next (&gsi))
    6082              :     {
    6083       237682 :       gphi *phi, *new_phi;
    6084              : 
    6085       237682 :       phi = gsi.phi ();
    6086       237682 :       var = gimple_phi_result (phi);
    6087       237682 :       new_phi = create_phi_node (var, bb);
    6088       237682 :       gimple_phi_set_result (phi, copy_ssa_name (var, phi));
    6089       237686 :       add_phi_arg (new_phi, gimple_phi_result (phi), fallthru,
    6090              :                    forward_location_p
    6091            4 :                    ? gimple_phi_arg_location (phi, 0) : UNKNOWN_LOCATION);
    6092              :     }
    6093              : 
    6094              :   /* Add the arguments we have stored on edges.  */
    6095       296889 :   FOR_EACH_EDGE (e, ei, bb->preds)
    6096              :     {
    6097       199036 :       if (e == fallthru)
    6098        97853 :         continue;
    6099              : 
    6100       101183 :       flush_pending_stmts (e);
    6101              :     }
    6102              : }
    6103              : 
    6104              : 
    6105              : /* Return a non-special label in the head of basic block BLOCK.
    6106              :    Create one if it doesn't exist.  */
    6107              : 
    6108              : tree
    6109      5979017 : gimple_block_label (basic_block bb)
    6110              : {
    6111      5979017 :   gimple_stmt_iterator i, s = gsi_start_bb (bb);
    6112      5979017 :   bool first = true;
    6113      5979017 :   tree label;
    6114      5979017 :   glabel *stmt;
    6115              : 
    6116      5979017 :   for (i = s; !gsi_end_p (i); first = false, gsi_next (&i))
    6117              :     {
    6118      4869777 :       stmt = dyn_cast <glabel *> (gsi_stmt (i));
    6119      4774482 :       if (!stmt)
    6120              :         break;
    6121      4774482 :       label = gimple_label_label (stmt);
    6122      4774482 :       if (!DECL_NONLOCAL (label))
    6123              :         {
    6124      4774482 :           if (!first)
    6125            0 :             gsi_move_before (&i, &s);
    6126              :           return label;
    6127              :         }
    6128              :     }
    6129              : 
    6130      1204535 :   label = create_artificial_label (UNKNOWN_LOCATION);
    6131      1204535 :   stmt = gimple_build_label (label);
    6132      1204535 :   gsi_insert_before (&s, stmt, GSI_NEW_STMT);
    6133      1204535 :   return label;
    6134              : }
    6135              : 
    6136              : 
    6137              : /* Attempt to perform edge redirection by replacing a possibly complex
    6138              :    jump instruction by a goto or by removing the jump completely.
    6139              :    This can apply only if all edges now point to the same block.  The
    6140              :    parameters and return values are equivalent to
    6141              :    redirect_edge_and_branch.  */
    6142              : 
    6143              : static edge
    6144     64850054 : gimple_try_redirect_by_replacing_jump (edge e, basic_block target)
    6145              : {
    6146     64850054 :   basic_block src = e->src;
    6147     64850054 :   gimple_stmt_iterator i;
    6148     64850054 :   gimple *stmt;
    6149              : 
    6150              :   /* We can replace or remove a complex jump only when we have exactly
    6151              :      two edges.  */
    6152     64850054 :   if (EDGE_COUNT (src->succs) != 2
    6153              :       /* Verify that all targets will be TARGET.  Specifically, the
    6154              :          edge that is not E must also go to TARGET.  */
    6155     64850054 :       || EDGE_SUCC (src, EDGE_SUCC (src, 0) == e)->dest != target)
    6156              :     return NULL;
    6157              : 
    6158       429228 :   i = gsi_last_bb (src);
    6159       429228 :   if (gsi_end_p (i))
    6160              :     return NULL;
    6161              : 
    6162       429228 :   stmt = gsi_stmt (i);
    6163              : 
    6164       429228 :   if (gimple_code (stmt) == GIMPLE_COND || gimple_code (stmt) == GIMPLE_SWITCH)
    6165              :     {
    6166       429198 :       gsi_remove (&i, true);
    6167       429198 :       e = ssa_redirect_edge (e, target);
    6168       429198 :       e->flags = EDGE_FALLTHRU;
    6169       429198 :       return e;
    6170              :     }
    6171              : 
    6172              :   return NULL;
    6173              : }
    6174              : 
    6175              : 
    6176              : /* Redirect E to DEST.  Return NULL on failure.  Otherwise, return the
    6177              :    edge representing the redirected branch.  */
    6178              : 
    6179              : static edge
    6180     66145949 : gimple_redirect_edge_and_branch (edge e, basic_block dest)
    6181              : {
    6182     66145949 :   basic_block bb = e->src;
    6183     66145949 :   gimple_stmt_iterator gsi;
    6184     66145949 :   edge ret;
    6185     66145949 :   gimple *stmt;
    6186              : 
    6187     66145949 :   if (e->flags & EDGE_ABNORMAL)
    6188              :     return NULL;
    6189              : 
    6190     66145949 :   if (e->dest == dest)
    6191              :     return NULL;
    6192              : 
    6193     66119663 :   if (e->flags & EDGE_EH)
    6194      1010612 :     return redirect_eh_edge (e, dest);
    6195              : 
    6196     65109051 :   if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun))
    6197              :     {
    6198     64850054 :       ret = gimple_try_redirect_by_replacing_jump (e, dest);
    6199     64850054 :       if (ret)
    6200              :         return ret;
    6201              :     }
    6202              : 
    6203     64679853 :   gsi = gsi_last_nondebug_bb (bb);
    6204     64679853 :   stmt = gsi_end_p (gsi) ? NULL : gsi_stmt (gsi);
    6205              : 
    6206     64679853 :   switch (stmt ? gimple_code (stmt) : GIMPLE_ERROR_MARK)
    6207              :     {
    6208              :     case GIMPLE_COND:
    6209              :       /* For COND_EXPR, we only need to redirect the edge.  */
    6210              :       break;
    6211              : 
    6212            0 :     case GIMPLE_GOTO:
    6213              :       /* No non-abnormal edges should lead from a non-simple goto, and
    6214              :          simple ones should be represented implicitly.  */
    6215            0 :       gcc_unreachable ();
    6216              : 
    6217       407324 :     case GIMPLE_SWITCH:
    6218       407324 :       {
    6219       407324 :         gswitch *switch_stmt = as_a <gswitch *> (stmt);
    6220       407324 :         tree label = gimple_block_label (dest);
    6221       407324 :         tree cases = get_cases_for_edge (e, switch_stmt);
    6222              : 
    6223              :         /* If we have a list of cases associated with E, then use it
    6224              :            as it's a lot faster than walking the entire case vector.  */
    6225       407324 :         if (cases)
    6226              :           {
    6227       348946 :             edge e2 = find_edge (e->src, dest);
    6228       348946 :             tree last, first;
    6229              : 
    6230       348946 :             first = cases;
    6231      1113848 :             while (cases)
    6232              :               {
    6233       415956 :                 last = cases;
    6234       415956 :                 CASE_LABEL (cases) = label;
    6235       415956 :                 cases = CASE_CHAIN (cases);
    6236              :               }
    6237              : 
    6238              :             /* If there was already an edge in the CFG, then we need
    6239              :                to move all the cases associated with E to E2.  */
    6240       348946 :             if (e2)
    6241              :               {
    6242         8963 :                 tree cases2 = get_cases_for_edge (e2, switch_stmt);
    6243              : 
    6244         8963 :                 CASE_CHAIN (last) = CASE_CHAIN (cases2);
    6245         8963 :                 CASE_CHAIN (cases2) = first;
    6246              :               }
    6247       348946 :             bitmap_set_bit (touched_switch_bbs, gimple_bb (stmt)->index);
    6248              :           }
    6249              :         else
    6250              :           {
    6251        58378 :             size_t i, n = gimple_switch_num_labels (switch_stmt);
    6252              : 
    6253      4785920 :             for (i = 0; i < n; i++)
    6254              :               {
    6255      4727542 :                 tree elt = gimple_switch_label (switch_stmt, i);
    6256      4727542 :                 if (label_to_block (cfun, CASE_LABEL (elt)) == e->dest)
    6257        66307 :                   CASE_LABEL (elt) = label;
    6258              :               }
    6259              :           }
    6260              :       }
    6261              :       break;
    6262              : 
    6263        10653 :     case GIMPLE_ASM:
    6264        10653 :       {
    6265        10653 :         gasm *asm_stmt = as_a <gasm *> (stmt);
    6266        10653 :         int i, n = gimple_asm_nlabels (asm_stmt);
    6267        10653 :         tree label = NULL;
    6268              : 
    6269        15396 :         for (i = 0; i < n; ++i)
    6270              :           {
    6271         4743 :             tree cons = gimple_asm_label_op (asm_stmt, i);
    6272         4743 :             if (label_to_block (cfun, TREE_VALUE (cons)) == e->dest)
    6273              :               {
    6274         2344 :                 if (!label)
    6275         2344 :                   label = gimple_block_label (dest);
    6276         2344 :                 TREE_VALUE (cons) = label;
    6277              :               }
    6278              :           }
    6279              : 
    6280              :         /* If we didn't find any label matching the former edge in the
    6281              :            asm labels, we must be redirecting the fallthrough
    6282              :            edge.  */
    6283        10653 :         gcc_assert (label || (e->flags & EDGE_FALLTHRU));
    6284              :       }
    6285              :       break;
    6286              : 
    6287          255 :     case GIMPLE_RETURN:
    6288          255 :       gsi_remove (&gsi, true);
    6289          255 :       e->flags |= EDGE_FALLTHRU;
    6290          255 :       break;
    6291              : 
    6292              :     case GIMPLE_OMP_RETURN:
    6293              :     case GIMPLE_OMP_CONTINUE:
    6294              :     case GIMPLE_OMP_SECTIONS_SWITCH:
    6295              :     case GIMPLE_OMP_FOR:
    6296              :       /* The edges from OMP constructs can be simply redirected.  */
    6297              :       break;
    6298              : 
    6299            0 :     case GIMPLE_EH_DISPATCH:
    6300            0 :       if (!(e->flags & EDGE_FALLTHRU))
    6301            0 :         redirect_eh_dispatch_edge (as_a <geh_dispatch *> (stmt), e, dest);
    6302              :       break;
    6303              : 
    6304          346 :     case GIMPLE_TRANSACTION:
    6305          346 :       if (e->flags & EDGE_TM_ABORT)
    6306           81 :         gimple_transaction_set_label_over (as_a <gtransaction *> (stmt),
    6307              :                                            gimple_block_label (dest));
    6308          265 :       else if (e->flags & EDGE_TM_UNINSTRUMENTED)
    6309          128 :         gimple_transaction_set_label_uninst (as_a <gtransaction *> (stmt),
    6310              :                                              gimple_block_label (dest));
    6311              :       else
    6312          137 :         gimple_transaction_set_label_norm (as_a <gtransaction *> (stmt),
    6313              :                                            gimple_block_label (dest));
    6314              :       break;
    6315              : 
    6316      7798353 :     default:
    6317              :       /* Otherwise it must be a fallthru edge, and we don't need to
    6318              :          do anything besides redirecting it.  */
    6319      7798353 :       gcc_assert (e->flags & EDGE_FALLTHRU);
    6320              :       break;
    6321              :     }
    6322              : 
    6323              :   /* Update/insert PHI nodes as necessary.  */
    6324              : 
    6325              :   /* Now update the edges in the CFG.  */
    6326     64679853 :   e = ssa_redirect_edge (e, dest);
    6327              : 
    6328     64679853 :   return e;
    6329              : }
    6330              : 
    6331              : /* Returns true if it is possible to remove edge E by redirecting
    6332              :    it to the destination of the other edge from E->src.  */
    6333              : 
    6334              : static bool
    6335       312718 : gimple_can_remove_branch_p (const_edge e)
    6336              : {
    6337       312718 :   if (e->flags & (EDGE_ABNORMAL | EDGE_EH))
    6338            0 :     return false;
    6339              : 
    6340              :   return true;
    6341              : }
    6342              : 
    6343              : /* Simple wrapper, as we can always redirect fallthru edges.  */
    6344              : 
    6345              : static basic_block
    6346      4036914 : gimple_redirect_edge_and_branch_force (edge e, basic_block dest)
    6347              : {
    6348      4036914 :   e = gimple_redirect_edge_and_branch (e, dest);
    6349      4036914 :   gcc_assert (e);
    6350              : 
    6351      4036914 :   return NULL;
    6352              : }
    6353              : 
    6354              : 
    6355              : /* Splits basic block BB after statement STMT (but at least after the
    6356              :    labels).  If STMT is NULL, BB is split just after the labels.  */
    6357              : 
    6358              : static basic_block
    6359      5902044 : gimple_split_block (basic_block bb, void *stmt)
    6360              : {
    6361      5902044 :   gimple_stmt_iterator gsi;
    6362      5902044 :   gimple_stmt_iterator gsi_tgt;
    6363      5902044 :   gimple_seq list;
    6364      5902044 :   basic_block new_bb;
    6365      5902044 :   edge e;
    6366      5902044 :   edge_iterator ei;
    6367              : 
    6368      5902044 :   new_bb = create_empty_bb (bb);
    6369      5902044 :   if (coverage_suppressed_p (bb))
    6370          161 :     suppress_coverage (new_bb);
    6371              : 
    6372              :   /* Redirect the outgoing edges.  */
    6373      5902044 :   new_bb->succs = bb->succs;
    6374      5902044 :   bb->succs = NULL;
    6375     13623893 :   FOR_EACH_EDGE (e, ei, new_bb->succs)
    6376      7721849 :     e->src = new_bb;
    6377              : 
    6378              :   /* Get a stmt iterator pointing to the first stmt to move.  */
    6379      5902044 :   if (!stmt || gimple_code ((gimple *) stmt) == GIMPLE_LABEL)
    6380      1461802 :     gsi = gsi_after_labels (bb);
    6381              :   else
    6382              :     {
    6383      4440242 :       gsi = gsi_for_stmt ((gimple *) stmt);
    6384      4440242 :       gsi_next (&gsi);
    6385              :     }
    6386              : 
    6387              :   /* Move everything from GSI to the new basic block.  */
    6388      5902044 :   if (gsi_end_p (gsi))
    6389              :     return new_bb;
    6390              : 
    6391              :   /* Split the statement list - avoid re-creating new containers as this
    6392              :      brings ugly quadratic memory consumption in the inliner.
    6393              :      (We are still quadratic since we need to update stmt BB pointers,
    6394              :      sadly.)  */
    6395      5479114 :   gsi_split_seq_before (&gsi, &list);
    6396      5479114 :   set_bb_seq (new_bb, list);
    6397      5479114 :   for (gsi_tgt = gsi_start (list);
    6398     31283025 :        !gsi_end_p (gsi_tgt); gsi_next (&gsi_tgt))
    6399     25803911 :     gimple_set_bb (gsi_stmt (gsi_tgt), new_bb);
    6400              : 
    6401              :   return new_bb;
    6402              : }
    6403              : 
    6404              : 
    6405              : /* Moves basic block BB after block AFTER.  */
    6406              : 
    6407              : static bool
    6408     24068893 : gimple_move_block_after (basic_block bb, basic_block after)
    6409              : {
    6410     24068893 :   if (bb->prev_bb == after)
    6411              :     return true;
    6412              : 
    6413      5780347 :   unlink_block (bb);
    6414      5780347 :   link_block (bb, after);
    6415              : 
    6416      5780347 :   return true;
    6417              : }
    6418              : 
    6419              : 
    6420              : /* Return TRUE if block BB has no executable statements, otherwise return
    6421              :    FALSE.  */
    6422              : 
    6423              : static bool
    6424     15429839 : gimple_empty_block_p (basic_block bb)
    6425              : {
    6426              :   /* BB must have no executable statements.  */
    6427     15429839 :   gimple_stmt_iterator gsi = gsi_after_labels (bb);
    6428     15429839 :   if (phi_nodes (bb))
    6429              :     return false;
    6430     21652941 :   while (!gsi_end_p (gsi))
    6431              :     {
    6432     11562368 :       gimple *stmt = gsi_stmt (gsi);
    6433     11562368 :       if (is_gimple_debug (stmt))
    6434              :         ;
    6435      4184814 :       else if (gimple_code (stmt) == GIMPLE_NOP
    6436      4184814 :                || gimple_code (stmt) == GIMPLE_PREDICT)
    6437              :         ;
    6438              :       else
    6439              :         return false;
    6440      7417346 :       gsi_next (&gsi);
    6441              :     }
    6442              :   return true;
    6443              : }
    6444              : 
    6445              : 
    6446              : /* Split a basic block if it ends with a conditional branch and if the
    6447              :    other part of the block is not empty.  */
    6448              : 
    6449              : static basic_block
    6450          557 : gimple_split_block_before_cond_jump (basic_block bb)
    6451              : {
    6452          557 :   gimple *last, *split_point;
    6453          557 :   gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
    6454          557 :   if (gsi_end_p (gsi))
    6455              :     return NULL;
    6456          557 :   last = gsi_stmt (gsi);
    6457          557 :   if (gimple_code (last) != GIMPLE_COND
    6458          557 :       && gimple_code (last) != GIMPLE_SWITCH)
    6459              :     return NULL;
    6460          557 :   gsi_prev (&gsi);
    6461          557 :   split_point = gsi_stmt (gsi);
    6462          557 :   return split_block (bb, split_point)->dest;
    6463              : }
    6464              : 
    6465              : 
    6466              : /* Return true if basic_block can be duplicated.  */
    6467              : 
    6468              : static bool
    6469     16262202 : gimple_can_duplicate_bb_p (const_basic_block bb)
    6470              : {
    6471     16262202 :   gimple *last = last_nondebug_stmt (const_cast<basic_block> (bb));
    6472              : 
    6473              :   /* Do checks that can only fail for the last stmt, to minimize the work in the
    6474              :      stmt loop.  */
    6475     16262202 :   if (last) {
    6476              :     /* A transaction is a single entry multiple exit region.  It
    6477              :        must be duplicated in its entirety or not at all.  */
    6478     14413564 :     if (gimple_code (last) == GIMPLE_TRANSACTION)
    6479              :       return false;
    6480              : 
    6481              :     /* An IFN_UNIQUE call must be duplicated as part of its group,
    6482              :        or not at all.  */
    6483     14413562 :     if (is_gimple_call (last)
    6484        75066 :         && gimple_call_internal_p (last)
    6485     14414035 :         && gimple_call_internal_unique_p (last))
    6486              :       return false;
    6487              : 
    6488              :     /* Prohibit duplication of returns_twice calls, otherwise associated
    6489              :        abnormal edges also need to be duplicated properly.
    6490              :        return_twice functions will always be the last statement.  */
    6491     14413562 :     if (is_gimple_call (last)
    6492     14413562 :         && (gimple_call_flags (last) & ECF_RETURNS_TWICE))
    6493              :       return false;
    6494              :   }
    6495              : 
    6496     32523988 :   for (gimple_stmt_iterator gsi = gsi_start_bb (const_cast<basic_block> (bb));
    6497    159719621 :        !gsi_end_p (gsi); gsi_next (&gsi))
    6498              :     {
    6499    143457627 :       gimple *g = gsi_stmt (gsi);
    6500              : 
    6501              :       /* An IFN_GOMP_SIMT_ENTER_ALLOC/IFN_GOMP_SIMT_EXIT call must be
    6502              :          duplicated as part of its group, or not at all.
    6503              :          The IFN_GOMP_SIMT_VOTE_ANY and IFN_GOMP_SIMT_XCHG_* are part of such a
    6504              :          group, so the same holds there.  */
    6505    143457627 :       if (is_gimple_call (g)
    6506    143457627 :           && (gimple_call_internal_p (g, IFN_GOMP_SIMT_ENTER_ALLOC)
    6507      2200172 :               || gimple_call_internal_p (g, IFN_GOMP_SIMT_EXIT)
    6508      2200172 :               || gimple_call_internal_p (g, IFN_GOMP_SIMT_VOTE_ANY)
    6509      2200172 :               || gimple_call_internal_p (g, IFN_GOMP_SIMT_XCHG_BFLY)
    6510      2200172 :               || gimple_call_internal_p (g, IFN_GOMP_SIMT_XCHG_IDX)))
    6511     16262202 :         return false;
    6512              :     }
    6513              : 
    6514              :   return true;
    6515              : }
    6516              : 
    6517              : /* Create a duplicate of the basic block BB.  NOTE: This does not
    6518              :    preserve SSA form.  */
    6519              : 
    6520              : static basic_block
    6521      4071607 : gimple_duplicate_bb (basic_block bb, copy_bb_data *id)
    6522              : {
    6523      4071607 :   basic_block new_bb;
    6524      4071607 :   gimple_stmt_iterator gsi_tgt;
    6525              : 
    6526      4071607 :   new_bb = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
    6527      4071607 :   if (coverage_suppressed_p (bb))
    6528            0 :     suppress_coverage (new_bb);
    6529              : 
    6530              :   /* Copy the PHI nodes.  We ignore PHI node arguments here because
    6531              :      the incoming edges have not been setup yet.  */
    6532      4071607 :   for (gphi_iterator gpi = gsi_start_phis (bb);
    6533      9597855 :        !gsi_end_p (gpi);
    6534      5526248 :        gsi_next (&gpi))
    6535              :     {
    6536      5526248 :       gphi *phi, *copy;
    6537      5526248 :       phi = gpi.phi ();
    6538      5526248 :       copy = create_phi_node (NULL_TREE, new_bb);
    6539      5526248 :       create_new_def_for (gimple_phi_result (phi), copy,
    6540              :                           gimple_phi_result_ptr (copy));
    6541      5526248 :       gimple_set_uid (copy, gimple_uid (phi));
    6542              :     }
    6543              : 
    6544      4071607 :   gsi_tgt = gsi_start_bb (new_bb);
    6545      8143214 :   for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    6546     28871105 :        !gsi_end_p (gsi);
    6547     24799498 :        gsi_next (&gsi))
    6548              :     {
    6549     24799498 :       def_operand_p def_p;
    6550     24799498 :       ssa_op_iter op_iter;
    6551     24799498 :       tree lhs;
    6552     24799498 :       gimple *stmt, *copy;
    6553              : 
    6554     24799498 :       stmt = gsi_stmt (gsi);
    6555     24799498 :       if (gimple_code (stmt) == GIMPLE_LABEL)
    6556        94037 :         continue;
    6557              : 
    6558              :       /* Don't duplicate label debug stmts.  */
    6559     24707623 :       if (gimple_debug_bind_p (stmt)
    6560     15703358 :           && TREE_CODE (gimple_debug_bind_get_var (stmt))
    6561              :              == LABEL_DECL)
    6562         2162 :         continue;
    6563              : 
    6564              :       /* Create a new copy of STMT and duplicate STMT's virtual
    6565              :          operands.  */
    6566     24705461 :       copy = gimple_copy (stmt);
    6567     24705461 :       gsi_insert_after (&gsi_tgt, copy, GSI_NEW_STMT);
    6568              : 
    6569     24705461 :       maybe_duplicate_eh_stmt (copy, stmt);
    6570     24705461 :       gimple_duplicate_stmt_histograms (cfun, copy, cfun, stmt);
    6571              : 
    6572              :       /* When copying around a stmt writing into a local non-user
    6573              :          aggregate, make sure it won't share stack slot with other
    6574              :          vars.  */
    6575     24705461 :       lhs = gimple_get_lhs (stmt);
    6576     24705461 :       if (lhs && TREE_CODE (lhs) != SSA_NAME)
    6577              :         {
    6578       878622 :           tree base = get_base_address (lhs);
    6579       878622 :           if (base
    6580       878622 :               && (VAR_P (base) || TREE_CODE (base) == RESULT_DECL)
    6581       566033 :               && DECL_IGNORED_P (base)
    6582       213320 :               && !TREE_STATIC (base)
    6583       211982 :               && !DECL_EXTERNAL (base)
    6584      1090602 :               && (!VAR_P (base) || !DECL_HAS_VALUE_EXPR_P (base)))
    6585       211980 :             DECL_NONSHAREABLE (base) = 1;
    6586              :         }
    6587              : 
    6588              :       /* If requested remap dependence info of cliques brought in
    6589              :          via inlining.  */
    6590     24705461 :       if (id)
    6591     66305524 :         for (unsigned i = 0; i < gimple_num_ops (copy); ++i)
    6592              :           {
    6593     45761176 :             tree op = gimple_op (copy, i);
    6594     45761176 :             if (!op)
    6595     11854865 :               continue;
    6596     33906311 :             if (TREE_CODE (op) == ADDR_EXPR
    6597     32560315 :                 || TREE_CODE (op) == WITH_SIZE_EXPR)
    6598      1345996 :               op = TREE_OPERAND (op, 0);
    6599     36037532 :             while (handled_component_p (op))
    6600      2131221 :               op = TREE_OPERAND (op, 0);
    6601     33906311 :             if ((TREE_CODE (op) == MEM_REF
    6602     33906311 :                  || TREE_CODE (op) == TARGET_MEM_REF)
    6603      1170241 :                 && MR_DEPENDENCE_CLIQUE (op) > 1
    6604     33983231 :                 && MR_DEPENDENCE_CLIQUE (op) != bb->loop_father->owned_clique)
    6605              :               {
    6606        27869 :                 if (!id->dependence_map)
    6607        14060 :                   id->dependence_map = new hash_map<dependence_hash,
    6608              :                                                     unsigned short>;
    6609        27869 :                 bool existed;
    6610        27869 :                 unsigned short &newc = id->dependence_map->get_or_insert
    6611        27869 :                     (MR_DEPENDENCE_CLIQUE (op), &existed);
    6612        27869 :                 if (!existed)
    6613              :                   {
    6614        18689 :                     gcc_assert (MR_DEPENDENCE_CLIQUE (op) <= cfun->last_clique);
    6615        37378 :                     newc = get_new_clique (cfun);
    6616              :                   }
    6617        27869 :                 MR_DEPENDENCE_CLIQUE (op) = newc;
    6618              :               }
    6619              :           }
    6620              : 
    6621              :       /* Create new names for all the definitions created by COPY and
    6622              :          add replacement mappings for each new name.  */
    6623     30453932 :       FOR_EACH_SSA_DEF_OPERAND (def_p, copy, op_iter, SSA_OP_ALL_DEFS)
    6624      5748471 :         create_new_def_for (DEF_FROM_PTR (def_p), copy, def_p);
    6625              :     }
    6626              : 
    6627      4071607 :   return new_bb;
    6628              : }
    6629              : 
    6630              : /* Adds phi node arguments for edge E_COPY after basic block duplication.  */
    6631              : 
    6632              : static void
    6633      5498005 : add_phi_args_after_copy_edge (edge e_copy)
    6634              : {
    6635      5498005 :   basic_block bb, bb_copy = e_copy->src, dest;
    6636      5498005 :   edge e;
    6637      5498005 :   edge_iterator ei;
    6638      5498005 :   gphi *phi, *phi_copy;
    6639      5498005 :   tree def;
    6640      5498005 :   gphi_iterator psi, psi_copy;
    6641              : 
    6642      5498005 :   if (gimple_seq_empty_p (phi_nodes (e_copy->dest)))
    6643      3513445 :     return;
    6644              : 
    6645      1984560 :   bb = bb_copy->flags & BB_DUPLICATED ? get_bb_original (bb_copy) : bb_copy;
    6646              : 
    6647      1984560 :   if (e_copy->dest->flags & BB_DUPLICATED)
    6648       810017 :     dest = get_bb_original (e_copy->dest);
    6649              :   else
    6650              :     dest = e_copy->dest;
    6651              : 
    6652      1984560 :   e = find_edge (bb, dest);
    6653      1984560 :   if (!e)
    6654              :     {
    6655              :       /* During loop unrolling the target of the latch edge is copied.
    6656              :          In this case we are not looking for edge to dest, but to
    6657              :          duplicated block whose original was dest.  */
    6658         1161 :       FOR_EACH_EDGE (e, ei, bb->succs)
    6659              :         {
    6660         1161 :           if ((e->dest->flags & BB_DUPLICATED)
    6661         1161 :               && get_bb_original (e->dest) == dest)
    6662              :             break;
    6663              :         }
    6664              : 
    6665         1161 :       gcc_assert (e != NULL);
    6666              :     }
    6667              : 
    6668      1984560 :   for (psi = gsi_start_phis (e->dest),
    6669      1984560 :        psi_copy = gsi_start_phis (e_copy->dest);
    6670      5873305 :        !gsi_end_p (psi);
    6671      3888745 :        gsi_next (&psi), gsi_next (&psi_copy))
    6672              :     {
    6673      3888745 :       phi = psi.phi ();
    6674      3888745 :       phi_copy = psi_copy.phi ();
    6675      3888745 :       def = PHI_ARG_DEF_FROM_EDGE (phi, e);
    6676      3888745 :       add_phi_arg (phi_copy, def, e_copy,
    6677              :                    gimple_phi_arg_location_from_edge (phi, e));
    6678              :     }
    6679              : }
    6680              : 
    6681              : 
    6682              : /* Basic block BB_COPY was created by code duplication.  Add phi node
    6683              :    arguments for edges going out of BB_COPY.  The blocks that were
    6684              :    duplicated have BB_DUPLICATED set.  */
    6685              : 
    6686              : void
    6687      3795247 : add_phi_args_after_copy_bb (basic_block bb_copy)
    6688              : {
    6689      3795247 :   edge e_copy;
    6690      3795247 :   edge_iterator ei;
    6691              : 
    6692      9293230 :   FOR_EACH_EDGE (e_copy, ei, bb_copy->succs)
    6693              :     {
    6694      5497983 :       add_phi_args_after_copy_edge (e_copy);
    6695              :     }
    6696      3795247 : }
    6697              : 
    6698              : /* Blocks in REGION_COPY array of length N_REGION were created by
    6699              :    duplication of basic blocks.  Add phi node arguments for edges
    6700              :    going from these blocks.  If E_COPY is not NULL, also add
    6701              :    phi node arguments for its destination.*/
    6702              : 
    6703              : void
    6704      1998952 : add_phi_args_after_copy (basic_block *region_copy, unsigned n_region,
    6705              :                          edge e_copy)
    6706              : {
    6707      1998952 :   unsigned i;
    6708              : 
    6709      4565886 :   for (i = 0; i < n_region; i++)
    6710      2566934 :     region_copy[i]->flags |= BB_DUPLICATED;
    6711              : 
    6712      4565886 :   for (i = 0; i < n_region; i++)
    6713      2566934 :     add_phi_args_after_copy_bb (region_copy[i]);
    6714      1998952 :   if (e_copy)
    6715           22 :     add_phi_args_after_copy_edge (e_copy);
    6716              : 
    6717      4565886 :   for (i = 0; i < n_region; i++)
    6718      2566934 :     region_copy[i]->flags &= ~BB_DUPLICATED;
    6719      1998952 : }
    6720              : 
    6721              : /* Duplicates a REGION (set of N_REGION basic blocks) with just a single
    6722              :    important exit edge EXIT.  By important we mean that no SSA name defined
    6723              :    inside region is live over the other exit edges of the region.  All entry
    6724              :    edges to the region must go to ENTRY->dest.  The edge ENTRY is redirected
    6725              :    to the duplicate of the region.  Dominance and loop information is
    6726              :    updated if UPDATE_DOMINANCE is true, but not the SSA web.  If
    6727              :    UPDATE_DOMINANCE is false then we assume that the caller will update the
    6728              :    dominance information after calling this function.  The new basic
    6729              :    blocks are stored to REGION_COPY in the same order as they had in REGION,
    6730              :    provided that REGION_COPY is not NULL.
    6731              :    The function returns false if it is unable to copy the region,
    6732              :    true otherwise.
    6733              : 
    6734              :    It is callers responsibility to update profile.  */
    6735              : 
    6736              : bool
    6737       584218 : gimple_duplicate_seme_region (edge entry, edge exit,
    6738              :                               basic_block *region, unsigned n_region,
    6739              :                               basic_block *region_copy,
    6740              :                               bool update_dominance)
    6741              : {
    6742       584218 :   unsigned i;
    6743       584218 :   bool free_region_copy = false, copying_header = false;
    6744       584218 :   class loop *loop = entry->dest->loop_father;
    6745       584218 :   edge exit_copy;
    6746       584218 :   edge redirected;
    6747              : 
    6748       584218 :   if (!can_copy_bbs_p (region, n_region))
    6749              :     return false;
    6750              : 
    6751              :   /* Some sanity checking.  Note that we do not check for all possible
    6752              :      missuses of the functions.  I.e. if you ask to copy something weird,
    6753              :      it will work, but the state of structures probably will not be
    6754              :      correct.  */
    6755      1178152 :   for (i = 0; i < n_region; i++)
    6756              :     {
    6757              :       /* We do not handle subloops, i.e. all the blocks must belong to the
    6758              :          same loop.  */
    6759       593934 :       if (region[i]->loop_father != loop)
    6760              :         return false;
    6761              : 
    6762       593934 :       if (region[i] != entry->dest
    6763         9716 :           && region[i] == loop->header)
    6764              :         return false;
    6765              :     }
    6766              : 
    6767              :   /* In case the function is used for loop header copying (which is the primary
    6768              :      use), ensure that EXIT and its copy will be new latch and entry edges.  */
    6769       584218 :   if (loop->header == entry->dest)
    6770              :     {
    6771       584218 :       copying_header = true;
    6772              : 
    6773       584218 :       if (!dominated_by_p (CDI_DOMINATORS, loop->latch, exit->src))
    6774              :         return false;
    6775              : 
    6776      1178152 :       for (i = 0; i < n_region; i++)
    6777       593934 :         if (region[i] != exit->src
    6778       593934 :             && dominated_by_p (CDI_DOMINATORS, region[i], exit->src))
    6779              :           return false;
    6780              :     }
    6781              : 
    6782       584218 :   initialize_original_copy_tables ();
    6783              : 
    6784       584218 :   if (copying_header)
    6785       584218 :     set_loop_copy (loop, loop_outer (loop));
    6786              :   else
    6787            0 :     set_loop_copy (loop, loop);
    6788              : 
    6789       584218 :   if (!region_copy)
    6790              :     {
    6791            0 :       region_copy = XNEWVEC (basic_block, n_region);
    6792            0 :       free_region_copy = true;
    6793              :     }
    6794              : 
    6795              :   /* Record blocks outside the region that are dominated by something
    6796              :      inside.  */
    6797       584218 :   auto_vec<basic_block> doms;
    6798       584218 :   if (update_dominance)
    6799       584218 :     doms = get_dominated_by_region (CDI_DOMINATORS, region, n_region);
    6800              : 
    6801       584218 :   copy_bbs (region, n_region, region_copy, &exit, 1, &exit_copy, loop,
    6802              :             split_edge_bb_loc (entry), update_dominance);
    6803              : 
    6804       584218 :   if (copying_header)
    6805              :     {
    6806       584218 :       loop->header = exit->dest;
    6807       584218 :       loop->latch = exit->src;
    6808              :     }
    6809              : 
    6810              :   /* Redirect the entry and add the phi node arguments.  */
    6811       584218 :   redirected = redirect_edge_and_branch (entry, get_bb_copy (entry->dest));
    6812       584218 :   gcc_assert (redirected != NULL);
    6813       584218 :   flush_pending_stmts (entry);
    6814              : 
    6815              :   /* Concerning updating of dominators:  We must recount dominators
    6816              :      for entry block and its copy.  Anything that is outside of the
    6817              :      region, but was dominated by something inside needs recounting as
    6818              :      well.  */
    6819       584218 :   if (update_dominance)
    6820              :     {
    6821       584218 :       set_immediate_dominator (CDI_DOMINATORS, entry->dest, entry->src);
    6822       584218 :       doms.safe_push (get_bb_original (entry->dest));
    6823       584218 :       iterate_fix_dominators (CDI_DOMINATORS, doms, false);
    6824              :     }
    6825              : 
    6826              :   /* Add the other PHI node arguments.  */
    6827       584218 :   add_phi_args_after_copy (region_copy, n_region, NULL);
    6828              : 
    6829       584218 :   if (free_region_copy)
    6830            0 :     free (region_copy);
    6831              : 
    6832       584218 :   free_original_copy_tables ();
    6833       584218 :   return true;
    6834       584218 : }
    6835              : 
    6836              : /* Checks if BB is part of the region defined by N_REGION BBS.  */
    6837              : static bool
    6838            0 : bb_part_of_region_p (basic_block bb, basic_block* bbs, unsigned n_region)
    6839              : {
    6840            0 :   unsigned int n;
    6841              : 
    6842            0 :   for (n = 0; n < n_region; n++)
    6843              :     {
    6844            0 :      if (bb == bbs[n])
    6845              :        return true;
    6846              :     }
    6847              :   return false;
    6848              : }
    6849              : 
    6850              : 
    6851              : /* For each PHI in BB, copy the argument associated with SRC_E to TGT_E.
    6852              :    Assuming the argument exists, just does not have a value.
    6853              :    If USE_MAP is true, then use the redirect edge var map of TGT_E
    6854              :    for the new arguments; clearing the map afterwards.  */
    6855              : 
    6856              : void
    6857     31687805 : copy_phi_arg_into_existing_phi (edge src_e, edge tgt_e, bool use_map)
    6858              : {
    6859     31687805 :   int src_idx = src_e->dest_idx;
    6860     31687805 :   int tgt_idx = tgt_e->dest_idx;
    6861              : 
    6862              :   /* Iterate over each PHI in e->dest.  */
    6863     31687805 :   for (gphi_iterator gsi = gsi_start_phis (src_e->dest),
    6864     31687805 :                            gsi2 = gsi_start_phis (tgt_e->dest);
    6865     80585361 :        !gsi_end_p (gsi);
    6866     48897556 :        gsi_next (&gsi), gsi_next (&gsi2))
    6867              :     {
    6868     48897556 :       gphi *src_phi = gsi.phi ();
    6869     48897556 :       gphi *dest_phi = gsi2.phi ();
    6870     48897556 :       tree val = gimple_phi_arg_def (src_phi, src_idx);
    6871     48897556 :       location_t locus = gimple_phi_arg_location (src_phi, src_idx);
    6872              : 
    6873     48897556 :       if (use_map && TREE_CODE (val) == SSA_NAME)
    6874              :         {
    6875              :           /* If DEF is one of the results of PHI nodes removed during
    6876              :              redirection, replace it with the PHI argument that used
    6877              :              to be on E.  */
    6878      4844501 :           vec<edge_var_map> *head = redirect_edge_var_map_vector (tgt_e);
    6879      9689002 :           size_t length = head ? head->length () : 0;
    6880      8395409 :           for (size_t i = 0; i < length; i++)
    6881              :             {
    6882      6998800 :               edge_var_map *vm = &(*head)[i];
    6883      6998800 :               tree old_arg = redirect_edge_var_map_result (vm);
    6884      6998800 :               tree new_arg = redirect_edge_var_map_def (vm);
    6885              : 
    6886      6998800 :               if (val == old_arg)
    6887              :                 {
    6888      3447892 :                   val = new_arg;
    6889      3447892 :                   location_t locus1 = redirect_edge_var_map_location (vm);
    6890              :                   /* Don't remove the location if we remap one does not have one.  */
    6891      3447892 :                   if (locus1 != UNKNOWN_LOCATION)
    6892       531607 :                     locus = locus1;
    6893              :                   break;
    6894              :                 }
    6895              :             }
    6896              :         }
    6897              : 
    6898     48897556 :       SET_PHI_ARG_DEF (dest_phi, tgt_idx, val);
    6899     48897556 :       gimple_phi_arg_set_location (dest_phi, tgt_idx, locus);
    6900              :     }
    6901     31687805 :   if (use_map)
    6902      2874329 :     redirect_edge_var_map_clear (tgt_e);
    6903     31687805 : }
    6904              : 
    6905              : /* Duplicates REGION consisting of N_REGION blocks.  The new blocks
    6906              :    are stored to REGION_COPY in the same order in that they appear
    6907              :    in REGION, if REGION_COPY is not NULL.  ENTRY is the entry to
    6908              :    the region, EXIT an exit from it.  The condition guarding EXIT
    6909              :    is moved to ENTRY.  Returns true if duplication succeeds, false
    6910              :    otherwise.
    6911              : 
    6912              :    For example,
    6913              : 
    6914              :    some_code;
    6915              :    if (cond)
    6916              :      A;
    6917              :    else
    6918              :      B;
    6919              : 
    6920              :    is transformed to
    6921              : 
    6922              :    if (cond)
    6923              :      {
    6924              :        some_code;
    6925              :        A;
    6926              :      }
    6927              :    else
    6928              :      {
    6929              :        some_code;
    6930              :        B;
    6931              :      }
    6932              : */
    6933              : 
    6934              : bool
    6935           22 : gimple_duplicate_sese_tail (edge entry, edge exit,
    6936              :                           basic_block *region, unsigned n_region,
    6937              :                           basic_block *region_copy)
    6938              : {
    6939           22 :   unsigned i;
    6940           22 :   bool free_region_copy = false;
    6941           22 :   class loop *loop = exit->dest->loop_father;
    6942           22 :   class loop *orig_loop = entry->dest->loop_father;
    6943           22 :   basic_block switch_bb, entry_bb, nentry_bb;
    6944           22 :   profile_count total_count = profile_count::uninitialized (),
    6945           22 :                 exit_count = profile_count::uninitialized ();
    6946           22 :   edge exits[2], nexits[2], e;
    6947           22 :   gimple_stmt_iterator gsi;
    6948           22 :   edge sorig, snew;
    6949           22 :   basic_block exit_bb;
    6950           22 :   class loop *target, *aloop, *cloop;
    6951              : 
    6952           22 :   gcc_assert (EDGE_COUNT (exit->src->succs) == 2);
    6953           22 :   exits[0] = exit;
    6954           22 :   exits[1] = EDGE_SUCC (exit->src, EDGE_SUCC (exit->src, 0) == exit);
    6955              : 
    6956           22 :   if (!can_copy_bbs_p (region, n_region))
    6957              :     return false;
    6958              : 
    6959           22 :   initialize_original_copy_tables ();
    6960           22 :   set_loop_copy (orig_loop, loop);
    6961              : 
    6962           22 :   target= loop;
    6963           22 :   for (aloop = orig_loop->inner; aloop; aloop = aloop->next)
    6964              :     {
    6965            0 :       if (bb_part_of_region_p (aloop->header, region, n_region))
    6966              :         {
    6967            0 :           cloop = duplicate_loop (aloop, target);
    6968            0 :           duplicate_subloops (aloop, cloop);
    6969              :         }
    6970              :     }
    6971              : 
    6972           22 :   if (!region_copy)
    6973              :     {
    6974            0 :       region_copy = XNEWVEC (basic_block, n_region);
    6975            0 :       free_region_copy = true;
    6976              :     }
    6977              : 
    6978           22 :   gcc_assert (!need_ssa_update_p (cfun));
    6979              : 
    6980              :   /* Record blocks outside the region that are dominated by something
    6981              :      inside.  */
    6982           22 :   auto_vec<basic_block> doms = get_dominated_by_region (CDI_DOMINATORS, region,
    6983           22 :                                                         n_region);
    6984              : 
    6985           22 :   total_count = exit->src->count;
    6986           22 :   exit_count = exit->count ();
    6987              :   /* Fix up corner cases, to avoid division by zero or creation of negative
    6988              :      frequencies.  */
    6989           22 :   if (exit_count > total_count)
    6990            0 :     exit_count = total_count;
    6991              : 
    6992           22 :   copy_bbs (region, n_region, region_copy, exits, 2, nexits, orig_loop,
    6993              :             split_edge_bb_loc (exit), true);
    6994           22 :   if (total_count.initialized_p () && exit_count.initialized_p ())
    6995              :     {
    6996           22 :       scale_bbs_frequencies_profile_count (region, n_region,
    6997              :                                            total_count - exit_count,
    6998              :                                            total_count);
    6999           22 :       scale_bbs_frequencies_profile_count (region_copy, n_region, exit_count,
    7000              :                                            total_count);
    7001              :     }
    7002              : 
    7003              :   /* Create the switch block, and put the exit condition to it.  */
    7004           22 :   entry_bb = entry->dest;
    7005           22 :   nentry_bb = get_bb_copy (entry_bb);
    7006           22 :   if (!*gsi_last_bb (entry->src)
    7007           22 :       || !stmt_ends_bb_p (*gsi_last_bb (entry->src)))
    7008           22 :     switch_bb = entry->src;
    7009              :   else
    7010            0 :     switch_bb = split_edge (entry);
    7011           22 :   set_immediate_dominator (CDI_DOMINATORS, nentry_bb, switch_bb);
    7012              : 
    7013           44 :   gcond *cond_stmt = as_a <gcond *> (*gsi_last_bb (exit->src));
    7014           22 :   cond_stmt = as_a <gcond *> (gimple_copy (cond_stmt));
    7015              : 
    7016           22 :   gsi = gsi_last_bb (switch_bb);
    7017           22 :   gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT);
    7018              : 
    7019           22 :   sorig = single_succ_edge (switch_bb);
    7020           22 :   sorig->flags = exits[1]->flags;
    7021           22 :   sorig->probability = exits[1]->probability;
    7022           22 :   snew = make_edge (switch_bb, nentry_bb, exits[0]->flags);
    7023           22 :   snew->probability = exits[0]->probability;
    7024              : 
    7025              : 
    7026              :   /* Register the new edge from SWITCH_BB in loop exit lists.  */
    7027           22 :   rescan_loop_exit (snew, true, false);
    7028              : 
    7029              :   /* Add the PHI node arguments.  */
    7030           22 :   add_phi_args_after_copy (region_copy, n_region, snew);
    7031              : 
    7032              :   /* Get rid of now superfluous conditions and associated edges (and phi node
    7033              :      arguments).  */
    7034           22 :   exit_bb = exit->dest;
    7035              : 
    7036           22 :   e = redirect_edge_and_branch (exits[0], exits[1]->dest);
    7037           22 :   PENDING_STMT (e) = NULL;
    7038              : 
    7039              :   /* The latch of ORIG_LOOP was copied, and so was the backedge
    7040              :      to the original header.  We redirect this backedge to EXIT_BB.  */
    7041           46 :   for (i = 0; i < n_region; i++)
    7042           24 :     if (get_bb_original (region_copy[i]) == orig_loop->latch)
    7043              :       {
    7044            0 :         gcc_assert (single_succ_edge (region_copy[i]));
    7045            0 :         e = redirect_edge_and_branch (single_succ_edge (region_copy[i]), exit_bb);
    7046            0 :         PENDING_STMT (e) = NULL;
    7047            0 :         copy_phi_arg_into_existing_phi (nexits[0], e);
    7048              :       }
    7049           22 :   e = redirect_edge_and_branch (nexits[1], nexits[0]->dest);
    7050           22 :   PENDING_STMT (e) = NULL;
    7051              : 
    7052              :   /* Anything that is outside of the region, but was dominated by something
    7053              :      inside needs to update dominance info.  */
    7054           22 :   iterate_fix_dominators (CDI_DOMINATORS, doms, false);
    7055              : 
    7056           22 :   if (free_region_copy)
    7057            0 :     free (region_copy);
    7058              : 
    7059           22 :   free_original_copy_tables ();
    7060           22 :   return true;
    7061           22 : }
    7062              : 
    7063              : /* Add all the blocks dominated by ENTRY to the array BBS_P.  Stop
    7064              :    adding blocks when the dominator traversal reaches EXIT.  This
    7065              :    function silently assumes that ENTRY strictly dominates EXIT.  */
    7066              : 
    7067              : void
    7068       603396 : gather_blocks_in_sese_region (basic_block entry, basic_block exit,
    7069              :                               vec<basic_block> *bbs_p)
    7070              : {
    7071       603396 :   basic_block son;
    7072              : 
    7073       603396 :   for (son = first_dom_son (CDI_DOMINATORS, entry);
    7074      1206671 :        son;
    7075       603275 :        son = next_dom_son (CDI_DOMINATORS, son))
    7076              :     {
    7077       603275 :       bbs_p->safe_push (son);
    7078       603275 :       if (son != exit)
    7079       558846 :         gather_blocks_in_sese_region (son, exit, bbs_p);
    7080              :     }
    7081       603396 : }
    7082              : 
    7083              : /* Replaces *TP with a duplicate (belonging to function TO_CONTEXT).
    7084              :    The duplicates are recorded in VARS_MAP.  */
    7085              : 
    7086              : static void
    7087      4508343 : replace_by_duplicate_decl (tree *tp, hash_map<tree, tree> *vars_map,
    7088              :                            tree to_context)
    7089              : {
    7090      4508343 :   tree t = *tp, new_t;
    7091      4508343 :   struct function *f = DECL_STRUCT_FUNCTION (to_context);
    7092              : 
    7093      4508343 :   if (DECL_CONTEXT (t) == to_context)
    7094        35546 :     return;
    7095              : 
    7096      4472797 :   bool existed;
    7097      4472797 :   tree &loc = vars_map->get_or_insert (t, &existed);
    7098              : 
    7099      4472797 :   if (!existed)
    7100              :     {
    7101      1323055 :       if (SSA_VAR_P (t))
    7102              :         {
    7103      1312460 :           new_t = copy_var_decl (t, DECL_NAME (t), TREE_TYPE (t));
    7104      1312460 :           add_local_decl (f, new_t);
    7105              :         }
    7106              :       else
    7107              :         {
    7108        10595 :           gcc_assert (TREE_CODE (t) == CONST_DECL);
    7109        10595 :           new_t = copy_node (t);
    7110              :         }
    7111      1323055 :       DECL_CONTEXT (new_t) = to_context;
    7112              : 
    7113      1323055 :       loc = new_t;
    7114              :     }
    7115              :   else
    7116      3149742 :     new_t = loc;
    7117              : 
    7118      4472797 :   *tp = new_t;
    7119              : }
    7120              : 
    7121              : 
    7122              : /* Creates an ssa name in TO_CONTEXT equivalent to NAME.
    7123              :    VARS_MAP maps old ssa names and var_decls to the new ones.  */
    7124              : 
    7125              : static tree
    7126        14396 : replace_ssa_name (tree name, hash_map<tree, tree> *vars_map,
    7127              :                   tree to_context)
    7128              : {
    7129        14396 :   tree new_name;
    7130              : 
    7131        28792 :   gcc_assert (!virtual_operand_p (name));
    7132              : 
    7133        14396 :   tree *loc = vars_map->get (name);
    7134              : 
    7135        14396 :   if (!loc)
    7136              :     {
    7137         5953 :       tree decl = SSA_NAME_VAR (name);
    7138         5953 :       if (decl)
    7139              :         {
    7140         1186 :           gcc_assert (!SSA_NAME_IS_DEFAULT_DEF (name));
    7141         1186 :           replace_by_duplicate_decl (&decl, vars_map, to_context);
    7142         1186 :           new_name = make_ssa_name_fn (DECL_STRUCT_FUNCTION (to_context),
    7143         1186 :                                        decl, SSA_NAME_DEF_STMT (name));
    7144              :         }
    7145              :       else
    7146         4767 :         new_name = copy_ssa_name_fn (DECL_STRUCT_FUNCTION (to_context),
    7147         4767 :                                      name, SSA_NAME_DEF_STMT (name));
    7148              : 
    7149              :       /* Now that we've used the def stmt to define new_name, make sure it
    7150              :          doesn't define name anymore.  */
    7151         5953 :       SSA_NAME_DEF_STMT (name) = NULL;
    7152              : 
    7153         5953 :       vars_map->put (name, new_name);
    7154              :     }
    7155              :   else
    7156         8443 :     new_name = *loc;
    7157              : 
    7158        14396 :   return new_name;
    7159              : }
    7160              : 
    7161              : struct move_stmt_d
    7162              : {
    7163              :   tree orig_block;
    7164              :   tree new_block;
    7165              :   tree from_context;
    7166              :   tree to_context;
    7167              :   hash_map<tree, tree> *vars_map;
    7168              :   htab_t new_label_map;
    7169              :   hash_map<void *, void *> *eh_map;
    7170              :   bool remap_decls_p;
    7171              : };
    7172              : 
    7173              : /* Helper for move_block_to_fn.  Set TREE_BLOCK in every expression
    7174              :    contained in *TP if it has been ORIG_BLOCK previously and change the
    7175              :    DECL_CONTEXT of every local variable referenced in *TP.  */
    7176              : 
    7177              : static tree
    7178      7451543 : move_stmt_op (tree *tp, int *walk_subtrees, void *data)
    7179              : {
    7180      7451543 :   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
    7181      7451543 :   struct move_stmt_d *p = (struct move_stmt_d *) wi->info;
    7182      7451543 :   tree t = *tp;
    7183              : 
    7184      7451543 :   if (EXPR_P (t))
    7185              :     {
    7186      1182842 :       tree block = TREE_BLOCK (t);
    7187      1182842 :       if (block == NULL_TREE)
    7188              :         ;
    7189        30912 :       else if (block == p->orig_block
    7190        28910 :                || p->orig_block == NULL_TREE)
    7191              :         {
    7192              :           /* tree_node_can_be_shared says we can share invariant
    7193              :              addresses but unshare_expr copies them anyways.  Make sure
    7194              :              to unshare before adjusting the block in place - we do not
    7195              :              always see a copy here.  */
    7196         2025 :           if (TREE_CODE (t) == ADDR_EXPR
    7197         2025 :               && is_gimple_min_invariant (t))
    7198         2002 :             *tp = t = unshare_expr (t);
    7199         2025 :           TREE_SET_BLOCK (t, p->new_block);
    7200              :         }
    7201        28887 :       else if (flag_checking)
    7202              :         {
    7203       104359 :           while (block && TREE_CODE (block) == BLOCK && block != p->orig_block)
    7204        75472 :             block = BLOCK_SUPERCONTEXT (block);
    7205        28887 :           gcc_assert (block == p->orig_block);
    7206              :         }
    7207              :     }
    7208      6268701 :   else if (DECL_P (t) || TREE_CODE (t) == SSA_NAME)
    7209              :     {
    7210      4564959 :       if (TREE_CODE (t) == SSA_NAME)
    7211        12862 :         *tp = replace_ssa_name (t, p->vars_map, p->to_context);
    7212      4552097 :       else if (TREE_CODE (t) == PARM_DECL
    7213      4552097 :                && gimple_in_ssa_p (cfun))
    7214          181 :         *tp = *(p->vars_map->get (t));
    7215      4551916 :       else if (TREE_CODE (t) == LABEL_DECL)
    7216              :         {
    7217         2820 :           if (p->new_label_map)
    7218              :             {
    7219          479 :               struct tree_map in, *out;
    7220          479 :               in.base.from = t;
    7221          479 :               out = (struct tree_map *)
    7222          479 :                 htab_find_with_hash (p->new_label_map, &in, DECL_UID (t));
    7223          479 :               if (out)
    7224           13 :                 *tp = t = out->to;
    7225              :             }
    7226              : 
    7227              :           /* For FORCED_LABELs we can end up with references from other
    7228              :              functions if some SESE regions are outlined.  It is UB to
    7229              :              jump in between them, but they could be used just for printing
    7230              :              addresses etc.  In that case, DECL_CONTEXT on the label should
    7231              :              be the function containing the glabel stmt with that LABEL_DECL,
    7232              :              rather than whatever function a reference to the label was seen
    7233              :              last time.  */
    7234         2820 :           if (!FORCED_LABEL (t) && !DECL_NONLOCAL (t))
    7235         2813 :             DECL_CONTEXT (t) = p->to_context;
    7236              :         }
    7237      4549096 :       else if (p->remap_decls_p)
    7238              :         {
    7239              :           /* Replace T with its duplicate.  T should no longer appear in the
    7240              :              parent function, so this looks wasteful; however, it may appear
    7241              :              in referenced_vars, and more importantly, as virtual operands of
    7242              :              statements, and in alias lists of other variables.  It would be
    7243              :              quite difficult to expunge it from all those places.  ??? It might
    7244              :              suffice to do this for addressable variables.  */
    7245      3828370 :           if ((VAR_P (t) && !is_global_var (t))
    7246      4589877 :               || TREE_CODE (t) == CONST_DECL)
    7247      3797978 :             replace_by_duplicate_decl (tp, p->vars_map, p->to_context);
    7248              :         }
    7249      4564959 :       *walk_subtrees = 0;
    7250      4564959 :     }
    7251      1703742 :   else if (TYPE_P (t))
    7252            0 :     *walk_subtrees = 0;
    7253              : 
    7254      7451543 :   return NULL_TREE;
    7255              : }
    7256              : 
    7257              : /* Helper for move_stmt_r.  Given an EH region number for the source
    7258              :    function, map that to the duplicate EH region number in the dest.  */
    7259              : 
    7260              : static int
    7261           18 : move_stmt_eh_region_nr (int old_nr, struct move_stmt_d *p)
    7262              : {
    7263           18 :   eh_region old_r, new_r;
    7264              : 
    7265           18 :   old_r = get_eh_region_from_number (old_nr);
    7266           18 :   new_r = static_cast<eh_region> (*p->eh_map->get (old_r));
    7267              : 
    7268           18 :   return new_r->index;
    7269              : }
    7270              : 
    7271              : /* Similar, but operate on INTEGER_CSTs.  */
    7272              : 
    7273              : static tree
    7274            5 : move_stmt_eh_region_tree_nr (tree old_t_nr, struct move_stmt_d *p)
    7275              : {
    7276            5 :   int old_nr, new_nr;
    7277              : 
    7278            5 :   old_nr = tree_to_shwi (old_t_nr);
    7279            5 :   new_nr = move_stmt_eh_region_nr (old_nr, p);
    7280              : 
    7281            5 :   return build_int_cst (integer_type_node, new_nr);
    7282              : }
    7283              : 
    7284              : /* Like move_stmt_op, but for gimple statements.
    7285              : 
    7286              :    Helper for move_block_to_fn.  Set GIMPLE_BLOCK in every expression
    7287              :    contained in the current statement in *GSI_P and change the
    7288              :    DECL_CONTEXT of every local variable referenced in the current
    7289              :    statement.  */
    7290              : 
    7291              : static tree
    7292      2090338 : move_stmt_r (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
    7293              :              struct walk_stmt_info *wi)
    7294              : {
    7295      2090338 :   struct move_stmt_d *p = (struct move_stmt_d *) wi->info;
    7296      2090338 :   gimple *stmt = gsi_stmt (*gsi_p);
    7297      2090338 :   tree block = gimple_block (stmt);
    7298              : 
    7299      2090338 :   if (block == p->orig_block
    7300      1856209 :       || (p->orig_block == NULL_TREE
    7301          955 :           && block != NULL_TREE))
    7302       235084 :     gimple_set_block (stmt, p->new_block);
    7303              : 
    7304      2090338 :   switch (gimple_code (stmt))
    7305              :     {
    7306       277275 :     case GIMPLE_CALL:
    7307              :       /* Remap the region numbers for __builtin_eh_{pointer,filter}.  */
    7308       277275 :       {
    7309       277275 :         tree r, fndecl = gimple_call_fndecl (stmt);
    7310       277275 :         if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
    7311        81815 :           switch (DECL_FUNCTION_CODE (fndecl))
    7312              :             {
    7313            0 :             case BUILT_IN_EH_COPY_VALUES:
    7314            0 :               r = gimple_call_arg (stmt, 1);
    7315            0 :               r = move_stmt_eh_region_tree_nr (r, p);
    7316            0 :               gimple_call_set_arg (stmt, 1, r);
    7317              :               /* FALLTHRU */
    7318              : 
    7319            5 :             case BUILT_IN_EH_POINTER:
    7320            5 :             case BUILT_IN_EH_FILTER:
    7321            5 :               r = gimple_call_arg (stmt, 0);
    7322            5 :               r = move_stmt_eh_region_tree_nr (r, p);
    7323            5 :               gimple_call_set_arg (stmt, 0, r);
    7324            5 :               break;
    7325              : 
    7326              :             default:
    7327              :               break;
    7328              :             }
    7329              :       }
    7330              :       break;
    7331              : 
    7332            8 :     case GIMPLE_RESX:
    7333            8 :       {
    7334            8 :         gresx *resx_stmt = as_a <gresx *> (stmt);
    7335            8 :         int r = gimple_resx_region (resx_stmt);
    7336            8 :         r = move_stmt_eh_region_nr (r, p);
    7337            8 :         gimple_resx_set_region (resx_stmt, r);
    7338              :       }
    7339            8 :       break;
    7340              : 
    7341            5 :     case GIMPLE_EH_DISPATCH:
    7342            5 :       {
    7343            5 :         geh_dispatch *eh_dispatch_stmt = as_a <geh_dispatch *> (stmt);
    7344            5 :         int r = gimple_eh_dispatch_region (eh_dispatch_stmt);
    7345            5 :         r = move_stmt_eh_region_nr (r, p);
    7346            5 :         gimple_eh_dispatch_set_region (eh_dispatch_stmt, r);
    7347              :       }
    7348            5 :       break;
    7349              : 
    7350              :     case GIMPLE_OMP_RETURN:
    7351              :     case GIMPLE_OMP_CONTINUE:
    7352              :       break;
    7353              : 
    7354         1418 :     case GIMPLE_LABEL:
    7355         1418 :       {
    7356              :         /* For FORCED_LABEL, move_stmt_op doesn't adjust DECL_CONTEXT,
    7357              :            so that such labels can be referenced from other regions.
    7358              :            Make sure to update it when seeing a GIMPLE_LABEL though,
    7359              :            that is the owner of the label.  */
    7360         1418 :         walk_gimple_op (stmt, move_stmt_op, wi);
    7361         1418 :         *handled_ops_p = true;
    7362         1418 :         tree label = gimple_label_label (as_a <glabel *> (stmt));
    7363         1418 :         if (FORCED_LABEL (label) || DECL_NONLOCAL (label))
    7364            5 :           DECL_CONTEXT (label) = p->to_context;
    7365              :       }
    7366              :       break;
    7367              : 
    7368      1811632 :     default:
    7369      1811632 :       if (is_gimple_omp (stmt))
    7370              :         {
    7371              :           /* Do not remap variables inside OMP directives.  Variables
    7372              :              referenced in clauses and directive header belong to the
    7373              :              parent function and should not be moved into the child
    7374              :              function.  */
    7375            0 :           bool save_remap_decls_p = p->remap_decls_p;
    7376            0 :           p->remap_decls_p = false;
    7377            0 :           *handled_ops_p = true;
    7378              : 
    7379            0 :           walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), move_stmt_r,
    7380              :                                move_stmt_op, wi);
    7381              : 
    7382            0 :           p->remap_decls_p = save_remap_decls_p;
    7383              :         }
    7384              :       break;
    7385              :     }
    7386              : 
    7387      2090338 :   return NULL_TREE;
    7388              : }
    7389              : 
    7390              : /* Move basic block BB from function CFUN to function DEST_FN.  The
    7391              :    block is moved out of the original linked list and placed after
    7392              :    block AFTER in the new list.  Also, the block is removed from the
    7393              :    original array of blocks and placed in DEST_FN's array of blocks.
    7394              :    If UPDATE_EDGE_COUNT_P is true, the edge counts on both CFGs is
    7395              :    updated to reflect the moved edges.
    7396              : 
    7397              :    The local variables are remapped to new instances, VARS_MAP is used
    7398              :    to record the mapping.  */
    7399              : 
    7400              : static void
    7401       645663 : move_block_to_fn (struct function *dest_cfun, basic_block bb,
    7402              :                   basic_block after, bool update_edge_count_p,
    7403              :                   struct move_stmt_d *d)
    7404              : {
    7405       645663 :   struct control_flow_graph *cfg;
    7406       645663 :   edge_iterator ei;
    7407       645663 :   edge e;
    7408       645663 :   gimple_stmt_iterator si;
    7409       645663 :   unsigned old_len;
    7410              : 
    7411              :   /* Remove BB from dominance structures.  */
    7412       645663 :   delete_from_dominance_info (CDI_DOMINATORS, bb);
    7413              : 
    7414              :   /* Move BB from its current loop to the copy in the new function.  */
    7415       645663 :   if (current_loops)
    7416              :     {
    7417       645663 :       class loop *new_loop = (class loop *)bb->loop_father->aux;
    7418       645663 :       if (new_loop)
    7419       363946 :         bb->loop_father = new_loop;
    7420              :     }
    7421              : 
    7422              :   /* Link BB to the new linked list.  */
    7423       645663 :   move_block_after (bb, after);
    7424              : 
    7425              :   /* Update the edge count in the corresponding flowgraphs.  */
    7426       645663 :   if (update_edge_count_p)
    7427      1382791 :     FOR_EACH_EDGE (e, ei, bb->succs)
    7428              :       {
    7429       781171 :         cfun->cfg->x_n_edges--;
    7430       781171 :         dest_cfun->cfg->x_n_edges++;
    7431              :       }
    7432              : 
    7433              :   /* Remove BB from the original basic block array.  */
    7434       645663 :   (*cfun->cfg->x_basic_block_info)[bb->index] = NULL;
    7435       645663 :   cfun->cfg->x_n_basic_blocks--;
    7436              : 
    7437              :   /* Grow DEST_CFUN's basic block array if needed.  */
    7438       645663 :   cfg = dest_cfun->cfg;
    7439       645663 :   cfg->x_n_basic_blocks++;
    7440       645663 :   if (bb->index >= cfg->x_last_basic_block)
    7441        44164 :     cfg->x_last_basic_block = bb->index + 1;
    7442              : 
    7443       645663 :   old_len = vec_safe_length (cfg->x_basic_block_info);
    7444       645663 :   if ((unsigned) cfg->x_last_basic_block >= old_len)
    7445        35650 :     vec_safe_grow_cleared (cfg->x_basic_block_info,
    7446        35650 :                            cfg->x_last_basic_block + 1);
    7447              : 
    7448       645663 :   (*cfg->x_basic_block_info)[bb->index] = bb;
    7449              : 
    7450              :   /* Remap the variables in phi nodes.  */
    7451       645663 :   for (gphi_iterator psi = gsi_start_phis (bb);
    7452       646829 :        !gsi_end_p (psi); )
    7453              :     {
    7454         1166 :       gphi *phi = psi.phi ();
    7455         1166 :       use_operand_p use;
    7456         1166 :       tree op = PHI_RESULT (phi);
    7457         1166 :       ssa_op_iter oi;
    7458         1166 :       unsigned i;
    7459              : 
    7460         2332 :       if (virtual_operand_p (op))
    7461              :         {
    7462              :           /* Remove the phi nodes for virtual operands (alias analysis will be
    7463              :              run for the new function, anyway).  But replace all uses that
    7464              :              might be outside of the region we move.  */
    7465          526 :           use_operand_p use_p;
    7466          526 :           imm_use_iterator iter;
    7467          526 :           gimple *use_stmt;
    7468         1045 :           FOR_EACH_IMM_USE_STMT (use_stmt, iter, op)
    7469         1038 :             FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
    7470          519 :               SET_USE (use_p, SSA_NAME_VAR (op));
    7471          526 :           remove_phi_node (&psi, true);
    7472          526 :           continue;
    7473          526 :         }
    7474              : 
    7475          640 :       SET_PHI_RESULT (phi,
    7476              :                       replace_ssa_name (op, d->vars_map, dest_cfun->decl));
    7477         1684 :       FOR_EACH_PHI_ARG (use, phi, oi, SSA_OP_USE)
    7478              :         {
    7479         1044 :           op = USE_FROM_PTR (use);
    7480         1044 :           if (TREE_CODE (op) == SSA_NAME)
    7481          894 :             SET_USE (use, replace_ssa_name (op, d->vars_map, dest_cfun->decl));
    7482              :         }
    7483              : 
    7484         1684 :       for (i = 0; i < EDGE_COUNT (bb->preds); i++)
    7485              :         {
    7486         1044 :           location_t locus = gimple_phi_arg_location (phi, i);
    7487         1044 :           tree block = LOCATION_BLOCK (locus);
    7488              : 
    7489         1044 :           if (locus == UNKNOWN_LOCATION)
    7490          884 :             continue;
    7491          160 :           if (d->orig_block == NULL_TREE || block == d->orig_block)
    7492              :             {
    7493          160 :               locus = set_block (locus, d->new_block);
    7494          160 :               gimple_phi_arg_set_location (phi, i, locus);
    7495              :             }
    7496              :         }
    7497              : 
    7498          640 :       gsi_next (&psi);
    7499              :     }
    7500              : 
    7501      3381664 :   for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
    7502              :     {
    7503      2090338 :       gimple *stmt = gsi_stmt (si);
    7504      2090338 :       struct walk_stmt_info wi;
    7505              : 
    7506      2090338 :       memset (&wi, 0, sizeof (wi));
    7507      2090338 :       wi.info = d;
    7508      2090338 :       walk_gimple_stmt (&si, move_stmt_r, move_stmt_op, &wi);
    7509              : 
    7510      2090338 :       if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
    7511              :         {
    7512         1418 :           tree label = gimple_label_label (label_stmt);
    7513         1418 :           int uid = LABEL_DECL_UID (label);
    7514              : 
    7515         1418 :           gcc_assert (uid > -1);
    7516              : 
    7517         1418 :           old_len = vec_safe_length (cfg->x_label_to_block_map);
    7518         1418 :           if (old_len <= (unsigned) uid)
    7519          171 :             vec_safe_grow_cleared (cfg->x_label_to_block_map, uid + 1);
    7520              : 
    7521         1418 :           (*cfg->x_label_to_block_map)[uid] = bb;
    7522         1418 :           (*cfun->cfg->x_label_to_block_map)[uid] = NULL;
    7523              : 
    7524         1418 :           gcc_assert (DECL_CONTEXT (label) == dest_cfun->decl);
    7525              : 
    7526         1418 :           if (uid >= dest_cfun->cfg->last_label_uid)
    7527          320 :             dest_cfun->cfg->last_label_uid = uid + 1;
    7528              :         }
    7529              : 
    7530      2090338 :       maybe_duplicate_eh_stmt_fn (dest_cfun, stmt, cfun, stmt, d->eh_map, 0);
    7531      2090338 :       remove_stmt_from_eh_lp_fn (cfun, stmt);
    7532              : 
    7533      2090338 :       gimple_duplicate_stmt_histograms (dest_cfun, stmt, cfun, stmt);
    7534      2090338 :       gimple_remove_stmt_histograms (cfun, stmt);
    7535              : 
    7536              :       /* We cannot leave any operands allocated from the operand caches of
    7537              :          the current function.  */
    7538      2090338 :       free_stmt_operands (cfun, stmt);
    7539      2090338 :       push_cfun (dest_cfun);
    7540      2090338 :       update_stmt (stmt);
    7541      2090338 :       if (is_gimple_call (stmt))
    7542       277275 :         notice_special_calls (as_a <gcall *> (stmt));
    7543      2090338 :       pop_cfun ();
    7544              :     }
    7545              : 
    7546      1426834 :   FOR_EACH_EDGE (e, ei, bb->succs)
    7547       781171 :     if (e->goto_locus != UNKNOWN_LOCATION)
    7548              :       {
    7549        35379 :         tree block = LOCATION_BLOCK (e->goto_locus);
    7550        35379 :         if (d->orig_block == NULL_TREE
    7551        35377 :             || block == d->orig_block)
    7552         2617 :           e->goto_locus = set_block (e->goto_locus, d->new_block);
    7553              :       }
    7554       645663 : }
    7555              : 
    7556              : /* Examine the statements in BB (which is in SRC_CFUN); find and return
    7557              :    the outermost EH region.  Use REGION as the incoming base EH region.
    7558              :    If there is no single outermost region, return NULL and set *ALL to
    7559              :    true.  */
    7560              : 
    7561              : static eh_region
    7562       645658 : find_outermost_region_in_block (struct function *src_cfun,
    7563              :                                 basic_block bb, eh_region region,
    7564              :                                 bool *all)
    7565              : {
    7566       645658 :   gimple_stmt_iterator si;
    7567              : 
    7568      3381642 :   for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
    7569              :     {
    7570      2090327 :       gimple *stmt = gsi_stmt (si);
    7571      2090327 :       eh_region stmt_region;
    7572      2090327 :       int lp_nr;
    7573              : 
    7574      2090327 :       lp_nr = lookup_stmt_eh_lp_fn (src_cfun, stmt);
    7575      2090327 :       stmt_region = get_eh_region_from_lp_number_fn (src_cfun, lp_nr);
    7576      2090327 :       if (stmt_region)
    7577              :         {
    7578         3916 :           if (region == NULL)
    7579              :             region = stmt_region;
    7580         1862 :           else if (stmt_region != region)
    7581              :             {
    7582          249 :               region = eh_region_outermost (src_cfun, stmt_region, region);
    7583          249 :               if (region == NULL)
    7584              :                 {
    7585            1 :                   *all = true;
    7586            1 :                   return NULL;
    7587              :                 }
    7588              :             }
    7589              :         }
    7590              :     }
    7591              : 
    7592              :   return region;
    7593              : }
    7594              : 
    7595              : static tree
    7596           13 : new_label_mapper (tree decl, void *data)
    7597              : {
    7598           13 :   htab_t hash = (htab_t) data;
    7599           13 :   struct tree_map *m;
    7600           13 :   void **slot;
    7601              : 
    7602           13 :   gcc_assert (TREE_CODE (decl) == LABEL_DECL);
    7603              : 
    7604           13 :   m = XNEW (struct tree_map);
    7605           13 :   m->hash = DECL_UID (decl);
    7606           13 :   m->base.from = decl;
    7607           13 :   m->to = create_artificial_label (UNKNOWN_LOCATION);
    7608           13 :   LABEL_DECL_UID (m->to) = LABEL_DECL_UID (decl);
    7609           13 :   if (LABEL_DECL_UID (m->to) >= cfun->cfg->last_label_uid)
    7610            5 :     cfun->cfg->last_label_uid = LABEL_DECL_UID (m->to) + 1;
    7611              : 
    7612           13 :   slot = htab_find_slot_with_hash (hash, m, m->hash, INSERT);
    7613           13 :   gcc_assert (*slot == NULL);
    7614              : 
    7615           13 :   *slot = m;
    7616              : 
    7617           13 :   return m->to;
    7618              : }
    7619              : 
    7620              : /* Tree walker to replace the decls used inside value expressions by
    7621              :    duplicates.  */
    7622              : 
    7623              : static tree
    7624       265996 : replace_block_vars_by_duplicates_1 (tree *tp, int *walk_subtrees, void *data)
    7625              : {
    7626       265996 :   struct replace_decls_d *rd = (struct replace_decls_d *)data;
    7627              : 
    7628       265996 :   switch (TREE_CODE (*tp))
    7629              :     {
    7630        50579 :     case VAR_DECL:
    7631        50579 :     case PARM_DECL:
    7632        50579 :     case RESULT_DECL:
    7633        50579 :       replace_by_duplicate_decl (tp, rd->vars_map, rd->to_context);
    7634        50579 :       break;
    7635              :     default:
    7636              :       break;
    7637              :     }
    7638              : 
    7639       265996 :   if (IS_TYPE_OR_DECL_P (*tp))
    7640        86162 :     *walk_subtrees = false;
    7641              : 
    7642       265996 :   return NULL;
    7643              : }
    7644              : 
    7645              : /* Change DECL_CONTEXT of all BLOCK_VARS in block, including
    7646              :    subblocks.  */
    7647              : 
    7648              : static void
    7649       171380 : replace_block_vars_by_duplicates (tree block, hash_map<tree, tree> *vars_map,
    7650              :                                   tree to_context)
    7651              : {
    7652       171380 :   tree *tp, t;
    7653              : 
    7654       834848 :   for (tp = &BLOCK_VARS (block); *tp; tp = &DECL_CHAIN (*tp))
    7655              :     {
    7656       663468 :       t = *tp;
    7657       663468 :       if (!VAR_P (t) && TREE_CODE (t) != CONST_DECL)
    7658         7259 :         continue;
    7659       656209 :       replace_by_duplicate_decl (&t, vars_map, to_context);
    7660       656209 :       if (t != *tp)
    7661              :         {
    7662       656209 :           if (VAR_P (*tp) && DECL_HAS_VALUE_EXPR_P (*tp))
    7663              :             {
    7664        45696 :               tree x = DECL_VALUE_EXPR (*tp);
    7665        45696 :               struct replace_decls_d rd = { vars_map, to_context };
    7666        45696 :               unshare_expr (x);
    7667        45696 :               walk_tree (&x, replace_block_vars_by_duplicates_1, &rd, NULL);
    7668        45696 :               SET_DECL_VALUE_EXPR (t, x);
    7669        45696 :               DECL_HAS_VALUE_EXPR_P (t) = 1;
    7670              :             }
    7671       656209 :           DECL_CHAIN (t) = DECL_CHAIN (*tp);
    7672       656209 :           *tp = t;
    7673              :         }
    7674              :     }
    7675              : 
    7676       298596 :   for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
    7677       127216 :     replace_block_vars_by_duplicates (block, vars_map, to_context);
    7678       171380 : }
    7679              : 
    7680              : /* Fixup the loop arrays and numbers after moving LOOP and its subloops
    7681              :    from FN1 to FN2.  */
    7682              : 
    7683              : static void
    7684        71737 : fixup_loop_arrays_after_move (struct function *fn1, struct function *fn2,
    7685              :                               class loop *loop)
    7686              : {
    7687              :   /* Discard it from the old loop array.  */
    7688       143474 :   (*get_loops (fn1))[loop->num] = NULL;
    7689              : 
    7690              :   /* Place it in the new loop array, assigning it a new number.  */
    7691        71737 :   loop->num = number_of_loops (fn2);
    7692        71737 :   vec_safe_push (loops_for_fn (fn2)->larray, loop);
    7693              : 
    7694              :   /* Recurse to children.  */
    7695       102481 :   for (loop = loop->inner; loop; loop = loop->next)
    7696        30744 :     fixup_loop_arrays_after_move (fn1, fn2, loop);
    7697        71737 : }
    7698              : 
    7699              : /* Verify that the blocks in BBS_P are a single-entry, single-exit region
    7700              :    delimited by ENTRY_BB and EXIT_BB, possibly containing noreturn blocks.  */
    7701              : 
    7702              : DEBUG_FUNCTION void
    7703        44164 : verify_sese (basic_block entry, basic_block exit, vec<basic_block> *bbs_p)
    7704              : {
    7705        44164 :   basic_block bb;
    7706        44164 :   edge_iterator ei;
    7707        44164 :   edge e;
    7708        44164 :   bitmap bbs = BITMAP_ALLOC (NULL);
    7709        44164 :   int i;
    7710              : 
    7711        44164 :   gcc_assert (entry != NULL);
    7712        44164 :   gcc_assert (entry != exit);
    7713        44164 :   gcc_assert (bbs_p != NULL);
    7714              : 
    7715        44164 :   gcc_assert (bbs_p->length () > 0);
    7716              : 
    7717       689827 :   FOR_EACH_VEC_ELT (*bbs_p, i, bb)
    7718       645663 :     bitmap_set_bit (bbs, bb->index);
    7719              : 
    7720        44164 :   gcc_assert (bitmap_bit_p (bbs, entry->index));
    7721        44164 :   gcc_assert (exit == NULL || bitmap_bit_p (bbs, exit->index));
    7722              : 
    7723       689827 :   FOR_EACH_VEC_ELT (*bbs_p, i, bb)
    7724              :     {
    7725       645663 :       if (bb == entry)
    7726              :         {
    7727        44164 :           gcc_assert (single_pred_p (entry));
    7728        44164 :           gcc_assert (!bitmap_bit_p (bbs, single_pred (entry)->index));
    7729              :         }
    7730              :       else
    7731      1382670 :         for (ei = ei_start (bb->preds); !ei_end_p (ei); ei_next (&ei))
    7732              :           {
    7733       781171 :             e = ei_edge (ei);
    7734       781171 :             gcc_assert (bitmap_bit_p (bbs, e->src->index));
    7735              :           }
    7736              : 
    7737       645663 :       if (bb == exit)
    7738              :         {
    7739        44043 :           gcc_assert (single_succ_p (exit));
    7740        44043 :           gcc_assert (!bitmap_bit_p (bbs, single_succ (exit)->index));
    7741              :         }
    7742              :       else
    7743      1382791 :         for (ei = ei_start (bb->succs); !ei_end_p (ei); ei_next (&ei))
    7744              :           {
    7745       781171 :             e = ei_edge (ei);
    7746       781171 :             gcc_assert (bitmap_bit_p (bbs, e->dest->index));
    7747              :           }
    7748              :     }
    7749              : 
    7750        44164 :   BITMAP_FREE (bbs);
    7751        44164 : }
    7752              : 
    7753              : /* If FROM is an SSA_NAME, mark the version in bitmap DATA.  */
    7754              : 
    7755              : bool
    7756      1329201 : gather_ssa_name_hash_map_from (tree const &from, tree const &, void *data)
    7757              : {
    7758      1329201 :   bitmap release_names = (bitmap)data;
    7759              : 
    7760      1329201 :   if (TREE_CODE (from) != SSA_NAME)
    7761              :     return true;
    7762              : 
    7763         5953 :   bitmap_set_bit (release_names, SSA_NAME_VERSION (from));
    7764         5953 :   return true;
    7765              : }
    7766              : 
    7767              : /* Return LOOP_DIST_ALIAS call if present in BB.  */
    7768              : 
    7769              : static gimple *
    7770            0 : find_loop_dist_alias (basic_block bb)
    7771              : {
    7772            0 :   gimple_stmt_iterator gsi = gsi_last_bb (bb);
    7773            0 :   if (!safe_is_a <gcond *> (*gsi))
    7774              :     return NULL;
    7775              : 
    7776            0 :   gsi_prev (&gsi);
    7777            0 :   if (gsi_end_p (gsi))
    7778              :     return NULL;
    7779              : 
    7780            0 :   gimple *g = gsi_stmt (gsi);
    7781            0 :   if (gimple_call_internal_p (g, IFN_LOOP_DIST_ALIAS))
    7782            0 :     return g;
    7783              :   return NULL;
    7784              : }
    7785              : 
    7786              : /* Fold loop internal call G like IFN_LOOP_VECTORIZED/IFN_LOOP_DIST_ALIAS
    7787              :    to VALUE and update any immediate uses of it's LHS.  */
    7788              : 
    7789              : void
    7790        26021 : fold_loop_internal_call (gimple *g, tree value)
    7791              : {
    7792        26021 :   tree lhs = gimple_call_lhs (g);
    7793        26021 :   use_operand_p use_p;
    7794        26021 :   imm_use_iterator iter;
    7795        26021 :   gimple *use_stmt;
    7796        26021 :   gimple_stmt_iterator gsi = gsi_for_stmt (g);
    7797              : 
    7798        26021 :   replace_call_with_value (&gsi, value);
    7799        51948 :   FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    7800              :     {
    7801        51854 :       FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
    7802        25927 :         SET_USE (use_p, value);
    7803        25927 :       update_stmt (use_stmt);
    7804              :       /* If we turn conditional to constant, scale profile counts.
    7805              :          We know that the conditional was created by loop distribution
    7806              :          and all basic blocks dominated by the taken edge are part of
    7807              :          the loop distributed.  */
    7808        25927 :       if (gimple_code (use_stmt) == GIMPLE_COND)
    7809              :         {
    7810        25927 :           edge true_edge, false_edge;
    7811        25927 :           extract_true_false_edges_from_block (gimple_bb (use_stmt),
    7812              :                                                &true_edge, &false_edge);
    7813        25927 :           edge taken_edge = NULL, other_edge = NULL;
    7814        25927 :           if (gimple_cond_true_p (as_a <gcond *>(use_stmt)))
    7815              :             {
    7816         6983 :               taken_edge = true_edge;
    7817         6983 :               other_edge = false_edge;
    7818              :             }
    7819        18944 :           else if (gimple_cond_false_p (as_a <gcond *>(use_stmt)))
    7820              :             {
    7821        18936 :               taken_edge = false_edge;
    7822        18936 :               other_edge = true_edge;
    7823              :             }
    7824        25919 :           if (taken_edge
    7825        25927 :               && !(taken_edge->probability == profile_probability::always ()))
    7826              :             {
    7827           30 :               profile_count old_count = taken_edge->count ();
    7828           30 :               profile_count new_count = taken_edge->src->count;
    7829           30 :               taken_edge->probability = profile_probability::always ();
    7830           30 :               other_edge->probability = profile_probability::never ();
    7831              :               /* If we have multiple predecessors, we can't use the dominance
    7832              :                  test.  This should not happen as the guarded code should
    7833              :                  start with pre-header.  */
    7834           30 :               gcc_assert (single_pred_edge (taken_edge->dest));
    7835           60 :               if (old_count.nonzero_p ())
    7836              :                 {
    7837           28 :                   taken_edge->dest->count
    7838           28 :                     = taken_edge->dest->count.apply_scale (new_count,
    7839              :                                                            old_count);
    7840           28 :                   scale_strictly_dominated_blocks (taken_edge->dest,
    7841              :                                                    new_count, old_count);
    7842              :                 }
    7843              :             }
    7844              :         }
    7845        26021 :     }
    7846        26021 : }
    7847              : 
    7848              : /* Move a single-entry, single-exit region delimited by ENTRY_BB and
    7849              :    EXIT_BB to function DEST_CFUN.  The whole region is replaced by a
    7850              :    single basic block in the original CFG and the new basic block is
    7851              :    returned.  DEST_CFUN must not have a CFG yet.
    7852              : 
    7853              :    Note that the region need not be a pure SESE region.  Blocks inside
    7854              :    the region may contain calls to abort/exit.  The only restriction
    7855              :    is that ENTRY_BB should be the only entry point and it must
    7856              :    dominate EXIT_BB.
    7857              : 
    7858              :    Change TREE_BLOCK of all statements in ORIG_BLOCK to the new
    7859              :    functions outermost BLOCK, move all subblocks of ORIG_BLOCK
    7860              :    to the new function.
    7861              : 
    7862              :    All local variables referenced in the region are assumed to be in
    7863              :    the corresponding BLOCK_VARS and unexpanded variable lists
    7864              :    associated with DEST_CFUN.
    7865              : 
    7866              :    TODO: investigate whether we can reuse gimple_duplicate_sese_region to
    7867              :    reimplement move_sese_region_to_fn by duplicating the region rather than
    7868              :    moving it.  */
    7869              : 
    7870              : basic_block
    7871        44164 : move_sese_region_to_fn (struct function *dest_cfun, basic_block entry_bb,
    7872              :                         basic_block exit_bb, tree orig_block)
    7873              : {
    7874        44164 :   vec<basic_block> bbs;
    7875        44164 :   basic_block dom_entry = get_immediate_dominator (CDI_DOMINATORS, entry_bb);
    7876        44164 :   basic_block after, bb, *entry_pred, *exit_succ, abb;
    7877        44164 :   struct function *saved_cfun = cfun;
    7878        44164 :   int *entry_flag, *exit_flag;
    7879        44164 :   profile_probability *entry_prob, *exit_prob;
    7880        44164 :   unsigned i, num_entry_edges, num_exit_edges, num_nodes;
    7881        44164 :   edge e;
    7882        44164 :   edge_iterator ei;
    7883        44164 :   htab_t new_label_map;
    7884        44164 :   hash_map<void *, void *> *eh_map;
    7885        44164 :   class loop *loop = entry_bb->loop_father;
    7886        44164 :   class loop *loop0 = get_loop (saved_cfun, 0);
    7887        44164 :   struct move_stmt_d d;
    7888              : 
    7889              :   /* If ENTRY does not strictly dominate EXIT, this cannot be an SESE
    7890              :      region.  */
    7891        44164 :   gcc_assert (entry_bb != exit_bb
    7892              :               && (!exit_bb
    7893              :                   || dominated_by_p (CDI_DOMINATORS, exit_bb, entry_bb)));
    7894              : 
    7895              :   /* Collect all the blocks in the region.  Manually add ENTRY_BB
    7896              :      because it won't be added by dfs_enumerate_from.  */
    7897        44164 :   bbs.create (0);
    7898        44164 :   bbs.safe_push (entry_bb);
    7899        44164 :   gather_blocks_in_sese_region (entry_bb, exit_bb, &bbs);
    7900              : 
    7901        44164 :   if (flag_checking)
    7902        44164 :     verify_sese (entry_bb, exit_bb, &bbs);
    7903              : 
    7904              :   /* The blocks that used to be dominated by something in BBS will now be
    7905              :      dominated by the new block.  */
    7906        44164 :   auto_vec<basic_block> dom_bbs = get_dominated_by_region (CDI_DOMINATORS,
    7907              :                                                            bbs.address (),
    7908        88328 :                                                            bbs.length ());
    7909              : 
    7910              :   /* Detach ENTRY_BB and EXIT_BB from CFUN->CFG.  We need to remember
    7911              :      the predecessor edges to ENTRY_BB and the successor edges to
    7912              :      EXIT_BB so that we can re-attach them to the new basic block that
    7913              :      will replace the region.  */
    7914        44164 :   num_entry_edges = EDGE_COUNT (entry_bb->preds);
    7915        44164 :   entry_pred = XNEWVEC (basic_block, num_entry_edges);
    7916        44164 :   entry_flag = XNEWVEC (int, num_entry_edges);
    7917        44164 :   entry_prob = XNEWVEC (profile_probability, num_entry_edges);
    7918        44164 :   i = 0;
    7919        88328 :   for (ei = ei_start (entry_bb->preds); (e = ei_safe_edge (ei)) != NULL;)
    7920              :     {
    7921        44164 :       entry_prob[i] = e->probability;
    7922        44164 :       entry_flag[i] = e->flags;
    7923        44164 :       entry_pred[i++] = e->src;
    7924        44164 :       remove_edge (e);
    7925              :     }
    7926              : 
    7927        44164 :   if (exit_bb)
    7928              :     {
    7929        44043 :       num_exit_edges = EDGE_COUNT (exit_bb->succs);
    7930        44043 :       exit_succ = XNEWVEC (basic_block, num_exit_edges);
    7931        44043 :       exit_flag = XNEWVEC (int, num_exit_edges);
    7932        44043 :       exit_prob = XNEWVEC (profile_probability, num_exit_edges);
    7933        44043 :       i = 0;
    7934        88086 :       for (ei = ei_start (exit_bb->succs); (e = ei_safe_edge (ei)) != NULL;)
    7935              :         {
    7936        44043 :           exit_prob[i] = e->probability;
    7937        44043 :           exit_flag[i] = e->flags;
    7938        44043 :           exit_succ[i++] = e->dest;
    7939        44043 :           remove_edge (e);
    7940              :         }
    7941              :     }
    7942              :   else
    7943              :     {
    7944              :       num_exit_edges = 0;
    7945              :       exit_succ = NULL;
    7946              :       exit_flag = NULL;
    7947              :       exit_prob = NULL;
    7948              :     }
    7949              : 
    7950              :   /* Switch context to the child function to initialize DEST_FN's CFG.  */
    7951        44164 :   gcc_assert (dest_cfun->cfg == NULL);
    7952        44164 :   push_cfun (dest_cfun);
    7953              : 
    7954        44164 :   init_empty_tree_cfg ();
    7955              : 
    7956              :   /* Initialize EH information for the new function.  */
    7957        44164 :   eh_map = NULL;
    7958        44164 :   new_label_map = NULL;
    7959        44164 :   if (saved_cfun->eh)
    7960              :     {
    7961        44164 :       eh_region region = NULL;
    7962        44164 :       bool all = false;
    7963              : 
    7964       689821 :       FOR_EACH_VEC_ELT (bbs, i, bb)
    7965              :         {
    7966       645658 :           region = find_outermost_region_in_block (saved_cfun, bb, region, &all);
    7967       645658 :           if (all)
    7968              :             break;
    7969              :         }
    7970              : 
    7971        44164 :       init_eh_for_function ();
    7972        44164 :       if (region != NULL || all)
    7973              :         {
    7974         2054 :           new_label_map = htab_create (17, tree_map_hash, tree_map_eq, free);
    7975         2054 :           eh_map = duplicate_eh_regions (saved_cfun, region, 0,
    7976              :                                          new_label_mapper, new_label_map);
    7977              :         }
    7978              :     }
    7979              : 
    7980              :   /* Initialize an empty loop tree.  */
    7981        44164 :   struct loops *loops = ggc_cleared_alloc<struct loops> ();
    7982        44164 :   init_loops_structure (dest_cfun, loops, 1);
    7983        44164 :   loops->state = LOOPS_MAY_HAVE_MULTIPLE_LATCHES;
    7984        44164 :   set_loops_for_fn (dest_cfun, loops);
    7985              : 
    7986        88328 :   vec<loop_p, va_gc> *larray = get_loops (saved_cfun)->copy ();
    7987              : 
    7988              :   /* Move the outlined loop tree part.  */
    7989        44164 :   num_nodes = bbs.length ();
    7990       689827 :   FOR_EACH_VEC_ELT (bbs, i, bb)
    7991              :     {
    7992       645663 :       if (bb->loop_father->header == bb)
    7993              :         {
    7994        71737 :           class loop *this_loop = bb->loop_father;
    7995              :           /* Avoid the need to remap SSA names used in nb_iterations.  */
    7996        71737 :           free_numbers_of_iterations_estimates (this_loop);
    7997        71737 :           class loop *outer = loop_outer (this_loop);
    7998        71737 :           if (outer == loop
    7999              :               /* If the SESE region contains some bbs ending with
    8000              :                  a noreturn call, those are considered to belong
    8001              :                  to the outermost loop in saved_cfun, rather than
    8002              :                  the entry_bb's loop_father.  */
    8003        71737 :               || outer == loop0)
    8004              :             {
    8005        40993 :               if (outer != loop)
    8006            8 :                 num_nodes -= this_loop->num_nodes;
    8007        40993 :               flow_loop_tree_node_remove (bb->loop_father);
    8008        40993 :               flow_loop_tree_node_add (get_loop (dest_cfun, 0), this_loop);
    8009        40993 :               fixup_loop_arrays_after_move (saved_cfun, cfun, this_loop);
    8010              :             }
    8011              :         }
    8012       573926 :       else if (bb->loop_father == loop0 && loop0 != loop)
    8013         1294 :         num_nodes--;
    8014              : 
    8015              :       /* Remove loop exits from the outlined region.  */
    8016       645663 :       if (loops_for_fn (saved_cfun)->exits)
    8017         5594 :         FOR_EACH_EDGE (e, ei, bb->succs)
    8018              :           {
    8019         3036 :             struct loops *l = loops_for_fn (saved_cfun);
    8020         3036 :             loop_exit **slot
    8021         3036 :               = l->exits->find_slot_with_hash (e, htab_hash_pointer (e),
    8022              :                                                NO_INSERT);
    8023         3036 :             if (slot)
    8024          287 :               l->exits->clear_slot (slot);
    8025              :           }
    8026              :     }
    8027              : 
    8028              :   /* Adjust the number of blocks in the tree root of the outlined part.  */
    8029        88328 :   get_loop (dest_cfun, 0)->num_nodes = bbs.length () + 2;
    8030              : 
    8031              :   /* Setup a mapping to be used by move_block_to_fn.  */
    8032        44164 :   loop->aux = current_loops->tree_root;
    8033        44164 :   loop0->aux = current_loops->tree_root;
    8034              : 
    8035              :   /* Fix up orig_loop_num.  If the block referenced in it has been moved
    8036              :      to dest_cfun, update orig_loop_num field, otherwise clear it.  */
    8037        44164 :   signed char *moved_orig_loop_num = NULL;
    8038       204229 :   for (auto dloop : loops_list (dest_cfun, 0))
    8039        71737 :     if (dloop->orig_loop_num)
    8040              :       {
    8041            0 :         if (moved_orig_loop_num == NULL)
    8042            0 :           moved_orig_loop_num
    8043            0 :             = XCNEWVEC (signed char, vec_safe_length (larray));
    8044            0 :         if ((*larray)[dloop->orig_loop_num] != NULL
    8045            0 :             && get_loop (saved_cfun, dloop->orig_loop_num) == NULL)
    8046              :           {
    8047            0 :             if (moved_orig_loop_num[dloop->orig_loop_num] >= 0
    8048            0 :                 && moved_orig_loop_num[dloop->orig_loop_num] < 2)
    8049            0 :               moved_orig_loop_num[dloop->orig_loop_num]++;
    8050            0 :             dloop->orig_loop_num = (*larray)[dloop->orig_loop_num]->num;
    8051              :           }
    8052              :         else
    8053              :           {
    8054            0 :             moved_orig_loop_num[dloop->orig_loop_num] = -1;
    8055            0 :             dloop->orig_loop_num = 0;
    8056              :           }
    8057        44164 :       }
    8058        44164 :   pop_cfun ();
    8059              : 
    8060        44164 :   if (moved_orig_loop_num)
    8061              :     {
    8062            0 :       FOR_EACH_VEC_ELT (bbs, i, bb)
    8063              :         {
    8064            0 :           gimple *g = find_loop_dist_alias (bb);
    8065            0 :           if (g == NULL)
    8066            0 :             continue;
    8067              : 
    8068            0 :           int orig_loop_num = tree_to_shwi (gimple_call_arg (g, 0));
    8069            0 :           gcc_assert (orig_loop_num
    8070              :                       && (unsigned) orig_loop_num < vec_safe_length (larray));
    8071            0 :           if (moved_orig_loop_num[orig_loop_num] == 2)
    8072              :             {
    8073              :               /* If we have moved both loops with this orig_loop_num into
    8074              :                  dest_cfun and the LOOP_DIST_ALIAS call is being moved there
    8075              :                  too, update the first argument.  */
    8076            0 :               gcc_assert ((*larray)[orig_loop_num] != NULL
    8077              :                           && (get_loop (saved_cfun, orig_loop_num) == NULL));
    8078            0 :               tree t = build_int_cst (integer_type_node,
    8079            0 :                                       (*larray)[orig_loop_num]->num);
    8080            0 :               gimple_call_set_arg (g, 0, t);
    8081            0 :               update_stmt (g);
    8082              :               /* Make sure the following loop will not update it.  */
    8083            0 :               moved_orig_loop_num[orig_loop_num] = 0;
    8084              :             }
    8085              :           else
    8086              :             /* Otherwise at least one of the loops stayed in saved_cfun.
    8087              :                Remove the LOOP_DIST_ALIAS call.  */
    8088            0 :             fold_loop_internal_call (g, gimple_call_arg (g, 1));
    8089              :         }
    8090            0 :       FOR_EACH_BB_FN (bb, saved_cfun)
    8091              :         {
    8092            0 :           gimple *g = find_loop_dist_alias (bb);
    8093            0 :           if (g == NULL)
    8094            0 :             continue;
    8095            0 :           int orig_loop_num = tree_to_shwi (gimple_call_arg (g, 0));
    8096            0 :           gcc_assert (orig_loop_num
    8097              :                       && (unsigned) orig_loop_num < vec_safe_length (larray));
    8098            0 :           if (moved_orig_loop_num[orig_loop_num])
    8099              :             /* LOOP_DIST_ALIAS call remained in saved_cfun, if at least one
    8100              :                of the corresponding loops was moved, remove it.  */
    8101            0 :             fold_loop_internal_call (g, gimple_call_arg (g, 1));
    8102              :         }
    8103            0 :       XDELETEVEC (moved_orig_loop_num);
    8104              :     }
    8105        44164 :   ggc_free (larray);
    8106              : 
    8107              :   /* Move blocks from BBS into DEST_CFUN.  */
    8108        44164 :   gcc_assert (bbs.length () >= 2);
    8109        44164 :   after = dest_cfun->cfg->x_entry_block_ptr;
    8110        44164 :   hash_map<tree, tree> vars_map;
    8111              : 
    8112        44164 :   memset (&d, 0, sizeof (d));
    8113        44164 :   d.orig_block = orig_block;
    8114        44164 :   d.new_block = DECL_INITIAL (dest_cfun->decl);
    8115        44164 :   d.from_context = cfun->decl;
    8116        44164 :   d.to_context = dest_cfun->decl;
    8117        44164 :   d.vars_map = &vars_map;
    8118        44164 :   d.new_label_map = new_label_map;
    8119        44164 :   d.eh_map = eh_map;
    8120        44164 :   d.remap_decls_p = true;
    8121              : 
    8122        44164 :   if (gimple_in_ssa_p (cfun))
    8123          386 :     for (tree arg = DECL_ARGUMENTS (d.to_context); arg; arg = DECL_CHAIN (arg))
    8124              :       {
    8125          193 :         tree narg = make_ssa_name_fn (dest_cfun, arg, gimple_build_nop ());
    8126          193 :         set_ssa_default_def (dest_cfun, arg, narg);
    8127          193 :         vars_map.put (arg, narg);
    8128              :       }
    8129              : 
    8130       689827 :   FOR_EACH_VEC_ELT (bbs, i, bb)
    8131              :     {
    8132              :       /* No need to update edge counts on the last block.  It has
    8133              :          already been updated earlier when we detached the region from
    8134              :          the original CFG.  */
    8135       645663 :       move_block_to_fn (dest_cfun, bb, after, bb != exit_bb, &d);
    8136       645663 :       after = bb;
    8137              :     }
    8138              : 
    8139              :   /* Adjust the maximum clique used.  */
    8140        44164 :   dest_cfun->last_clique = saved_cfun->last_clique;
    8141              : 
    8142        44164 :   loop->aux = NULL;
    8143        44164 :   loop0->aux = NULL;
    8144              :   /* Loop sizes are no longer correct, fix them up.  */
    8145        44164 :   loop->num_nodes -= num_nodes;
    8146        44164 :   for (class loop *outer = loop_outer (loop);
    8147        48351 :        outer; outer = loop_outer (outer))
    8148         4187 :     outer->num_nodes -= num_nodes;
    8149        44164 :   loop0->num_nodes -= bbs.length () - num_nodes;
    8150              : 
    8151        44164 :   if (saved_cfun->has_simduid_loops || saved_cfun->has_force_vectorize_loops)
    8152              :     {
    8153         9320 :       class loop *aloop;
    8154        31309 :       for (i = 0; vec_safe_iterate (loops->larray, i, &aloop); i++)
    8155        21989 :         if (aloop != NULL)
    8156              :           {
    8157        21989 :             if (aloop->simduid)
    8158              :               {
    8159         2391 :                 replace_by_duplicate_decl (&aloop->simduid, d.vars_map,
    8160              :                                            d.to_context);
    8161         2391 :                 dest_cfun->has_simduid_loops = true;
    8162              :               }
    8163        21989 :             if (aloop->force_vectorize)
    8164         4245 :               dest_cfun->has_force_vectorize_loops = true;
    8165              :           }
    8166              :     }
    8167              : 
    8168              :   /* Rewire BLOCK_SUBBLOCKS of orig_block.  */
    8169        44164 :   if (orig_block)
    8170              :     {
    8171        43971 :       tree block;
    8172        43971 :       gcc_assert (BLOCK_SUBBLOCKS (DECL_INITIAL (dest_cfun->decl))
    8173              :                   == NULL_TREE);
    8174        87942 :       BLOCK_SUBBLOCKS (DECL_INITIAL (dest_cfun->decl))
    8175        43971 :         = BLOCK_SUBBLOCKS (orig_block);
    8176        43971 :       for (block = BLOCK_SUBBLOCKS (orig_block);
    8177        84410 :            block; block = BLOCK_CHAIN (block))
    8178        40439 :         BLOCK_SUPERCONTEXT (block) = DECL_INITIAL (dest_cfun->decl);
    8179        43971 :       BLOCK_SUBBLOCKS (orig_block) = NULL_TREE;
    8180              :     }
    8181              : 
    8182        44164 :   replace_block_vars_by_duplicates (DECL_INITIAL (dest_cfun->decl),
    8183              :                                     &vars_map, dest_cfun->decl);
    8184              : 
    8185        44164 :   if (new_label_map)
    8186         2054 :     htab_delete (new_label_map);
    8187        44164 :   if (eh_map)
    8188         2054 :     delete eh_map;
    8189              : 
    8190              :   /* We need to release ssa-names in a defined order, so first find them,
    8191              :      and then iterate in ascending version order.  */
    8192        44164 :   bitmap release_names = BITMAP_ALLOC (NULL);
    8193      1373365 :   vars_map.traverse<void *, gather_ssa_name_hash_map_from> (release_names);
    8194        44164 :   bitmap_iterator bi;
    8195        50117 :   EXECUTE_IF_SET_IN_BITMAP (release_names, 0, i, bi)
    8196         5953 :     release_ssa_name (ssa_name (i));
    8197        44164 :   BITMAP_FREE (release_names);
    8198              : 
    8199              :   /* Rewire the entry and exit blocks.  The successor to the entry
    8200              :      block turns into the successor of DEST_FN's ENTRY_BLOCK_PTR in
    8201              :      the child function.  Similarly, the predecessor of DEST_FN's
    8202              :      EXIT_BLOCK_PTR turns into the predecessor of EXIT_BLOCK_PTR.  We
    8203              :      need to switch CFUN between DEST_CFUN and SAVED_CFUN so that the
    8204              :      various CFG manipulation function get to the right CFG.
    8205              : 
    8206              :      FIXME, this is silly.  The CFG ought to become a parameter to
    8207              :      these helpers.  */
    8208        44164 :   push_cfun (dest_cfun);
    8209        44164 :   ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = entry_bb->count;
    8210        44164 :   make_single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), entry_bb, EDGE_FALLTHRU);
    8211        44164 :   if (exit_bb)
    8212              :     {
    8213        44043 :       make_single_succ_edge (exit_bb,  EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
    8214        44043 :       EXIT_BLOCK_PTR_FOR_FN (cfun)->count = exit_bb->count;
    8215              :     }
    8216              :   else
    8217          121 :     EXIT_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ();
    8218        44164 :   pop_cfun ();
    8219              : 
    8220              :   /* Back in the original function, the SESE region has disappeared,
    8221              :      create a new basic block in its place.  */
    8222        44164 :   bb = create_empty_bb (entry_pred[0]);
    8223        44164 :   if (current_loops)
    8224        44164 :     add_bb_to_loop (bb, loop);
    8225        44164 :   profile_count count = profile_count::zero ();
    8226        88328 :   for (i = 0; i < num_entry_edges; i++)
    8227              :     {
    8228        44164 :       e = make_edge (entry_pred[i], bb, entry_flag[i]);
    8229        44164 :       e->probability = entry_prob[i];
    8230        44164 :       count += e->count ();
    8231              :     }
    8232        44164 :   bb->count = count;
    8233              : 
    8234        88207 :   for (i = 0; i < num_exit_edges; i++)
    8235              :     {
    8236        44043 :       e = make_edge (bb, exit_succ[i], exit_flag[i]);
    8237        44043 :       e->probability = exit_prob[i];
    8238              :     }
    8239              : 
    8240        44164 :   set_immediate_dominator (CDI_DOMINATORS, bb, dom_entry);
    8241        84353 :   FOR_EACH_VEC_ELT (dom_bbs, i, abb)
    8242        40189 :     set_immediate_dominator (CDI_DOMINATORS, abb, bb);
    8243              : 
    8244        44164 :   if (exit_bb)
    8245              :     {
    8246        44043 :       free (exit_prob);
    8247        44043 :       free (exit_flag);
    8248        44043 :       free (exit_succ);
    8249              :     }
    8250        44164 :   free (entry_prob);
    8251        44164 :   free (entry_flag);
    8252        44164 :   free (entry_pred);
    8253        44164 :   bbs.release ();
    8254              : 
    8255        44164 :   return bb;
    8256        44164 : }
    8257              : 
    8258              : /* Dump default def DEF to file FILE using FLAGS and indentation
    8259              :    SPC.  */
    8260              : 
    8261              : static void
    8262          554 : dump_default_def (FILE *file, tree def, int spc, dump_flags_t flags)
    8263              : {
    8264         1662 :   for (int i = 0; i < spc; ++i)
    8265         1108 :     fprintf (file, " ");
    8266          554 :   dump_ssaname_info_to_file (file, def, spc);
    8267              : 
    8268          554 :   print_generic_expr (file, TREE_TYPE (def), flags);
    8269          554 :   fprintf (file, " ");
    8270          554 :   print_generic_expr (file, def, flags);
    8271          554 :   fprintf (file, " = ");
    8272          554 :   print_generic_expr (file, SSA_NAME_VAR (def), flags);
    8273          554 :   fprintf (file, ";\n");
    8274          554 : }
    8275              : 
    8276              : /* Print no_sanitize attribute to FILE for a given attribute VALUE.  */
    8277              : 
    8278              : static void
    8279           69 : print_no_sanitize_attr_value (FILE *file, tree value)
    8280              : {
    8281           69 :   sanitize_code_type flags = tree_to_sanitize_code_type (value);
    8282           69 :   bool first = true;
    8283         2415 :   for (int i = 0; sanitizer_opts[i].name != NULL; ++i)
    8284              :     {
    8285         2346 :       if ((sanitizer_opts[i].flag & flags) == sanitizer_opts[i].flag)
    8286              :         {
    8287          289 :           if (!first)
    8288          220 :             fprintf (file, " | ");
    8289          289 :           fprintf (file, "%s", sanitizer_opts[i].name);
    8290          289 :           first = false;
    8291              :         }
    8292              :     }
    8293           69 : }
    8294              : 
    8295              : /* Dump FUNCTION_DECL FN to file FILE using FLAGS (see TDF_* in dumpfile.h)
    8296              :    */
    8297              : 
    8298              : void
    8299        68974 : dump_function_to_file (tree fndecl, FILE *file, dump_flags_t flags)
    8300              : {
    8301        68974 :   tree arg, var, old_current_fndecl = current_function_decl;
    8302        68974 :   struct function *dsf;
    8303        68974 :   bool ignore_topmost_bind = false, any_var = false;
    8304        68974 :   basic_block bb;
    8305        68974 :   tree chain;
    8306        68974 :   bool tmclone = (TREE_CODE (fndecl) == FUNCTION_DECL
    8307       132079 :                   && decl_is_tm_clone (fndecl));
    8308        68974 :   struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
    8309              : 
    8310        68974 :   tree fntype = TREE_TYPE (fndecl);
    8311        68974 :   tree attrs[] = { DECL_ATTRIBUTES (fndecl), TYPE_ATTRIBUTES (fntype) };
    8312              : 
    8313       206922 :   for (int i = 0; i != 2; ++i)
    8314              :     {
    8315       137948 :       if (!attrs[i])
    8316       113004 :         continue;
    8317              : 
    8318        24944 :       fprintf (file, "__attribute__((");
    8319              : 
    8320        24944 :       bool first = true;
    8321        24944 :       tree chain;
    8322        94999 :       for (chain = attrs[i]; chain; first = false, chain = TREE_CHAIN (chain))
    8323              :         {
    8324        45111 :           if (!first)
    8325        20167 :             fprintf (file, ", ");
    8326              : 
    8327        45111 :           tree name = get_attribute_name (chain);
    8328        45111 :           print_generic_expr (file, name, flags);
    8329        45111 :           if (TREE_VALUE (chain) != NULL_TREE)
    8330              :             {
    8331        11463 :               fprintf (file, " (");
    8332              : 
    8333        11463 :               if (strstr (IDENTIFIER_POINTER (name), "no_sanitize"))
    8334           69 :                 print_no_sanitize_attr_value (file, TREE_VALUE (chain));
    8335        11394 :               else if (!strcmp (IDENTIFIER_POINTER (name),
    8336              :                                 "omp declare variant base"))
    8337              :                 {
    8338          254 :                   tree a = TREE_VALUE (chain);
    8339          254 :                   print_generic_expr (file, TREE_PURPOSE (a), flags);
    8340          254 :                   fprintf (file, " match ");
    8341          254 :                   print_omp_context_selector (file, TREE_VALUE (a),
    8342              :                                               flags);
    8343              :                 }
    8344              :               else
    8345        11140 :                 print_generic_expr (file, TREE_VALUE (chain), flags);
    8346        11463 :               fprintf (file, ")");
    8347              :             }
    8348              :         }
    8349              : 
    8350        24944 :       fprintf (file, "))\n");
    8351              :     }
    8352              : 
    8353        68974 :   current_function_decl = fndecl;
    8354        68974 :   if (flags & TDF_GIMPLE)
    8355              :     {
    8356           44 :       static bool hotness_bb_param_printed = false;
    8357           44 :       if (profile_info != NULL
    8358            0 :           && !hotness_bb_param_printed)
    8359              :         {
    8360            0 :           hotness_bb_param_printed = true;
    8361            0 :           fprintf (file,
    8362              :                    "/* --param=gimple-fe-computed-hot-bb-threshold=%" PRId64
    8363              :                    " */\n", get_hot_bb_threshold ());
    8364              :         }
    8365              : 
    8366           44 :       print_generic_expr (file, TREE_TYPE (TREE_TYPE (fndecl)),
    8367              :                           flags | TDF_SLIM);
    8368           44 :       fprintf (file, " __GIMPLE (%s",
    8369           44 :                (fun->curr_properties & PROP_ssa) ? "ssa"
    8370            9 :                : (fun->curr_properties & PROP_cfg) ? "cfg"
    8371              :                : "");
    8372              : 
    8373           44 :       if (fun && fun->cfg)
    8374              :         {
    8375           40 :           basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (fun);
    8376           40 :           if (bb->count.initialized_p ())
    8377            0 :             fprintf (file, ",%s(%" PRIu64 ")",
    8378              :                      profile_quality_as_string (bb->count.quality ()),
    8379              :                      bb->count.value ());
    8380           40 :           if (flags & TDF_UID)
    8381            0 :             fprintf (file, ")\n%sD_%u (", function_name (fun),
    8382            0 :                      DECL_UID (fndecl));
    8383              :           else
    8384           40 :             fprintf (file, ")\n%s (", function_name (fun));
    8385              :         }
    8386              :     }
    8387              :   else
    8388              :     {
    8389        68930 :       print_generic_expr (file, TREE_TYPE (fntype), flags);
    8390        68930 :       if (flags & TDF_UID)
    8391         1298 :         fprintf (file, " %sD.%u %s(", function_name (fun), DECL_UID (fndecl),
    8392              :                  tmclone ? "[tm-clone] " : "");
    8393              :       else
    8394       136525 :         fprintf (file, " %s %s(", function_name (fun),
    8395              :                  tmclone ? "[tm-clone] " : "");
    8396              :     }
    8397              : 
    8398        68974 :   arg = DECL_ARGUMENTS (fndecl);
    8399       162638 :   while (arg)
    8400              :     {
    8401        93664 :       print_generic_expr (file, TREE_TYPE (arg), flags);
    8402        93664 :       fprintf (file, " ");
    8403        93664 :       print_generic_expr (file, arg, flags);
    8404        93664 :       if (DECL_CHAIN (arg))
    8405        47353 :         fprintf (file, ", ");
    8406        93664 :       arg = DECL_CHAIN (arg);
    8407              :     }
    8408        68974 :   fprintf (file, ")\n");
    8409              : 
    8410        68974 :   dsf = DECL_STRUCT_FUNCTION (fndecl);
    8411        68974 :   if (dsf && (flags & TDF_EH))
    8412          664 :     dump_eh_tree (file, dsf);
    8413              : 
    8414        68974 :   if (flags & TDF_RAW && !gimple_has_body_p (fndecl))
    8415              :     {
    8416            2 :       dump_node (fndecl, TDF_SLIM | flags, file);
    8417            2 :       current_function_decl = old_current_fndecl;
    8418            2 :       return;
    8419              :     }
    8420              : 
    8421              :   /* When GIMPLE is lowered, the variables are no longer available in
    8422              :      BIND_EXPRs, so display them separately.  */
    8423        68972 :   if (fun && fun->decl == fndecl && (fun->curr_properties & PROP_gimple_lcf))
    8424              :     {
    8425        56148 :       unsigned ix;
    8426        56148 :       ignore_topmost_bind = true;
    8427              : 
    8428        56148 :       fprintf (file, "{\n");
    8429       111072 :       if (gimple_in_ssa_p (fun)
    8430        54924 :           && (flags & TDF_ALIAS))
    8431              :         {
    8432         1236 :           for (arg = DECL_ARGUMENTS (fndecl); arg != NULL;
    8433          580 :                arg = DECL_CHAIN (arg))
    8434              :             {
    8435          580 :               tree def = ssa_default_def (fun, arg);
    8436          580 :               if (def)
    8437          554 :                 dump_default_def (file, def, 2, flags);
    8438              :             }
    8439              : 
    8440          656 :           tree res = DECL_RESULT (fun->decl);
    8441          656 :           if (res != NULL_TREE
    8442          656 :               && DECL_BY_REFERENCE (res))
    8443              :             {
    8444            0 :               tree def = ssa_default_def (fun, res);
    8445            0 :               if (def)
    8446            0 :                 dump_default_def (file, def, 2, flags);
    8447              :             }
    8448              : 
    8449          656 :           tree static_chain = fun->static_chain_decl;
    8450          656 :           if (static_chain != NULL_TREE)
    8451              :             {
    8452            0 :               tree def = ssa_default_def (fun, static_chain);
    8453            0 :               if (def)
    8454            0 :                 dump_default_def (file, def, 2, flags);
    8455              :             }
    8456              :         }
    8457              : 
    8458       107411 :       if (!vec_safe_is_empty (fun->local_decls))
    8459       313505 :         FOR_EACH_LOCAL_DECL (fun, ix, var)
    8460              :           {
    8461       280952 :             print_generic_decl (file, var, flags);
    8462       280952 :             fprintf (file, "\n");
    8463              : 
    8464       280952 :             any_var = true;
    8465              :           }
    8466              : 
    8467        56148 :       tree name;
    8468              : 
    8469        56148 :       if (gimple_in_ssa_p (fun))
    8470      1840400 :         FOR_EACH_SSA_NAME (ix, name, fun)
    8471              :           {
    8472      1340167 :             if (!SSA_NAME_VAR (name)
    8473              :                 /* SSA name with decls without a name still get
    8474              :                    dumped as _N, list those explicitly as well even
    8475              :                    though we've dumped the decl declaration as D.xxx
    8476              :                    above.  */
    8477       775326 :                 || !SSA_NAME_IDENTIFIER (name))
    8478              :               {
    8479       567583 :                 fprintf (file, "  ");
    8480       567583 :                 print_generic_expr (file, TREE_TYPE (name), flags);
    8481       567583 :                 fprintf (file, " ");
    8482       567583 :                 print_generic_expr (file, name, flags);
    8483       567583 :                 fprintf (file, ";\n");
    8484              : 
    8485       567583 :                 any_var = true;
    8486              :               }
    8487              :           }
    8488              :     }
    8489              : 
    8490        68972 :   if (fun && fun->decl == fndecl
    8491        68972 :       && fun->cfg
    8492        55864 :       && basic_block_info_for_fn (fun))
    8493              :     {
    8494              :       /* If the CFG has been built, emit a CFG-based dump.  */
    8495        55864 :       if (!ignore_topmost_bind)
    8496            0 :         fprintf (file, "{\n");
    8497              : 
    8498        55864 :       if (any_var && n_basic_blocks_for_fn (fun))
    8499        48533 :         fprintf (file, "\n");
    8500              : 
    8501       333677 :       FOR_EACH_BB_FN (bb, fun)
    8502       277813 :         dump_bb (file, bb, 2, flags);
    8503              : 
    8504        55864 :       fprintf (file, "}\n");
    8505              :     }
    8506        13108 :   else if (fun && (fun->curr_properties & PROP_gimple_any))
    8507              :     {
    8508              :       /* The function is now in GIMPLE form but the CFG has not been
    8509              :          built yet.  Emit the single sequence of GIMPLE statements
    8510              :          that make up its body.  */
    8511         6670 :       gimple_seq body = gimple_body (fndecl);
    8512              : 
    8513         6670 :       if (gimple_seq_first_stmt (body)
    8514         6670 :           && gimple_seq_first_stmt (body) == gimple_seq_last_stmt (body)
    8515        13068 :           && gimple_code (gimple_seq_first_stmt (body)) == GIMPLE_BIND)
    8516         6386 :         print_gimple_seq (file, body, 0, flags);
    8517              :       else
    8518              :         {
    8519          284 :           if (!ignore_topmost_bind)
    8520            0 :             fprintf (file, "{\n");
    8521              : 
    8522          284 :           if (any_var)
    8523          189 :             fprintf (file, "\n");
    8524              : 
    8525          284 :           print_gimple_seq (file, body, 2, flags);
    8526          284 :           fprintf (file, "}\n");
    8527              :         }
    8528              :     }
    8529              :   else
    8530              :     {
    8531         6438 :       int indent;
    8532              : 
    8533              :       /* Make a tree based dump.  */
    8534         6438 :       chain = DECL_SAVED_TREE (fndecl);
    8535         6438 :       if (chain && TREE_CODE (chain) == BIND_EXPR)
    8536              :         {
    8537         6438 :           if (ignore_topmost_bind)
    8538              :             {
    8539            0 :               chain = BIND_EXPR_BODY (chain);
    8540            0 :               indent = 2;
    8541              :             }
    8542              :           else
    8543              :             indent = 0;
    8544              :         }
    8545              :       else
    8546              :         {
    8547            0 :           if (!ignore_topmost_bind)
    8548              :             {
    8549            0 :               fprintf (file, "{\n");
    8550              :               /* No topmost bind, pretend it's ignored for later.  */
    8551            0 :               ignore_topmost_bind = true;
    8552              :             }
    8553              :           indent = 2;
    8554              :         }
    8555              : 
    8556         6438 :       if (any_var)
    8557            0 :         fprintf (file, "\n");
    8558              : 
    8559         6438 :       print_generic_stmt_indented (file, chain, flags, indent);
    8560         6438 :       if (ignore_topmost_bind)
    8561            0 :         fprintf (file, "}\n");
    8562              :     }
    8563              : 
    8564        68972 :   if (flags & TDF_ENUMERATE_LOCALS)
    8565           39 :     dump_enumerated_decls (file, flags);
    8566        68972 :   fprintf (file, "\n\n");
    8567              : 
    8568        68972 :   current_function_decl = old_current_fndecl;
    8569              : }
    8570              : 
    8571              : /* Dump FUNCTION_DECL FN to stderr using FLAGS (see TDF_* in tree.h)  */
    8572              : 
    8573              : DEBUG_FUNCTION void
    8574            0 : debug_function (tree fn, dump_flags_t flags)
    8575              : {
    8576            0 :   dump_function_to_file (fn, stderr, flags);
    8577            0 : }
    8578              : 
    8579              : 
    8580              : /* Print on FILE the indexes for the predecessors of basic_block BB.  */
    8581              : 
    8582              : static void
    8583         5344 : print_pred_bbs (FILE *file, basic_block bb)
    8584              : {
    8585         5344 :   edge e;
    8586         5344 :   edge_iterator ei;
    8587              : 
    8588        12101 :   FOR_EACH_EDGE (e, ei, bb->preds)
    8589         6757 :     fprintf (file, "bb_%d ", e->src->index);
    8590         5344 : }
    8591              : 
    8592              : 
    8593              : /* Print on FILE the indexes for the successors of basic_block BB.  */
    8594              : 
    8595              : static void
    8596         5344 : print_succ_bbs (FILE *file, basic_block bb)
    8597              : {
    8598         5344 :   edge e;
    8599         5344 :   edge_iterator ei;
    8600              : 
    8601        12196 :   FOR_EACH_EDGE (e, ei, bb->succs)
    8602         6852 :     fprintf (file, "bb_%d ", e->dest->index);
    8603         5344 : }
    8604              : 
    8605              : /* Print to FILE the basic block BB following the VERBOSITY level.  */
    8606              : 
    8607              : void
    8608         5344 : print_loops_bb (FILE *file, basic_block bb, int indent, int verbosity)
    8609              : {
    8610         5344 :   char *s_indent = (char *) alloca ((size_t) indent + 1);
    8611         5344 :   memset ((void *) s_indent, ' ', (size_t) indent);
    8612         5344 :   s_indent[indent] = '\0';
    8613              : 
    8614              :   /* Print basic_block's header.  */
    8615         5344 :   if (verbosity >= 2)
    8616              :     {
    8617         5344 :       fprintf (file, "%s  bb_%d (preds = {", s_indent, bb->index);
    8618         5344 :       print_pred_bbs (file, bb);
    8619         5344 :       fprintf (file, "}, succs = {");
    8620         5344 :       print_succ_bbs (file, bb);
    8621         5344 :       fprintf (file, "})\n");
    8622              :     }
    8623              : 
    8624              :   /* Print basic_block's body.  */
    8625         5344 :   if (verbosity >= 3)
    8626              :     {
    8627         2928 :       fprintf (file, "%s  {\n", s_indent);
    8628         2928 :       dump_bb (file, bb, indent + 4, TDF_VOPS|TDF_MEMSYMS);
    8629         2928 :       fprintf (file, "%s  }\n", s_indent);
    8630              :     }
    8631         5344 : }
    8632              : 
    8633              : /* Print loop information.  */
    8634              : 
    8635              : void
    8636        10513 : print_loop_info (FILE *file, const class loop *loop, const char *prefix)
    8637              : {
    8638        10513 :   if (loop->can_be_parallel)
    8639            0 :     fprintf (file, ", can_be_parallel");
    8640        10513 :   if (loop->warned_aggressive_loop_optimizations)
    8641            0 :     fprintf (file, ", warned_aggressive_loop_optimizations");
    8642        10513 :   if (loop->dont_vectorize)
    8643            7 :     fprintf (file, ", dont_vectorize");
    8644        10513 :   if (loop->force_vectorize)
    8645            0 :     fprintf (file, ", force_vectorize");
    8646        10513 :   if (loop->in_oacc_kernels_region)
    8647          181 :     fprintf (file, ", in_oacc_kernels_region");
    8648        10513 :   if (loop->finite_p)
    8649         2122 :     fprintf (file, ", finite_p");
    8650        10513 :   if (loop->unroll)
    8651           68 :     fprintf (file, "\n%sunroll %d", prefix, loop->unroll);
    8652        10513 :   if (loop->nb_iterations)
    8653              :     {
    8654           44 :       fprintf (file, "\n%sniter ", prefix);
    8655           44 :       print_generic_expr (file, loop->nb_iterations);
    8656              :     }
    8657              : 
    8658        10513 :   if (loop->any_upper_bound)
    8659              :     {
    8660         2073 :       fprintf (file, "\n%supper_bound ", prefix);
    8661         2073 :       print_decu (loop->nb_iterations_upper_bound, file);
    8662              :     }
    8663        10513 :   if (loop->any_likely_upper_bound)
    8664              :     {
    8665         2073 :       fprintf (file, "\n%slikely_upper_bound ", prefix);
    8666         2073 :       print_decu (loop->nb_iterations_likely_upper_bound, file);
    8667              :     }
    8668              : 
    8669        10513 :   if (loop->any_estimate)
    8670              :     {
    8671          939 :       fprintf (file, "\n%sestimate ", prefix);
    8672          939 :       print_decu (loop->nb_iterations_estimate, file);
    8673              :     }
    8674        10513 :   bool reliable;
    8675        10513 :   sreal iterations;
    8676        10513 :   if (loop->num && expected_loop_iterations_by_profile (loop, &iterations, &reliable))
    8677              :     {
    8678         5297 :       fprintf (file, "\n%siterations by profile: %f (%s%s) entry count:", prefix,
    8679              :                iterations.to_double (), reliable ? "reliable" : "unreliable",
    8680         2425 :                maybe_flat_loop_profile (loop) ? ", maybe flat" : "");
    8681         2425 :       loop_count_in (loop).dump (file, cfun);
    8682              :     }
    8683              : 
    8684        10513 : }
    8685              : 
    8686              : static void print_loop_and_siblings (FILE *, class loop *, int, int);
    8687              : 
    8688              : /* Pretty print LOOP on FILE, indented INDENT spaces.  Following
    8689              :    VERBOSITY level this outputs the contents of the loop, or just its
    8690              :    structure.  */
    8691              : 
    8692              : static void
    8693         1266 : print_loop (FILE *file, class loop *loop, int indent, int verbosity)
    8694              : {
    8695         1266 :   char *s_indent;
    8696         1266 :   basic_block bb;
    8697              : 
    8698         1266 :   if (loop == NULL)
    8699              :     return;
    8700              : 
    8701         1266 :   s_indent = (char *) alloca ((size_t) indent + 1);
    8702         1266 :   memset ((void *) s_indent, ' ', (size_t) indent);
    8703         1266 :   s_indent[indent] = '\0';
    8704              : 
    8705              :   /* Print loop's header.  */
    8706         1266 :   fprintf (file, "%sloop_%d (", s_indent, loop->num);
    8707         1266 :   if (loop->header)
    8708         1266 :     fprintf (file, "header = %d", loop->header->index);
    8709              :   else
    8710              :     {
    8711            0 :       fprintf (file, "deleted)\n");
    8712            0 :       return;
    8713              :     }
    8714         1266 :   if (loop->latch)
    8715         1266 :     fprintf (file, ", latch = %d", loop->latch->index);
    8716              :   else
    8717            0 :     fprintf (file, ", multiple latches");
    8718         1266 :   print_loop_info (file, loop, s_indent);
    8719         1266 :   fprintf (file, ")\n");
    8720              : 
    8721              :   /* Print loop's body.  */
    8722         1266 :   if (verbosity >= 1)
    8723              :     {
    8724         1266 :       fprintf (file, "%s{\n", s_indent);
    8725        21548 :       FOR_EACH_BB_FN (bb, cfun)
    8726        20282 :         if (bb->loop_father == loop)
    8727         4832 :           print_loops_bb (file, bb, indent, verbosity);
    8728              : 
    8729         1266 :       print_loop_and_siblings (file, loop->inner, indent + 2, verbosity);
    8730         1266 :       fprintf (file, "%s}\n", s_indent);
    8731              :     }
    8732              : }
    8733              : 
    8734              : /* Print the LOOP and its sibling loops on FILE, indented INDENT
    8735              :    spaces.  Following VERBOSITY level this outputs the contents of the
    8736              :    loop, or just its structure.  */
    8737              : 
    8738              : static void
    8739         1266 : print_loop_and_siblings (FILE *file, class loop *loop, int indent,
    8740              :                          int verbosity)
    8741              : {
    8742         2154 :   if (loop == NULL)
    8743         1266 :     return;
    8744              : 
    8745         1266 :   print_loop (file, loop, indent, verbosity);
    8746          378 :   print_loop_and_siblings (file, loop->next, indent, verbosity);
    8747              : }
    8748              : 
    8749              : /* Follow a CFG edge from the entry point of the program, and on entry
    8750              :    of a loop, pretty print the loop structure on FILE.  */
    8751              : 
    8752              : void
    8753          378 : print_loops (FILE *file, int verbosity)
    8754              : {
    8755          378 :   basic_block bb;
    8756              : 
    8757          378 :   bb = ENTRY_BLOCK_PTR_FOR_FN (cfun);
    8758          378 :   fprintf (file, "\nLoops in function: %s\n", current_function_name ());
    8759          378 :   if (bb && bb->loop_father)
    8760          756 :     print_loop_and_siblings (file, bb->loop_father, 0, verbosity);
    8761          378 : }
    8762              : 
    8763              : /* Dump a loop.  */
    8764              : 
    8765              : DEBUG_FUNCTION void
    8766            0 : debug (class loop &ref)
    8767              : {
    8768            0 :   print_loop (stderr, &ref, 0, /*verbosity*/0);
    8769            0 : }
    8770              : 
    8771              : DEBUG_FUNCTION void
    8772            0 : debug (class loop *ptr)
    8773              : {
    8774            0 :   if (ptr)
    8775            0 :     debug (*ptr);
    8776              :   else
    8777            0 :     fprintf (stderr, "<nil>\n");
    8778            0 : }
    8779              : 
    8780              : /* Dump a loop verbosely.  */
    8781              : 
    8782              : DEBUG_FUNCTION void
    8783            0 : debug_verbose (class loop &ref)
    8784              : {
    8785            0 :   print_loop (stderr, &ref, 0, /*verbosity*/3);
    8786            0 : }
    8787              : 
    8788              : DEBUG_FUNCTION void
    8789            0 : debug_verbose (class loop *ptr)
    8790              : {
    8791            0 :   if (ptr)
    8792            0 :     debug (*ptr);
    8793              :   else
    8794            0 :     fprintf (stderr, "<nil>\n");
    8795            0 : }
    8796              : 
    8797              : 
    8798              : /* Debugging loops structure at tree level, at some VERBOSITY level.  */
    8799              : 
    8800              : DEBUG_FUNCTION void
    8801            0 : debug_loops (int verbosity)
    8802              : {
    8803            0 :   print_loops (stderr, verbosity);
    8804            0 : }
    8805              : 
    8806              : /* Print on stderr the code of LOOP, at some VERBOSITY level.  */
    8807              : 
    8808              : DEBUG_FUNCTION void
    8809            0 : debug_loop (class loop *loop, int verbosity)
    8810              : {
    8811            0 :   print_loop (stderr, loop, 0, verbosity);
    8812            0 : }
    8813              : 
    8814              : /* Print on stderr the code of loop number NUM, at some VERBOSITY
    8815              :    level.  */
    8816              : 
    8817              : DEBUG_FUNCTION void
    8818            0 : debug_loop_num (unsigned num, int verbosity)
    8819              : {
    8820            0 :   debug_loop (get_loop (cfun, num), verbosity);
    8821            0 : }
    8822              : 
    8823              : /* Return true if BB ends with a call, possibly followed by some
    8824              :    instructions that must stay with the call.  Return false,
    8825              :    otherwise.  */
    8826              : 
    8827              : static bool
    8828            3 : gimple_block_ends_with_call_p (basic_block bb)
    8829              : {
    8830            3 :   gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
    8831            3 :   return !gsi_end_p (gsi) && is_gimple_call (gsi_stmt (gsi));
    8832              : }
    8833              : 
    8834              : 
    8835              : /* Return true if BB ends with a conditional branch.  Return false,
    8836              :    otherwise.  */
    8837              : 
    8838              : static bool
    8839         1805 : gimple_block_ends_with_condjump_p (const_basic_block bb)
    8840              : {
    8841         3610 :   return safe_is_a <gcond *> (*gsi_last_bb (const_cast <basic_block> (bb)));
    8842              : }
    8843              : 
    8844              : 
    8845              : /* Return true if statement T may terminate execution of BB in ways not
    8846              :    explicitly represtented in the CFG.  */
    8847              : 
    8848              : bool
    8849    458452776 : stmt_can_terminate_bb_p (gimple *t)
    8850              : {
    8851    458452776 :   tree fndecl = NULL_TREE;
    8852    458452776 :   int call_flags = 0;
    8853              : 
    8854              :   /* Eh exception not handled internally terminates execution of the whole
    8855              :      function.  */
    8856    458452776 :   if (stmt_can_throw_external (cfun, t))
    8857              :     return true;
    8858              : 
    8859              :   /* NORETURN and LONGJMP calls already have an edge to exit.
    8860              :      CONST and PURE calls do not need one.
    8861              :      We don't currently check for CONST and PURE here, although
    8862              :      it would be a good idea, because those attributes are
    8863              :      figured out from the RTL in mark_constant_function, and
    8864              :      the counter incrementation code from -fprofile-arcs
    8865              :      leads to different results from -fbranch-probabilities.  */
    8866    450366479 :   if (is_gimple_call (t))
    8867              :     {
    8868     14892165 :       fndecl = gimple_call_fndecl (t);
    8869     14892165 :       call_flags = gimple_call_flags (t);
    8870              :     }
    8871              : 
    8872    450366479 :   if (is_gimple_call (t)
    8873     14892165 :       && fndecl
    8874     12926911 :       && fndecl_built_in_p (fndecl)
    8875      5260394 :       && (call_flags & ECF_NOTHROW)
    8876      4569982 :       && !(call_flags & ECF_RETURNS_TWICE)
    8877              :       /* fork() doesn't really return twice, but the effect of
    8878              :          wrapping it in __gcov_fork() which calls __gcov_dump() and
    8879              :          __gcov_reset() and clears the counters before forking has the same
    8880              :          effect as returning twice.  Force a fake edge.  */
    8881    454936094 :       && !fndecl_built_in_p (fndecl, BUILT_IN_FORK))
    8882              :     return false;
    8883              : 
    8884    445797079 :   if (is_gimple_call (t))
    8885              :     {
    8886     10322765 :       edge_iterator ei;
    8887     10322765 :       edge e;
    8888     10322765 :       basic_block bb;
    8889              : 
    8890     10322765 :       if (call_flags & (ECF_PURE | ECF_CONST)
    8891      2132909 :           && !(call_flags & ECF_LOOPING_CONST_OR_PURE))
    8892      9132426 :         return false;
    8893              : 
    8894              :       /* Function call may do longjmp, terminate program or do other things.
    8895              :          Special case noreturn that have non-abnormal edges out as in this case
    8896              :          the fact is sufficiently represented by lack of edges out of T.  */
    8897      8320498 :       if (!(call_flags & ECF_NORETURN))
    8898              :         return true;
    8899              : 
    8900      1289792 :       bb = gimple_bb (t);
    8901      2133264 :       FOR_EACH_EDGE (e, ei, bb->succs)
    8902       942925 :         if ((e->flags & EDGE_FAKE) == 0)
    8903              :           return true;
    8904              :     }
    8905              : 
    8906    436664653 :   if (gasm *asm_stmt = dyn_cast <gasm *> (t))
    8907       313989 :     if (gimple_asm_volatile_p (asm_stmt) || gimple_asm_basic_p (asm_stmt))
    8908              :       return true;
    8909              : 
    8910              :   return false;
    8911              : }
    8912              : 
    8913              : 
    8914              : /* Add fake edges to the function exit for any non constant and non
    8915              :    noreturn calls (or noreturn calls with EH/abnormal edges),
    8916              :    volatile inline assembly in the bitmap of blocks specified by BLOCKS
    8917              :    or to the whole CFG if BLOCKS is zero.  Return the number of blocks
    8918              :    that were split.
    8919              : 
    8920              :    The goal is to expose cases in which entering a basic block does
    8921              :    not imply that all subsequent instructions must be executed.  */
    8922              : 
    8923              : static int
    8924         3024 : gimple_flow_call_edges_add (sbitmap blocks)
    8925              : {
    8926         3024 :   int i;
    8927         3024 :   int blocks_split = 0;
    8928         3024 :   int last_bb = last_basic_block_for_fn (cfun);
    8929         3024 :   bool check_last_block = false;
    8930              : 
    8931         3024 :   if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
    8932              :     return 0;
    8933              : 
    8934         3024 :   if (! blocks)
    8935              :     check_last_block = true;
    8936              :   else
    8937            0 :     check_last_block = bitmap_bit_p (blocks,
    8938            0 :                                      EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->index);
    8939              : 
    8940              :   /* In the last basic block, before epilogue generation, there will be
    8941              :      a fallthru edge to EXIT.  Special care is required if the last insn
    8942              :      of the last basic block is a call because make_edge folds duplicate
    8943              :      edges, which would result in the fallthru edge also being marked
    8944              :      fake, which would result in the fallthru edge being removed by
    8945              :      remove_fake_edges, which would result in an invalid CFG.
    8946              : 
    8947              :      Moreover, we can't elide the outgoing fake edge, since the block
    8948              :      profiler needs to take this into account in order to solve the minimal
    8949              :      spanning tree in the case that the call doesn't return.
    8950              : 
    8951              :      Handle this by adding a dummy instruction in a new last basic block.  */
    8952            0 :   if (check_last_block)
    8953              :     {
    8954         3024 :       basic_block bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
    8955         3024 :       gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
    8956         3024 :       gimple *t = NULL;
    8957              : 
    8958         3024 :       if (!gsi_end_p (gsi))
    8959         3012 :         t = gsi_stmt (gsi);
    8960              : 
    8961         3012 :       if (t && stmt_can_terminate_bb_p (t))
    8962              :         {
    8963          202 :           edge e;
    8964              : 
    8965          202 :           e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
    8966          202 :           if (e)
    8967              :             {
    8968            0 :               gsi_insert_on_edge (e, gimple_build_nop ());
    8969            0 :               gsi_commit_edge_inserts ();
    8970              :             }
    8971              :         }
    8972              :     }
    8973              : 
    8974              :   /* Now add fake edges to the function exit for any non constant
    8975              :      calls since there is no way that we can determine if they will
    8976              :      return or not...  */
    8977        23161 :   for (i = 0; i < last_bb; i++)
    8978              :     {
    8979        20137 :       basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
    8980        20137 :       gimple_stmt_iterator gsi;
    8981        20137 :       gimple *stmt, *last_stmt;
    8982              : 
    8983        20137 :       if (!bb)
    8984            6 :         continue;
    8985              : 
    8986        20131 :       if (blocks && !bitmap_bit_p (blocks, i))
    8987            0 :         continue;
    8988              : 
    8989        20131 :       gsi = gsi_last_nondebug_bb (bb);
    8990        20131 :       if (!gsi_end_p (gsi))
    8991              :         {
    8992              :           last_stmt = gsi_stmt (gsi);
    8993        31043 :           do
    8994              :             {
    8995        31043 :               stmt = gsi_stmt (gsi);
    8996        31043 :               if (stmt_can_terminate_bb_p (stmt))
    8997              :                 {
    8998         4969 :                   edge e;
    8999              : 
    9000              :                   /* The handling above of the final block before the
    9001              :                      epilogue should be enough to verify that there is
    9002              :                      no edge to the exit block in CFG already.
    9003              :                      Calling make_edge in such case would cause us to
    9004              :                      mark that edge as fake and remove it later.  */
    9005         4969 :                   if (flag_checking && stmt == last_stmt)
    9006              :                     {
    9007         1647 :                       e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
    9008         1647 :                       gcc_assert (e == NULL);
    9009              :                     }
    9010              : 
    9011              :                   /* Note that the following may create a new basic block
    9012              :                      and renumber the existing basic blocks.  */
    9013         4969 :                   if (stmt != last_stmt)
    9014              :                     {
    9015         3322 :                       e = split_block (bb, stmt);
    9016         3322 :                       if (e)
    9017         3322 :                         blocks_split++;
    9018              :                     }
    9019         4969 :                   e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
    9020         4969 :                   e->probability = profile_probability::guessed_never ();
    9021              :                 }
    9022        31043 :               gsi_prev (&gsi);
    9023              :             }
    9024        31043 :           while (!gsi_end_p (gsi));
    9025              :         }
    9026              :     }
    9027              : 
    9028         3024 :   if (blocks_split)
    9029         1252 :     checking_verify_flow_info ();
    9030              : 
    9031              :   return blocks_split;
    9032              : }
    9033              : 
    9034              : /* Removes edge E and all the blocks dominated by it, and updates dominance
    9035              :    information.  The IL in E->src needs to be updated separately.
    9036              :    If dominance info is not available, only the edge E is removed.*/
    9037              : 
    9038              : void
    9039      3850695 : remove_edge_and_dominated_blocks (edge e)
    9040              : {
    9041      3850695 :   vec<basic_block> bbs_to_fix_dom = vNULL;
    9042      3850695 :   edge f;
    9043      3850695 :   edge_iterator ei;
    9044      3850695 :   bool none_removed = false;
    9045      3850695 :   unsigned i;
    9046      3850695 :   basic_block bb, dbb;
    9047      3850695 :   bitmap_iterator bi;
    9048              : 
    9049              :   /* If we are removing a path inside a non-root loop that may change
    9050              :      loop ownership of blocks or remove loops.  Mark loops for fixup.  */
    9051      3850695 :   class loop *src_loop = e->src->loop_father;
    9052      3850695 :   if (current_loops
    9053      3586397 :       && loop_outer (src_loop) != NULL
    9054      4556726 :       && src_loop == e->dest->loop_father)
    9055              :     {
    9056       295890 :       loops_state_set (LOOPS_NEED_FIXUP);
    9057              :       /* If we are removing a backedge clear the number of iterations
    9058              :          and estimates.  */
    9059       295890 :       class loop *dest_loop = e->dest->loop_father;
    9060       295890 :       if (e->dest == src_loop->header
    9061       295890 :           || (e->dest == dest_loop->header
    9062            0 :               && flow_loop_nested_p (dest_loop, src_loop)))
    9063              :         {
    9064         1845 :           free_numbers_of_iterations_estimates (dest_loop);
    9065              :           /* If we removed the last backedge mark the loop for removal.  */
    9066         5518 :           FOR_EACH_EDGE (f, ei, dest_loop->header->preds)
    9067         3698 :             if (f != e
    9068         3698 :                 && (f->src->loop_father == dest_loop
    9069         1845 :                     || flow_loop_nested_p (dest_loop, f->src->loop_father)))
    9070              :               break;
    9071         1845 :           if (!f)
    9072         1820 :             mark_loop_for_removal (dest_loop);
    9073              :         }
    9074              :     }
    9075              : 
    9076      3850695 :   if (!dom_info_available_p (CDI_DOMINATORS))
    9077              :     {
    9078      3654761 :       remove_edge (e);
    9079      7309522 :       return;
    9080              :     }
    9081              : 
    9082              :   /* No updating is needed for edges to exit.  */
    9083       195934 :   if (e->dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
    9084              :     {
    9085            0 :       if (cfgcleanup_altered_bbs)
    9086            0 :         bitmap_set_bit (cfgcleanup_altered_bbs, e->src->index);
    9087            0 :       remove_edge (e);
    9088            0 :       return;
    9089              :     }
    9090              : 
    9091              :   /* First, we find the basic blocks to remove.  If E->dest has a predecessor
    9092              :      that is not dominated by E->dest, then this set is empty.  Otherwise,
    9093              :      all the basic blocks dominated by E->dest are removed.
    9094              : 
    9095              :      Also, to DF_IDOM we store the immediate dominators of the blocks in
    9096              :      the dominance frontier of E (i.e., of the successors of the
    9097              :      removed blocks, if there are any, and of E->dest otherwise).  */
    9098       209325 :   FOR_EACH_EDGE (f, ei, e->dest->preds)
    9099              :     {
    9100       204267 :       if (f == e)
    9101        13345 :         continue;
    9102              : 
    9103       190922 :       if (!dominated_by_p (CDI_DOMINATORS, f->src, e->dest))
    9104              :         {
    9105              :           none_removed = true;
    9106              :           break;
    9107              :         }
    9108              :     }
    9109              : 
    9110       195934 :   auto_bitmap df, df_idom;
    9111       195934 :   auto_vec<basic_block> bbs_to_remove;
    9112       195934 :   if (none_removed)
    9113       190876 :     bitmap_set_bit (df_idom,
    9114       190876 :                     get_immediate_dominator (CDI_DOMINATORS, e->dest)->index);
    9115              :   else
    9116              :     {
    9117         5058 :       bbs_to_remove = get_all_dominated_blocks (CDI_DOMINATORS, e->dest);
    9118        11455 :       FOR_EACH_VEC_ELT (bbs_to_remove, i, bb)
    9119              :         {
    9120        12916 :           FOR_EACH_EDGE (f, ei, bb->succs)
    9121              :             {
    9122         6519 :               if (f->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
    9123         6517 :                 bitmap_set_bit (df, f->dest->index);
    9124              :             }
    9125              :         }
    9126        11455 :       FOR_EACH_VEC_ELT (bbs_to_remove, i, bb)
    9127         6397 :         bitmap_clear_bit (df, bb->index);
    9128              : 
    9129         9838 :       EXECUTE_IF_SET_IN_BITMAP (df, 0, i, bi)
    9130              :         {
    9131         4780 :           bb = BASIC_BLOCK_FOR_FN (cfun, i);
    9132         4780 :           bitmap_set_bit (df_idom,
    9133         4780 :                           get_immediate_dominator (CDI_DOMINATORS, bb)->index);
    9134              :         }
    9135              :     }
    9136              : 
    9137       195934 :   if (cfgcleanup_altered_bbs)
    9138              :     {
    9139              :       /* Record the set of the altered basic blocks.  */
    9140        13091 :       bitmap_set_bit (cfgcleanup_altered_bbs, e->src->index);
    9141        13091 :       bitmap_ior_into (cfgcleanup_altered_bbs, df);
    9142              :     }
    9143              : 
    9144              :   /* Remove E and the cancelled blocks.  */
    9145       195934 :   if (none_removed)
    9146       190876 :     remove_edge (e);
    9147              :   else
    9148              :     {
    9149              :       /* Walk backwards so as to get a chance to substitute all
    9150              :          released DEFs into debug stmts.  See
    9151              :          eliminate_unnecessary_stmts() in tree-ssa-dce.cc for more
    9152              :          details.  */
    9153        16513 :       for (i = bbs_to_remove.length (); i-- > 0; )
    9154         6397 :         delete_basic_block (bbs_to_remove[i]);
    9155              :     }
    9156              : 
    9157              :   /* Update the dominance information.  The immediate dominator may change only
    9158              :      for blocks whose immediate dominator belongs to DF_IDOM:
    9159              : 
    9160              :      Suppose that idom(X) = Y before removal of E and idom(X) != Y after the
    9161              :      removal.  Let Z the arbitrary block such that idom(Z) = Y and
    9162              :      Z dominates X after the removal.  Before removal, there exists a path P
    9163              :      from Y to X that avoids Z.  Let F be the last edge on P that is
    9164              :      removed, and let W = F->dest.  Before removal, idom(W) = Y (since Y
    9165              :      dominates W, and because of P, Z does not dominate W), and W belongs to
    9166              :      the dominance frontier of E.  Therefore, Y belongs to DF_IDOM.  */
    9167       390939 :   EXECUTE_IF_SET_IN_BITMAP (df_idom, 0, i, bi)
    9168              :     {
    9169       195005 :       bb = BASIC_BLOCK_FOR_FN (cfun, i);
    9170       195005 :       for (dbb = first_dom_son (CDI_DOMINATORS, bb);
    9171       689288 :            dbb;
    9172       494283 :            dbb = next_dom_son (CDI_DOMINATORS, dbb))
    9173       494283 :         bbs_to_fix_dom.safe_push (dbb);
    9174              :     }
    9175              : 
    9176       195934 :   iterate_fix_dominators (CDI_DOMINATORS, bbs_to_fix_dom, true);
    9177              : 
    9178       195934 :   bbs_to_fix_dom.release ();
    9179       195934 : }
    9180              : 
    9181              : /* Purge dead EH edges from basic block BB.  */
    9182              : 
    9183              : bool
    9184    443993771 : gimple_purge_dead_eh_edges (basic_block bb)
    9185              : {
    9186    443993771 :   bool changed = false;
    9187    443993771 :   edge e;
    9188    443993771 :   edge_iterator ei;
    9189    443993771 :   gimple *stmt = *gsi_last_bb (bb);
    9190              : 
    9191    443993771 :   if (stmt && stmt_can_throw_internal (cfun, stmt))
    9192              :     return false;
    9193              : 
    9194    955296259 :   for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
    9195              :     {
    9196    555171367 :       if (e->flags & EDGE_EH)
    9197              :         {
    9198       959088 :           remove_edge_and_dominated_blocks (e);
    9199       959088 :           changed = true;
    9200              :         }
    9201              :       else
    9202    554212279 :         ei_next (&ei);
    9203              :     }
    9204              : 
    9205              :   return changed;
    9206              : }
    9207              : 
    9208              : /* Purge dead EH edges from basic block listed in BLOCKS.  */
    9209              : 
    9210              : bool
    9211      7442298 : gimple_purge_all_dead_eh_edges (const_bitmap blocks)
    9212              : {
    9213      7442298 :   bool changed = false;
    9214      7442298 :   unsigned i;
    9215      7442298 :   bitmap_iterator bi;
    9216              : 
    9217      7630722 :   EXECUTE_IF_SET_IN_BITMAP (blocks, 0, i, bi)
    9218              :     {
    9219       188424 :       basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
    9220              : 
    9221              :       /* Earlier gimple_purge_dead_eh_edges could have removed
    9222              :          this basic block already.  */
    9223       188424 :       gcc_assert (bb || changed);
    9224       188424 :       if (bb != NULL)
    9225       188391 :         changed |= gimple_purge_dead_eh_edges (bb);
    9226              :     }
    9227              : 
    9228      7442298 :   return changed;
    9229              : }
    9230              : 
    9231              : /* Purge dead abnormal call edges from basic block BB.  */
    9232              : 
    9233              : bool
    9234     86256353 : gimple_purge_dead_abnormal_call_edges (basic_block bb)
    9235              : {
    9236     86256353 :   bool changed = false;
    9237     86256353 :   edge e;
    9238     86256353 :   edge_iterator ei;
    9239     86256353 :   gimple *stmt = *gsi_last_bb (bb);
    9240              : 
    9241     86256353 :   if (stmt && stmt_can_make_abnormal_goto (stmt))
    9242              :     return false;
    9243              : 
    9244    203568299 :   for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
    9245              :     {
    9246    117360123 :       if (e->flags & EDGE_ABNORMAL)
    9247              :         {
    9248         1420 :           if (e->flags & EDGE_FALLTHRU)
    9249            0 :             e->flags &= ~EDGE_ABNORMAL;
    9250              :           else
    9251         1420 :             remove_edge_and_dominated_blocks (e);
    9252    117360123 :           changed = true;
    9253              :         }
    9254              :       else
    9255    117358703 :         ei_next (&ei);
    9256              :     }
    9257              : 
    9258              :   return changed;
    9259              : }
    9260              : 
    9261              : /* Purge dead abnormal call edges from basic block listed in BLOCKS.  */
    9262              : 
    9263              : bool
    9264      7383027 : gimple_purge_all_dead_abnormal_call_edges (const_bitmap blocks)
    9265              : {
    9266      7383027 :   bool changed = false;
    9267      7383027 :   unsigned i;
    9268      7383027 :   bitmap_iterator bi;
    9269              : 
    9270      7383047 :   EXECUTE_IF_SET_IN_BITMAP (blocks, 0, i, bi)
    9271              :     {
    9272           20 :       basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
    9273              : 
    9274              :       /* Earlier gimple_purge_dead_abnormal_call_edges could have removed
    9275              :          this basic block already.  */
    9276           20 :       gcc_assert (bb || changed);
    9277           20 :       if (bb != NULL)
    9278           20 :         changed |= gimple_purge_dead_abnormal_call_edges (bb);
    9279              :     }
    9280              : 
    9281      7383027 :   return changed;
    9282              : }
    9283              : 
    9284              : /* This function is called whenever a new edge is created or
    9285              :    redirected.  */
    9286              : 
    9287              : static void
    9288    165691434 : gimple_execute_on_growing_pred (edge e)
    9289              : {
    9290    165691434 :   basic_block bb = e->dest;
    9291              : 
    9292    165691434 :   if (!gimple_seq_empty_p (phi_nodes (bb)))
    9293     32004775 :     reserve_phi_args_for_new_edge (bb);
    9294    165691434 : }
    9295              : 
    9296              : /* This function is called immediately before edge E is removed from
    9297              :    the edge vector E->dest->preds.  */
    9298              : 
    9299              : static void
    9300    135849482 : gimple_execute_on_shrinking_pred (edge e)
    9301              : {
    9302    135849482 :   if (!gimple_seq_empty_p (phi_nodes (e->dest)))
    9303     36285575 :     remove_phi_args (e);
    9304    135849482 : }
    9305              : 
    9306              : /*---------------------------------------------------------------------------
    9307              :   Helper functions for Loop versioning
    9308              :   ---------------------------------------------------------------------------*/
    9309              : 
    9310              : /* Adjust phi nodes for 'first' basic block.  'second' basic block is a copy
    9311              :    of 'first'. Both of them are dominated by 'new_head' basic block. When
    9312              :    'new_head' was created by 'second's incoming edge it received phi arguments
    9313              :    on the edge by split_edge(). Later, additional edge 'e' was created to
    9314              :    connect 'new_head' and 'first'. Now this routine adds phi args on this
    9315              :    additional edge 'e' that new_head to second edge received as part of edge
    9316              :    splitting.  */
    9317              : 
    9318              : static void
    9319        36166 : gimple_lv_adjust_loop_header_phi (basic_block first, basic_block second,
    9320              :                                   basic_block new_head, edge e)
    9321              : {
    9322        36166 :   gphi *phi1, *phi2;
    9323        36166 :   gphi_iterator psi1, psi2;
    9324        36166 :   tree def;
    9325        36166 :   edge e2 = find_edge (new_head, second);
    9326              : 
    9327              :   /* Because NEW_HEAD has been created by splitting SECOND's incoming
    9328              :      edge, we should always have an edge from NEW_HEAD to SECOND.  */
    9329        36166 :   gcc_assert (e2 != NULL);
    9330              : 
    9331              :   /* Browse all 'second' basic block phi nodes and add phi args to
    9332              :      edge 'e' for 'first' head. PHI args are always in correct order.  */
    9333              : 
    9334        36166 :   for (psi2 = gsi_start_phis (second),
    9335        36166 :        psi1 = gsi_start_phis (first);
    9336       141302 :        !gsi_end_p (psi2) && !gsi_end_p (psi1);
    9337       105136 :        gsi_next (&psi2),  gsi_next (&psi1))
    9338              :     {
    9339       105136 :       phi1 = psi1.phi ();
    9340       105136 :       phi2 = psi2.phi ();
    9341       105136 :       def = PHI_ARG_DEF (phi2, e2->dest_idx);
    9342       105136 :       add_phi_arg (phi1, def, e, gimple_phi_arg_location_from_edge (phi2, e2));
    9343              :     }
    9344        36166 : }
    9345              : 
    9346              : 
    9347              : /* Adds a if else statement to COND_BB with condition COND_EXPR.
    9348              :    SECOND_HEAD is the destination of the THEN and FIRST_HEAD is
    9349              :    the destination of the ELSE part.  */
    9350              : 
    9351              : static void
    9352        36166 : gimple_lv_add_condition_to_bb (basic_block first_head ATTRIBUTE_UNUSED,
    9353              :                                basic_block second_head ATTRIBUTE_UNUSED,
    9354              :                                basic_block cond_bb, void *cond_e)
    9355              : {
    9356        36166 :   gimple_stmt_iterator gsi;
    9357        36166 :   gimple *new_cond_expr;
    9358        36166 :   tree cond_expr = (tree) cond_e;
    9359        36166 :   edge e0;
    9360              : 
    9361              :   /* Build new conditional expr */
    9362        36166 :   gsi = gsi_last_bb (cond_bb);
    9363              : 
    9364        36166 :   cond_expr = force_gimple_operand_gsi_1 (&gsi, cond_expr,
    9365              :                                           is_gimple_condexpr_for_cond,
    9366              :                                           NULL_TREE, false,
    9367              :                                           GSI_CONTINUE_LINKING);
    9368        36166 :   new_cond_expr = gimple_build_cond_from_tree (cond_expr,
    9369              :                                                NULL_TREE, NULL_TREE);
    9370              : 
    9371              :   /* Add new cond in cond_bb.  */
    9372        36166 :   gsi_insert_after (&gsi, new_cond_expr, GSI_NEW_STMT);
    9373              : 
    9374              :   /* Adjust edges appropriately to connect new head with first head
    9375              :      as well as second head.  */
    9376        36166 :   e0 = single_succ_edge (cond_bb);
    9377        36166 :   e0->flags &= ~EDGE_FALLTHRU;
    9378        36166 :   e0->flags |= EDGE_FALSE_VALUE;
    9379        36166 : }
    9380              : 
    9381              : 
    9382              : /* Do book-keeping of basic block BB for the profile consistency checker.
    9383              :    Store the counting in RECORD.  */
    9384              : static void
    9385            0 : gimple_account_profile_record (basic_block bb,
    9386              :                                struct profile_record *record)
    9387              : {
    9388            0 :   gimple_stmt_iterator i;
    9389            0 :   for (i = gsi_start_nondebug_after_labels_bb (bb); !gsi_end_p (i);
    9390            0 :        gsi_next_nondebug (&i))
    9391              :     {
    9392            0 :       record->size
    9393            0 :         += estimate_num_insns (gsi_stmt (i), &eni_size_weights);
    9394            0 :       if (profile_info)
    9395              :         {
    9396            0 :           if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ().initialized_p ()
    9397            0 :               && ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ().nonzero_p ()
    9398            0 :               && bb->count.ipa ().initialized_p ())
    9399            0 :             record->time
    9400            0 :               += estimate_num_insns (gsi_stmt (i),
    9401              :                                      &eni_time_weights)
    9402            0 :                                      * bb->count.ipa ().to_gcov_type ();
    9403              :         }
    9404            0 :       else if (bb->count.initialized_p ()
    9405            0 :                && ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ())
    9406            0 :         record->time
    9407            0 :           += estimate_num_insns
    9408            0 :                 (gsi_stmt (i),
    9409              :                  &eni_time_weights)
    9410            0 :                  * bb->count.to_sreal_scale
    9411            0 :                         (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count).to_double ();
    9412              :      else
    9413            0 :       record->time
    9414            0 :         += estimate_num_insns (gsi_stmt (i), &eni_time_weights);
    9415              :     }
    9416            0 : }
    9417              : 
    9418              : const struct cfg_hooks gimple_cfg_hooks = {
    9419              :   IR_GIMPLE,
    9420              :   gimple_verify_flow_info,
    9421              :   gimple_dump_bb,               /* dump_bb  */
    9422              :   gimple_dump_bb_for_graph,     /* dump_bb_for_graph  */
    9423              :   gimple_dump_bb_as_sarif_properties,
    9424              :   create_bb,                    /* create_basic_block  */
    9425              :   gimple_redirect_edge_and_branch, /* redirect_edge_and_branch  */
    9426              :   gimple_redirect_edge_and_branch_force, /* redirect_edge_and_branch_force  */
    9427              :   gimple_can_remove_branch_p,   /* can_remove_branch_p  */
    9428              :   remove_bb,                    /* delete_basic_block  */
    9429              :   gimple_split_block,           /* split_block  */
    9430              :   gimple_move_block_after,      /* move_block_after  */
    9431              :   gimple_can_merge_blocks_p,    /* can_merge_blocks_p  */
    9432              :   gimple_merge_blocks,          /* merge_blocks  */
    9433              :   gimple_predict_edge,          /* predict_edge  */
    9434              :   gimple_predicted_by_p,        /* predicted_by_p  */
    9435              :   gimple_can_duplicate_bb_p,    /* can_duplicate_block_p  */
    9436              :   gimple_duplicate_bb,          /* duplicate_block  */
    9437              :   gimple_split_edge,            /* split_edge  */
    9438              :   gimple_make_forwarder_block,  /* make_forward_block  */
    9439              :   NULL,                         /* tidy_fallthru_edge  */
    9440              :   NULL,                         /* force_nonfallthru */
    9441              :   gimple_block_ends_with_call_p,/* block_ends_with_call_p */
    9442              :   gimple_block_ends_with_condjump_p, /* block_ends_with_condjump_p */
    9443              :   gimple_flow_call_edges_add,   /* flow_call_edges_add */
    9444              :   gimple_execute_on_growing_pred,       /* execute_on_growing_pred */
    9445              :   gimple_execute_on_shrinking_pred, /* execute_on_shrinking_pred */
    9446              :   gimple_duplicate_loop_body_to_header_edge, /* duplicate loop for trees */
    9447              :   gimple_lv_add_condition_to_bb, /* lv_add_condition_to_bb */
    9448              :   gimple_lv_adjust_loop_header_phi, /* lv_adjust_loop_header_phi*/
    9449              :   extract_true_false_edges_from_block, /* extract_cond_bb_edges */
    9450              :   flush_pending_stmts,          /* flush_pending_stmts */
    9451              :   gimple_empty_block_p,           /* block_empty_p */
    9452              :   gimple_split_block_before_cond_jump, /* split_block_before_cond_jump */
    9453              :   gimple_account_profile_record,
    9454              : };
    9455              : 
    9456              : 
    9457              : /* Split all critical edges.  Split some extra (not necessarily critical) edges
    9458              :    if FOR_EDGE_INSERTION_P is true.  */
    9459              : 
    9460              : unsigned int
    9461      4314892 : split_critical_edges (bool for_edge_insertion_p /* = false */)
    9462              : {
    9463      4314892 :   basic_block bb;
    9464      4314892 :   edge e;
    9465      4314892 :   edge_iterator ei;
    9466              : 
    9467              :   /* split_edge can redirect edges out of SWITCH_EXPRs, which can get
    9468              :      expensive.  So we want to enable recording of edge to CASE_LABEL_EXPR
    9469              :      mappings around the calls to split_edge.  */
    9470      4314892 :   start_recording_case_labels ();
    9471     75966297 :   FOR_ALL_BB_FN (bb, cfun)
    9472              :     {
    9473    159051096 :       FOR_EACH_EDGE (e, ei, bb->succs)
    9474              :         {
    9475     87399691 :           if (EDGE_CRITICAL_P (e) && !(e->flags & EDGE_ABNORMAL))
    9476     15031802 :             split_edge (e);
    9477              :           /* PRE inserts statements to edges and expects that
    9478              :              since split_critical_edges was done beforehand, committing edge
    9479              :              insertions will not split more edges.  In addition to critical
    9480              :              edges we must split edges that have multiple successors and
    9481              :              end by control flow statements, such as RESX.
    9482              :              Go ahead and split them too.  This matches the logic in
    9483              :              gimple_find_edge_insert_loc.  */
    9484     72367889 :           else if (for_edge_insertion_p
    9485     56554074 :                    && (!single_pred_p (e->dest)
    9486     31565895 :                        || !gimple_seq_empty_p (phi_nodes (e->dest))
    9487     31319243 :                        || e->dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
    9488     28395817 :                    && e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
    9489    100763297 :                    && !(e->flags & EDGE_ABNORMAL))
    9490              :             {
    9491     28379210 :               gimple_stmt_iterator gsi;
    9492              : 
    9493     28379210 :               gsi = gsi_last_bb (e->src);
    9494     28379210 :               if (!gsi_end_p (gsi)
    9495     14022224 :                   && stmt_ends_bb_p (gsi_stmt (gsi))
    9496     31761419 :                   && (gimple_code (gsi_stmt (gsi)) != GIMPLE_RETURN
    9497       213564 :                       && !gimple_call_builtin_p (gsi_stmt (gsi),
    9498              :                                                  BUILT_IN_RETURN)))
    9499       212513 :                 split_edge (e);
    9500              :             }
    9501              :         }
    9502              :     }
    9503      4314892 :   end_recording_case_labels ();
    9504      4314892 :   return 0;
    9505              : }
    9506              : 
    9507              : namespace {
    9508              : 
    9509              : const pass_data pass_data_split_crit_edges =
    9510              : {
    9511              :   GIMPLE_PASS, /* type */
    9512              :   "crited", /* name */
    9513              :   OPTGROUP_NONE, /* optinfo_flags */
    9514              :   TV_TREE_SPLIT_EDGES, /* tv_id */
    9515              :   PROP_cfg, /* properties_required */
    9516              :   PROP_no_crit_edges, /* properties_provided */
    9517              :   0, /* properties_destroyed */
    9518              :   0, /* todo_flags_start */
    9519              :   0, /* todo_flags_finish */
    9520              : };
    9521              : 
    9522              : class pass_split_crit_edges : public gimple_opt_pass
    9523              : {
    9524              : public:
    9525       594360 :   pass_split_crit_edges (gcc::context *ctxt)
    9526      1188720 :     : gimple_opt_pass (pass_data_split_crit_edges, ctxt)
    9527              :   {}
    9528              : 
    9529              :   /* opt_pass methods: */
    9530      1065442 :   unsigned int execute (function *)  final override
    9531              :   {
    9532      1065442 :     return split_critical_edges ();
    9533              :   }
    9534              : 
    9535       297180 :   opt_pass * clone () final override
    9536              :   {
    9537       297180 :     return new pass_split_crit_edges (m_ctxt);
    9538              :   }
    9539              : }; // class pass_split_crit_edges
    9540              : 
    9541              : } // anon namespace
    9542              : 
    9543              : gimple_opt_pass *
    9544       297180 : make_pass_split_crit_edges (gcc::context *ctxt)
    9545              : {
    9546       297180 :   return new pass_split_crit_edges (ctxt);
    9547              : }
    9548              : 
    9549              : 
    9550              : /* Insert COND expression which is GIMPLE_COND after STMT
    9551              :    in basic block BB with appropriate basic block split
    9552              :    and creation of a new conditionally executed basic block.
    9553              :    Update profile so the new bb is visited with probability PROB.
    9554              :    Return created basic block.  */
    9555              : basic_block
    9556         2205 : insert_cond_bb (basic_block bb, gimple *stmt, gimple *cond,
    9557              :                 profile_probability prob)
    9558              : {
    9559         2205 :   edge fall = split_block (bb, stmt);
    9560         2205 :   gimple_stmt_iterator iter = gsi_last_bb (bb);
    9561         2205 :   basic_block new_bb;
    9562              : 
    9563              :   /* Insert cond statement.  */
    9564         2205 :   gcc_assert (gimple_code (cond) == GIMPLE_COND);
    9565         2205 :   if (gsi_end_p (iter))
    9566            0 :     gsi_insert_before (&iter, cond, GSI_CONTINUE_LINKING);
    9567              :   else
    9568         2205 :     gsi_insert_after (&iter, cond, GSI_CONTINUE_LINKING);
    9569              : 
    9570              :   /* Create conditionally executed block.  */
    9571         2205 :   new_bb = create_empty_bb (bb);
    9572         2205 :   if (coverage_suppressed_p (bb))
    9573            0 :     suppress_coverage (new_bb);
    9574         2205 :   edge e = make_edge (bb, new_bb, EDGE_TRUE_VALUE);
    9575         2205 :   e->probability = prob;
    9576         2205 :   new_bb->count = e->count ();
    9577         2205 :   make_single_succ_edge (new_bb, fall->dest, EDGE_FALLTHRU);
    9578              : 
    9579              :   /* Fix edge for split bb.  */
    9580         2205 :   fall->flags = EDGE_FALSE_VALUE;
    9581         2205 :   fall->probability -= e->probability;
    9582              : 
    9583              :   /* Update dominance info.  */
    9584         2205 :   if (dom_info_available_p (CDI_DOMINATORS))
    9585              :     {
    9586         2205 :       set_immediate_dominator (CDI_DOMINATORS, new_bb, bb);
    9587         2205 :       set_immediate_dominator (CDI_DOMINATORS, fall->dest, bb);
    9588              :     }
    9589              : 
    9590              :   /* Update loop info.  */
    9591         2205 :   if (current_loops)
    9592         2205 :     add_bb_to_loop (new_bb, bb->loop_father);
    9593              : 
    9594         2205 :   return new_bb;
    9595              : }
    9596              : 
    9597              : 
    9598              : 
    9599              : /* Given a basic block B which ends with a conditional and has
    9600              :    precisely two successors, determine which of the edges is taken if
    9601              :    the conditional is true and which is taken if the conditional is
    9602              :    false.  Set TRUE_EDGE and FALSE_EDGE appropriately.  */
    9603              : 
    9604              : void
    9605    856716620 : extract_true_false_edges_from_block (basic_block b,
    9606              :                                      edge *true_edge,
    9607              :                                      edge *false_edge)
    9608              : {
    9609    856716620 :   edge e = EDGE_SUCC (b, 0);
    9610              : 
    9611    856716620 :   if (e->flags & EDGE_TRUE_VALUE)
    9612              :     {
    9613    815609232 :       *true_edge = e;
    9614    815609232 :       *false_edge = EDGE_SUCC (b, 1);
    9615              :     }
    9616              :   else
    9617              :     {
    9618     41107388 :       *false_edge = e;
    9619     41107388 :       *true_edge = EDGE_SUCC (b, 1);
    9620              :     }
    9621    856716620 : }
    9622              : 
    9623              : 
    9624              : /* From a controlling predicate in the immediate dominator DOM of
    9625              :    PHIBLOCK determine the edges into PHIBLOCK that are chosen if the
    9626              :    predicate evaluates to true and false and store them to
    9627              :    *TRUE_CONTROLLED_EDGE and *FALSE_CONTROLLED_EDGE if
    9628              :    they are non-NULL.  Returns true if the edges can be determined,
    9629              :    else return false.  */
    9630              : 
    9631              : bool
    9632       142610 : extract_true_false_controlled_edges (basic_block dom, basic_block phiblock,
    9633              :                                      edge *true_controlled_edge,
    9634              :                                      edge *false_controlled_edge)
    9635              : {
    9636       142610 :   basic_block bb = phiblock;
    9637       142610 :   edge true_edge, false_edge, tem;
    9638       142610 :   edge e0 = NULL, e1 = NULL;
    9639              : 
    9640              :   /* We have to verify that one edge into the PHI node is dominated
    9641              :      by the true edge of the predicate block and the other edge
    9642              :      dominated by the false edge.  This ensures that the PHI argument
    9643              :      we are going to take is completely determined by the path we
    9644              :      take from the predicate block.
    9645              :      We can only use BB dominance checks below if the destination of
    9646              :      the true/false edges are dominated by their edge, thus only
    9647              :      have a single predecessor.  */
    9648       142610 :   extract_true_false_edges_from_block (dom, &true_edge, &false_edge);
    9649       142610 :   tem = EDGE_PRED (bb, 0);
    9650       142610 :   if (tem == true_edge
    9651       142610 :       || (single_pred_p (true_edge->dest)
    9652       103786 :           && (tem->src == true_edge->dest
    9653        63985 :               || dominated_by_p (CDI_DOMINATORS,
    9654              :                                  tem->src, true_edge->dest))))
    9655              :     e0 = tem;
    9656        59085 :   else if (tem == false_edge
    9657        59085 :            || (single_pred_p (false_edge->dest)
    9658        48293 :                && (tem->src == false_edge->dest
    9659        35906 :                    || dominated_by_p (CDI_DOMINATORS,
    9660              :                                       tem->src, false_edge->dest))))
    9661              :     e1 = tem;
    9662              :   else
    9663              :     return false;
    9664       118293 :   tem = EDGE_PRED (bb, 1);
    9665       118293 :   if (tem == true_edge
    9666       118293 :       || (single_pred_p (true_edge->dest)
    9667        83525 :           && (tem->src == true_edge->dest
    9668        66164 :               || dominated_by_p (CDI_DOMINATORS,
    9669              :                                  tem->src, true_edge->dest))))
    9670              :     e0 = tem;
    9671        86935 :   else if (tem == false_edge
    9672        86935 :            || (single_pred_p (false_edge->dest)
    9673        69435 :                && (tem->src == false_edge->dest
    9674        29601 :                    || dominated_by_p (CDI_DOMINATORS,
    9675              :                                       tem->src, false_edge->dest))))
    9676              :     e1 = tem;
    9677              :   else
    9678              :     return false;
    9679       106915 :   if (!e0 || !e1)
    9680              :     return false;
    9681              : 
    9682       106915 :   if (true_controlled_edge)
    9683       106915 :     *true_controlled_edge = e0;
    9684       106915 :   if (false_controlled_edge)
    9685       106915 :     *false_controlled_edge = e1;
    9686              : 
    9687              :   return true;
    9688              : }
    9689              : 
    9690              : /* Generate a range test LHS CODE RHS that determines whether INDEX is in the
    9691              :     range [low, high].  Place associated stmts before *GSI.  */
    9692              : 
    9693              : void
    9694         4611 : generate_range_test (basic_block bb, tree index, tree low, tree high,
    9695              :                      tree *lhs, tree *rhs)
    9696              : {
    9697         4611 :   tree type = TREE_TYPE (index);
    9698         4611 :   tree utype = range_check_type (type);
    9699              : 
    9700         4611 :   low = fold_convert (utype, low);
    9701         4611 :   high = fold_convert (utype, high);
    9702              : 
    9703         4611 :   gimple_seq seq = NULL;
    9704         4611 :   index = gimple_convert (&seq, utype, index);
    9705         4611 :   *lhs = gimple_build (&seq, MINUS_EXPR, utype, index, low);
    9706         4611 :   *rhs = const_binop (MINUS_EXPR, utype, high, low);
    9707              : 
    9708         4611 :   gimple_stmt_iterator gsi = gsi_last_bb (bb);
    9709         4611 :   gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
    9710         4611 : }
    9711              : 
    9712              : /* Return the basic block that belongs to label numbered INDEX
    9713              :    of a switch statement.  */
    9714              : 
    9715              : basic_block
    9716     62874969 : gimple_switch_label_bb (function *ifun, gswitch *gs, unsigned index)
    9717              : {
    9718     62874969 :   return label_to_block (ifun, CASE_LABEL (gimple_switch_label (gs, index)));
    9719              : }
    9720              : 
    9721              : /* Return the default basic block of a switch statement.  */
    9722              : 
    9723              : basic_block
    9724       351803 : gimple_switch_default_bb (function *ifun, gswitch *gs)
    9725              : {
    9726       351803 :   return gimple_switch_label_bb (ifun, gs, 0);
    9727              : }
    9728              : 
    9729              : /* Return the edge that belongs to label numbered INDEX
    9730              :    of a switch statement.  */
    9731              : 
    9732              : edge
    9733      1459604 : gimple_switch_edge (function *ifun, gswitch *gs, unsigned index)
    9734              : {
    9735      1459604 :   return find_edge (gimple_bb (gs), gimple_switch_label_bb (ifun, gs, index));
    9736              : }
    9737              : 
    9738              : /* Return the default edge of a switch statement.  */
    9739              : 
    9740              : edge
    9741       184661 : gimple_switch_default_edge (function *ifun, gswitch *gs)
    9742              : {
    9743       184661 :   return gimple_switch_edge (ifun, gs, 0);
    9744              : }
    9745              : 
    9746              : /* Return true if the only executable statement in BB is a GIMPLE_COND.  */
    9747              : 
    9748              : bool
    9749        15376 : cond_only_block_p (basic_block bb)
    9750              : {
    9751              :   /* BB must have no executable statements.  */
    9752        15376 :   gimple_stmt_iterator gsi = gsi_after_labels (bb);
    9753        15376 :   if (phi_nodes (bb))
    9754              :     return false;
    9755        34875 :   while (!gsi_end_p (gsi))
    9756              :     {
    9757        20232 :       gimple *stmt = gsi_stmt (gsi);
    9758        20232 :       if (is_gimple_debug (stmt))
    9759              :         ;
    9760        15376 :       else if (gimple_code (stmt) == GIMPLE_NOP
    9761              :                || gimple_code (stmt) == GIMPLE_PREDICT
    9762              :                || gimple_code (stmt) == GIMPLE_COND)
    9763              :         ;
    9764              :       else
    9765              :         return false;
    9766        19499 :       gsi_next (&gsi);
    9767              :     }
    9768              :   return true;
    9769              : }
    9770              : 
    9771              : 
    9772              : /* Emit return warnings.  */
    9773              : 
    9774              : namespace {
    9775              : 
    9776              : const pass_data pass_data_warn_function_return =
    9777              : {
    9778              :   GIMPLE_PASS, /* type */
    9779              :   "*warn_function_return", /* name */
    9780              :   OPTGROUP_NONE, /* optinfo_flags */
    9781              :   TV_NONE, /* tv_id */
    9782              :   PROP_cfg, /* properties_required */
    9783              :   0, /* properties_provided */
    9784              :   0, /* properties_destroyed */
    9785              :   0, /* todo_flags_start */
    9786              :   0, /* todo_flags_finish */
    9787              : };
    9788              : 
    9789              : class pass_warn_function_return : public gimple_opt_pass
    9790              : {
    9791              : public:
    9792       297180 :   pass_warn_function_return (gcc::context *ctxt)
    9793       594360 :     : gimple_opt_pass (pass_data_warn_function_return, ctxt)
    9794              :   {}
    9795              : 
    9796              :   /* opt_pass methods: */
    9797              :   unsigned int execute (function *) final override;
    9798              : 
    9799              : }; // class pass_warn_function_return
    9800              : 
    9801              : unsigned int
    9802      3010131 : pass_warn_function_return::execute (function *fun)
    9803              : {
    9804      3010131 :   location_t location;
    9805      3010131 :   gimple *last;
    9806      3010131 :   edge e;
    9807      3010131 :   edge_iterator ei;
    9808              : 
    9809      3010131 :   if (!targetm.warn_func_return (fun->decl))
    9810              :     return 0;
    9811              : 
    9812              :   /* If we have a path to EXIT, then we do return.  */
    9813      3010057 :   if (TREE_THIS_VOLATILE (fun->decl)
    9814      3010057 :       && EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (fun)->preds) > 0)
    9815              :     {
    9816           90 :       location = UNKNOWN_LOCATION;
    9817           90 :       for (ei = ei_start (EXIT_BLOCK_PTR_FOR_FN (fun)->preds);
    9818          180 :            (e = ei_safe_edge (ei)); )
    9819              :         {
    9820           90 :           last = *gsi_last_bb (e->src);
    9821              :           /* Warn about __builtin_return .*/
    9822           90 :           if (gimple_call_builtin_p (last, BUILT_IN_RETURN)
    9823           90 :               && location == UNKNOWN_LOCATION)
    9824              :             {
    9825            1 :               location = LOCATION_LOCUS (gimple_location (last));
    9826            1 :               ei_next (&ei);
    9827              :             }
    9828              :           /* Replace return stmts in noreturn functions
    9829              :              with __builtin_unreachable () call.  */
    9830           89 :           else if (gimple_code (last) == GIMPLE_RETURN)
    9831              :             {
    9832           89 :               location_t loc = gimple_location (last);
    9833           89 :               if (location == UNKNOWN_LOCATION)
    9834           89 :                 location = LOCATION_LOCUS (loc);
    9835           89 :               gimple *new_stmt = gimple_build_builtin_unreachable (loc);
    9836           89 :               gimple_stmt_iterator gsi = gsi_for_stmt (last);
    9837           89 :               gsi_replace (&gsi, new_stmt, true);
    9838           89 :               remove_edge (e);
    9839              :             }
    9840              :           else
    9841            0 :             ei_next (&ei);
    9842              :         }
    9843           90 :       if (location == UNKNOWN_LOCATION)
    9844            2 :         location = cfun->function_end_locus;
    9845           90 :       warning_at (location, 0, "%<noreturn%> function does return");
    9846              :     }
    9847              : 
    9848              :   /* If we see "return;" in some basic block, then we do reach the end
    9849              :      without returning a value.  */
    9850      3009967 :   else if (warn_return_type > 0
    9851      2021922 :            && !warning_suppressed_p (fun->decl, OPT_Wreturn_type)
    9852      5028660 :            && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fun->decl))))
    9853              :     {
    9854      2138615 :       FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (fun)->preds)
    9855              :         {
    9856      3200497 :           greturn *return_stmt = dyn_cast <greturn *> (*gsi_last_bb (e->src));
    9857      1066747 :           if (return_stmt
    9858      1066747 :               && gimple_return_retval (return_stmt) == NULL
    9859           37 :               && !warning_suppressed_p (return_stmt, OPT_Wreturn_type))
    9860              :             {
    9861           16 :               location = gimple_location (return_stmt);
    9862           16 :               if (LOCATION_LOCUS (location) == UNKNOWN_LOCATION)
    9863            0 :                 location = fun->function_end_locus;
    9864           16 :               if (warning_at (location, OPT_Wreturn_type,
    9865              :                               "control reaches end of non-void function"))
    9866           14 :                 suppress_warning (fun->decl, OPT_Wreturn_type);
    9867              :               break;
    9868              :             }
    9869              :         }
    9870              :       /* The C++ FE turns fallthrough from the end of non-void function
    9871              :          into __builtin_unreachable () call with BUILTINS_LOCATION.
    9872              :          Recognize those as well as calls from ubsan_instrument_return.  */
    9873      1071748 :       basic_block bb;
    9874      1071748 :       if (!warning_suppressed_p (fun->decl, OPT_Wreturn_type))
    9875      5058572 :         FOR_EACH_BB_FN (bb, fun)
    9876      3987372 :           if (EDGE_COUNT (bb->succs) == 0)
    9877              :             {
    9878       193507 :               gimple *last = *gsi_last_bb (bb);
    9879       193507 :               const enum built_in_function ubsan_missing_ret
    9880              :                 = BUILT_IN_UBSAN_HANDLE_MISSING_RETURN;
    9881       193507 :               if (last
    9882       193507 :                   && ((LOCATION_LOCUS (gimple_location (last))
    9883              :                        == BUILTINS_LOCATION
    9884          519 :                        && (gimple_call_builtin_p (last, BUILT_IN_UNREACHABLE)
    9885          190 :                            || gimple_call_builtin_p (last,
    9886              :                                                      BUILT_IN_UNREACHABLE_TRAP)
    9887            3 :                            || gimple_call_builtin_p (last, BUILT_IN_TRAP)))
    9888       192988 :                       || gimple_call_builtin_p (last, ubsan_missing_ret)))
    9889              :                 {
    9890          534 :                   gimple_stmt_iterator gsi = gsi_for_stmt (last);
    9891          534 :                   gsi_prev_nondebug (&gsi);
    9892          534 :                   gimple *prev = gsi_stmt (gsi);
    9893          534 :                   if (prev == NULL)
    9894              :                     location = UNKNOWN_LOCATION;
    9895              :                   else
    9896          186 :                     location = gimple_location (prev);
    9897          534 :                   if (LOCATION_LOCUS (location) == UNKNOWN_LOCATION)
    9898          374 :                     location = fun->function_end_locus;
    9899          534 :                   if (warning_at (location, OPT_Wreturn_type,
    9900              :                                   "control reaches end of non-void function"))
    9901          123 :                     suppress_warning (fun->decl, OPT_Wreturn_type);
    9902          534 :                   break;
    9903              :                 }
    9904              :             }
    9905              :     }
    9906              :   return 0;
    9907              : }
    9908              : 
    9909              : } // anon namespace
    9910              : 
    9911              : gimple_opt_pass *
    9912       297180 : make_pass_warn_function_return (gcc::context *ctxt)
    9913              : {
    9914       297180 :   return new pass_warn_function_return (ctxt);
    9915              : }
    9916              : 
    9917              : /* Walk a gimplified function and warn for functions whose return value is
    9918              :    ignored and attribute((warn_unused_result)) is set.  This is done before
    9919              :    inlining, so we don't have to worry about that.  */
    9920              : 
    9921              : static void
    9922     10330604 : do_warn_unused_result (gimple_seq seq)
    9923              : {
    9924     10330604 :   tree fdecl, ftype;
    9925     10330604 :   gimple_stmt_iterator i;
    9926              : 
    9927     64173996 :   for (i = gsi_start (seq); !gsi_end_p (i); gsi_next (&i))
    9928              :     {
    9929     53843392 :       gimple *g = gsi_stmt (i);
    9930              : 
    9931     53843392 :       switch (gimple_code (g))
    9932              :         {
    9933      3647745 :         case GIMPLE_BIND:
    9934      3647745 :           do_warn_unused_result (gimple_bind_body (as_a <gbind *>(g)));
    9935      3647745 :           break;
    9936      2087685 :         case GIMPLE_TRY:
    9937      2087685 :           do_warn_unused_result (gimple_try_eval (g));
    9938      2087685 :           do_warn_unused_result (gimple_try_cleanup (g));
    9939      2087685 :           break;
    9940        23286 :         case GIMPLE_CATCH:
    9941        46572 :           do_warn_unused_result (gimple_catch_handler (
    9942        23286 :                                    as_a <gcatch *> (g)));
    9943        23286 :           break;
    9944         4491 :         case GIMPLE_EH_FILTER:
    9945         4491 :           do_warn_unused_result (gimple_eh_filter_failure (g));
    9946         4491 :           break;
    9947              : 
    9948      8145767 :         case GIMPLE_CALL:
    9949      8145767 :           if (gimple_call_lhs (g))
    9950              :             break;
    9951      4492632 :           if (gimple_call_internal_p (g))
    9952              :             break;
    9953      4469969 :           if (warning_suppressed_p (g, OPT_Wunused_result))
    9954              :             break;
    9955              : 
    9956              :           /* This is a naked call, as opposed to a GIMPLE_CALL with an
    9957              :              LHS.  All calls whose value is ignored should be
    9958              :              represented like this.  Look for the attribute.  */
    9959      4427503 :           fdecl = gimple_call_fndecl (g);
    9960      4427503 :           ftype = gimple_call_fntype (g);
    9961              : 
    9962      4427503 :           if (lookup_attribute ("warn_unused_result", TYPE_ATTRIBUTES (ftype)))
    9963              :             {
    9964          186 :               auto_urlify_attributes sentinel;
    9965              : 
    9966          186 :               location_t loc = gimple_location (g);
    9967              : 
    9968          186 :               if (fdecl)
    9969          114 :                 warning_at (loc, OPT_Wunused_result,
    9970              :                             "ignoring return value of %qD "
    9971              :                             "declared with attribute %<warn_unused_result%>",
    9972              :                             fdecl);
    9973              :               else
    9974           72 :                 warning_at (loc, OPT_Wunused_result,
    9975              :                             "ignoring return value of function "
    9976              :                             "declared with attribute %<warn_unused_result%>");
    9977          186 :             }
    9978              :           break;
    9979              : 
    9980              :         default:
    9981              :           /* Not a container, not a call, or a call whose value is used.  */
    9982              :           break;
    9983              :         }
    9984              :     }
    9985     10330604 : }
    9986              : 
    9987              : namespace {
    9988              : 
    9989              : const pass_data pass_data_warn_unused_result =
    9990              : {
    9991              :   GIMPLE_PASS, /* type */
    9992              :   "*warn_unused_result", /* name */
    9993              :   OPTGROUP_NONE, /* optinfo_flags */
    9994              :   TV_NONE, /* tv_id */
    9995              :   PROP_gimple_any, /* properties_required */
    9996              :   0, /* properties_provided */
    9997              :   0, /* properties_destroyed */
    9998              :   0, /* todo_flags_start */
    9999              :   0, /* todo_flags_finish */
   10000              : };
   10001              : 
   10002              : class pass_warn_unused_result : public gimple_opt_pass
   10003              : {
   10004              : public:
   10005       297180 :   pass_warn_unused_result (gcc::context *ctxt)
   10006       594360 :     : gimple_opt_pass (pass_data_warn_unused_result, ctxt)
   10007              :   {}
   10008              : 
   10009              :   /* opt_pass methods: */
   10010      3010155 :   bool gate (function *)  final override { return flag_warn_unused_result; }
   10011      2479712 :   unsigned int execute (function *) final override
   10012              :     {
   10013      2479712 :       do_warn_unused_result (gimple_body (current_function_decl));
   10014      2479712 :       return 0;
   10015              :     }
   10016              : 
   10017              : }; // class pass_warn_unused_result
   10018              : 
   10019              : } // anon namespace
   10020              : 
   10021              : gimple_opt_pass *
   10022       297180 : make_pass_warn_unused_result (gcc::context *ctxt)
   10023              : {
   10024       297180 :   return new pass_warn_unused_result (ctxt);
   10025              : }
   10026              : 
   10027              : /* Maybe Remove stores to variables we marked write-only.
   10028              :    Return true if a store was removed. */
   10029              : static bool
   10030    473332573 : maybe_remove_writeonly_store (gimple_stmt_iterator &gsi, gimple *stmt,
   10031              :                               bitmap dce_ssa_names)
   10032              : {
   10033              :   /* Keep access when store has side effect, i.e. in case when source
   10034              :      is volatile.  */
   10035    473332573 :   if (!gimple_store_p (stmt)
   10036     64608814 :       || gimple_has_side_effects (stmt)
   10037    525289983 :       || optimize_debug)
   10038              :     return false;
   10039              : 
   10040     51905978 :   tree lhs = get_base_address (gimple_get_lhs (stmt));
   10041              : 
   10042     51905978 :   if (!VAR_P (lhs)
   10043     40735317 :       || (!TREE_STATIC (lhs) && !DECL_EXTERNAL (lhs))
   10044     59030497 :       || !varpool_node::get (lhs)->writeonly)
   10045              :     return false;
   10046              : 
   10047        31491 :   if (dump_file && (dump_flags & TDF_DETAILS))
   10048              :     {
   10049            0 :       fprintf (dump_file, "Removing statement, writes"
   10050              :                " to write only var:\n");
   10051            0 :       print_gimple_stmt (dump_file, stmt, 0,
   10052              :                          TDF_VOPS|TDF_MEMSYMS);
   10053              :     }
   10054              : 
   10055              :   /* Mark ssa name defining to be checked for simple dce. */
   10056        31491 :   if (gimple_assign_single_p (stmt))
   10057              :     {
   10058        31491 :       tree rhs = gimple_assign_rhs1 (stmt);
   10059        31491 :       if (TREE_CODE (rhs) == SSA_NAME
   10060        31491 :           && !SSA_NAME_IS_DEFAULT_DEF (rhs))
   10061         9824 :         bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION (rhs));
   10062              :     }
   10063        31491 :   unlink_stmt_vdef (stmt);
   10064        31491 :   gsi_remove (&gsi, true);
   10065        31491 :   release_defs (stmt);
   10066        31491 :   return true;
   10067              : }
   10068              : 
   10069              : /* IPA passes, compilation of earlier functions or inlining
   10070              :    might have changed some properties, such as marked functions nothrow,
   10071              :    pure, const or noreturn.
   10072              :    Remove redundant edges and basic blocks, and create new ones if necessary. */
   10073              : 
   10074              : unsigned int
   10075     11453485 : execute_fixup_cfg (void)
   10076              : {
   10077     11453485 :   basic_block bb;
   10078     11453485 :   gimple_stmt_iterator gsi;
   10079     11453485 :   int todo = 0;
   10080     11453485 :   cgraph_node *node = cgraph_node::get (current_function_decl);
   10081              :   /* Same scaling is also done by ipa_merge_profiles.  */
   10082     11453485 :   profile_count num = node->count;
   10083     11453485 :   profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
   10084     11453485 :   bool scale = num.initialized_p () && !(num == den);
   10085     11453485 :   auto_bitmap dce_ssa_names;
   10086              : 
   10087     11453485 :   if (scale)
   10088              :     {
   10089        30303 :       profile_count::adjust_for_ipa_scaling (&num, &den);
   10090        30303 :       ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
   10091        30303 :       EXIT_BLOCK_PTR_FOR_FN (cfun)->count
   10092        30303 :         = EXIT_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale (num, den);
   10093              :     }
   10094              : 
   10095    102742294 :   FOR_EACH_BB_FN (bb, cfun)
   10096              :     {
   10097     91288809 :       if (scale)
   10098      1510328 :         bb->count = bb->count.apply_scale (num, den);
   10099    655910191 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
   10100              :         {
   10101    473332573 :           gimple *stmt = gsi_stmt (gsi);
   10102    473332573 :           tree decl = is_gimple_call (stmt)
   10103    473332573 :                       ? gimple_call_fndecl (stmt)
   10104              :                       : NULL;
   10105     45628323 :           if (decl)
   10106              :             {
   10107     42481848 :               int flags = gimple_call_flags (stmt);
   10108     42481848 :               if (flags & (ECF_CONST | ECF_PURE | ECF_LOOPING_CONST_OR_PURE))
   10109              :                 {
   10110      6949805 :                   if (gimple_in_ssa_p (cfun))
   10111              :                     {
   10112      6245744 :                       todo |= TODO_update_ssa | TODO_cleanup_cfg;
   10113      6245744 :                       update_stmt (stmt);
   10114              :                     }
   10115              :                 }
   10116     42481848 :               if (flags & ECF_NORETURN
   10117     42481848 :                   && fixup_noreturn_call (stmt))
   10118        97918 :                 todo |= TODO_cleanup_cfg;
   10119              :              }
   10120              : 
   10121              :           /* Remove stores to variables we marked write-only. */
   10122    473332573 :           if (maybe_remove_writeonly_store (gsi, stmt, dce_ssa_names))
   10123              :             {
   10124        31491 :               todo |= TODO_update_ssa | TODO_cleanup_cfg;
   10125        31491 :               continue;
   10126              :             }
   10127              : 
   10128              :           /* For calls we can simply remove LHS when it is known
   10129              :              to be write-only.  */
   10130    473301082 :           if (is_gimple_call (stmt)
   10131    473301082 :               && gimple_get_lhs (stmt))
   10132              :             {
   10133     18760806 :               tree lhs = get_base_address (gimple_get_lhs (stmt));
   10134              : 
   10135     18760806 :               if (VAR_P (lhs)
   10136      4911304 :                   && (TREE_STATIC (lhs) || DECL_EXTERNAL (lhs))
   10137     18771447 :                   && varpool_node::get (lhs)->writeonly)
   10138              :                 {
   10139            2 :                   gimple_call_set_lhs (stmt, NULL);
   10140            2 :                   update_stmt (stmt);
   10141            2 :                   todo |= TODO_update_ssa | TODO_cleanup_cfg;
   10142              :                 }
   10143              :             }
   10144              : 
   10145    473301082 :           gsi_next (&gsi);
   10146              :         }
   10147    182577618 :       if (gimple *last = *gsi_last_bb (bb))
   10148              :         {
   10149     85123765 :           if (maybe_clean_eh_stmt (last)
   10150     85123765 :               && gimple_purge_dead_eh_edges (bb))
   10151       234082 :             todo |= TODO_cleanup_cfg;
   10152     85123765 :           if (gimple_purge_dead_abnormal_call_edges (bb))
   10153         1069 :             todo |= TODO_cleanup_cfg;
   10154              :         }
   10155              : 
   10156              :       /* If we have a basic block with no successors that does not
   10157              :          end with a control statement or a noreturn call end it with
   10158              :          a call to __builtin_unreachable.  This situation can occur
   10159              :          when inlining a noreturn call that does in fact return.  */
   10160     91288809 :       if (EDGE_COUNT (bb->succs) == 0)
   10161              :         {
   10162      6680733 :           gimple *stmt = last_nondebug_stmt (bb);
   10163      6680733 :           if (!stmt
   10164      6680733 :               || (!is_ctrl_stmt (stmt)
   10165      5799484 :                   && (!is_gimple_call (stmt)
   10166      5799466 :                       || !gimple_call_noreturn_p (stmt))))
   10167              :             {
   10168          133 :               if (stmt && is_gimple_call (stmt))
   10169          104 :                 gimple_call_set_ctrl_altering (stmt, false);
   10170          133 :               stmt = gimple_build_builtin_unreachable (UNKNOWN_LOCATION);
   10171          133 :               gimple_stmt_iterator gsi = gsi_last_bb (bb);
   10172          133 :               gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
   10173          133 :               if (!cfun->after_inlining)
   10174          109 :                 if (tree fndecl = gimple_call_fndecl (stmt))
   10175              :                   {
   10176          109 :                     gcall *call_stmt = dyn_cast <gcall *> (stmt);
   10177          109 :                     node->create_edge (cgraph_node::get_create (fndecl),
   10178              :                                        call_stmt, bb->count);
   10179              :                   }
   10180              :             }
   10181              :         }
   10182              :     }
   10183     11453485 :   if (scale)
   10184              :     {
   10185        30303 :       update_max_bb_count ();
   10186        30303 :       compute_function_frequency ();
   10187              :     }
   10188              : 
   10189     11453485 :   if (current_loops
   10190     11453485 :       && (todo & TODO_cleanup_cfg))
   10191      2009041 :     loops_state_set (LOOPS_NEED_FIXUP);
   10192              : 
   10193     11453485 :   simple_dce_from_worklist (dce_ssa_names);
   10194              : 
   10195     11453485 :   return todo;
   10196     11453485 : }
   10197              : 
   10198              : namespace {
   10199              : 
   10200              : const pass_data pass_data_fixup_cfg =
   10201              : {
   10202              :   GIMPLE_PASS, /* type */
   10203              :   "fixup_cfg", /* name */
   10204              :   OPTGROUP_NONE, /* optinfo_flags */
   10205              :   TV_NONE, /* tv_id */
   10206              :   PROP_cfg, /* properties_required */
   10207              :   0, /* properties_provided */
   10208              :   0, /* properties_destroyed */
   10209              :   0, /* todo_flags_start */
   10210              :   0, /* todo_flags_finish */
   10211              : };
   10212              : 
   10213              : class pass_fixup_cfg : public gimple_opt_pass
   10214              : {
   10215              : public:
   10216       891540 :   pass_fixup_cfg (gcc::context *ctxt)
   10217      1783080 :     : gimple_opt_pass (pass_data_fixup_cfg, ctxt)
   10218              :   {}
   10219              : 
   10220              :   /* opt_pass methods: */
   10221       594360 :   opt_pass * clone () final override { return new pass_fixup_cfg (m_ctxt); }
   10222      7490635 :   unsigned int execute (function *) final override
   10223              :   {
   10224      7490635 :     return execute_fixup_cfg ();
   10225              :   }
   10226              : 
   10227              : }; // class pass_fixup_cfg
   10228              : 
   10229              : } // anon namespace
   10230              : 
   10231              : gimple_opt_pass *
   10232       297180 : make_pass_fixup_cfg (gcc::context *ctxt)
   10233              : {
   10234       297180 :   return new pass_fixup_cfg (ctxt);
   10235              : }
   10236              : 
   10237              : 
   10238              : /* Sort PHI argument values for make_forwarders_with_degenerate_phis.  */
   10239              : 
   10240              : static int
   10241     46335941 : sort_phi_args (const void *a_, const void *b_)
   10242              : {
   10243     46335941 :   auto *a = (const std::pair<edge, hashval_t> *) a_;
   10244     46335941 :   auto *b = (const std::pair<edge, hashval_t> *) b_;
   10245     46335941 :   hashval_t ha = a->second;
   10246     46335941 :   hashval_t hb = b->second;
   10247     46335941 :   if (ha < hb)
   10248              :     return -1;
   10249     28884610 :   else if (ha > hb)
   10250              :     return 1;
   10251     14127530 :   else if (a->first->dest_idx < b->first->dest_idx)
   10252              :     return -1;
   10253      7367012 :   else if (a->first->dest_idx > b->first->dest_idx)
   10254              :     return 1;
   10255              :   else
   10256            0 :     return 0;
   10257              : }
   10258              : 
   10259              : /* Returns true if edge E comes from a possible ifconvertable branch.
   10260              :    That is:
   10261              :    ```
   10262              :    BB0:
   10263              :    if (a) goto BB1; else goto BB2;
   10264              :    BB1:
   10265              :    BB2:
   10266              :    ```
   10267              :    Returns true for the edge from BB0->BB2 or BB1->BB2.
   10268              :    This function assumes we only have one middle block.
   10269              :    And the middle block is empty. */
   10270              : 
   10271              : static bool
   10272      1231542 : ifconvertable_edge (edge e)
   10273              : {
   10274      1231542 :   basic_block bb2 = e->dest;
   10275      1231542 :   basic_block bb0 = e->src;
   10276      1231542 :   basic_block bb1 = nullptr, rbb1;
   10277      1231542 :   if (e->src == e->dest)
   10278              :     return false;
   10279      1231542 :   if (EDGE_COUNT (bb0->succs) > 2)
   10280              :     return false;
   10281      1224561 :   if (single_succ_p (bb0))
   10282              :     {
   10283       485589 :       if (!single_pred_p (bb0))
   10284              :         return false;
   10285              :       /* The middle block can only be empty,
   10286              :          otherwise the phis will be
   10287              :          different anyways. */
   10288       391759 :       if (!empty_block_p (bb0))
   10289              :         return false;
   10290        53203 :       bb1 = bb0;
   10291        53203 :       bb0 = single_pred (bb0);
   10292        53203 :       if (EDGE_COUNT (bb0->succs) != 2)
   10293              :         return false;
   10294              :     }
   10295              : 
   10296              :   /* If convertibles are only for conditionals. */
   10297       765828 :   if (!is_a<gcond*>(*gsi_last_nondebug_bb (bb0)))
   10298              :     return false;
   10299              : 
   10300              :   /* Find the other basic block.  */
   10301       712574 :   if (EDGE_SUCC (bb0, 0)->dest == bb2)
   10302       343221 :     rbb1 = EDGE_SUCC (bb0, 1)->dest;
   10303       369353 :   else if (EDGE_SUCC (bb0, 1)->dest == bb2)
   10304              :     rbb1 = EDGE_SUCC (bb0, 0)->dest;
   10305              :   else
   10306              :     return false;
   10307              : 
   10308              :   /* If we already know bb1, then just test it. */
   10309       712472 :   if (bb1)
   10310        26753 :     return rbb1 == bb1;
   10311              : 
   10312       896117 :   if (!single_succ_p (rbb1) || !single_pred_p (rbb1))
   10313              :     return false;
   10314              :   /* The middle block can only be empty,
   10315              :      otherwise the phis will be
   10316              :      different anyways. */
   10317       140723 :   if (!empty_block_p (rbb1))
   10318              :     return false;
   10319              : 
   10320        26753 :   return single_succ (rbb1) == bb2;
   10321              : }
   10322              : 
   10323              : /* Look for a non-virtual PHIs and make a forwarder block when all PHIs
   10324              :    have the same argument on a set of edges.  This is to not consider
   10325              :    control dependences of individual edges for same values but only for
   10326              :    the common set.  Returns true if changed the CFG.  */
   10327              : 
   10328              : bool
   10329      4693664 : make_forwarders_with_degenerate_phis (function *fn, bool skip_ifcvtable)
   10330              : {
   10331      4693664 :   bool didsomething = false;
   10332              : 
   10333      4693664 :   basic_block bb;
   10334     44903734 :   FOR_EACH_BB_FN (bb, fn)
   10335              :     {
   10336              :       /* Only PHIs with three or more arguments have opportunities.  */
   10337     40210070 :       if (EDGE_COUNT (bb->preds) < 3)
   10338     39732438 :         continue;
   10339              :       /* Do not touch loop headers or blocks with abnormal predecessors.
   10340              :          ???  This is to avoid creating valid loops here, see PR103458.
   10341              :          We might want to improve things to either explicitly add those
   10342              :          loops or at least consider blocks with no backedges.  */
   10343      1874501 :       if (bb->loop_father->header == bb
   10344      1871068 :           || bb_has_abnormal_pred (bb))
   10345         3433 :         continue;
   10346              : 
   10347              :       /* Take one PHI node as template to look for identical
   10348              :          arguments.  Build a vector of candidates forming sets
   10349              :          of argument edges with equal values.  Note optimality
   10350              :          depends on the particular choice of the template PHI
   10351              :          since equal arguments are unordered leaving other PHIs
   10352              :          with more than one set of equal arguments within this
   10353              :          argument range unsorted.  We'd have to break ties by
   10354              :          looking at other PHI nodes.  */
   10355      1867635 :       gphi_iterator gsi = gsi_start_nonvirtual_phis (bb);
   10356      1867635 :       if (gsi_end_p (gsi))
   10357      1082229 :         continue;
   10358       785406 :       gphi *phi = gsi.phi ();
   10359       785406 :       auto_vec<std::pair<edge, hashval_t>, 8> args;
   10360       785406 :       bool need_resort = false;
   10361      4732386 :       for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
   10362              :         {
   10363      3946980 :           edge e = gimple_phi_arg_edge (phi, i);
   10364              :           /* Skip abnormal edges since we cannot redirect them.  */
   10365      3946980 :           if (e->flags & EDGE_ABNORMAL)
   10366      3946980 :             continue;
   10367              :           /* Skip loop exit edges when we are in loop-closed SSA form
   10368              :              since the forwarder we'd create does not have a PHI node.  */
   10369      3946980 :           if (loops_state_satisfies_p (LOOP_CLOSED_SSA)
   10370      3946980 :               && loop_exit_edge_p (e->src->loop_father, e))
   10371        24873 :             continue;
   10372              : 
   10373              :           /* Skip ifconvertable edges when asked as we want the
   10374              :              copy/constant on that edge still when going out of ssa.
   10375              :              FIXME: phiopt should produce COND_EXPR but there
   10376              :              are regressions with that.  */
   10377      3922107 :           if (skip_ifcvtable && ifconvertable_edge (e))
   10378        53506 :             continue;
   10379              : 
   10380      3868601 :           tree arg = gimple_phi_arg_def (phi, i);
   10381      3868601 :           if (!CONSTANT_CLASS_P (arg) && TREE_CODE (arg) != SSA_NAME)
   10382      3868601 :             need_resort = true;
   10383      3868601 :           args.safe_push (std::make_pair (e, iterative_hash_expr (arg, 0)));
   10384              :         }
   10385       785406 :       if (args.length () < 2)
   10386        13153 :         continue;
   10387       772253 :       args.qsort (sort_phi_args);
   10388              :       /* The above sorting can be different between -g and -g0, as e.g. decls
   10389              :          can have different uids (-g could have bigger gaps in between them).
   10390              :          So, only use that to determine which args are equal, then change
   10391              :          second from hash value to smallest dest_idx of the edges which have
   10392              :          equal argument and sort again.  If all the phi arguments are
   10393              :          constants or SSA_NAME, there is no need for the second sort, the hash
   10394              :          values are stable in that case.  */
   10395       772253 :       hashval_t hash = args[0].second;
   10396       772253 :       args[0].second = args[0].first->dest_idx;
   10397       772253 :       bool any_equal = false;
   10398      3858103 :       for (unsigned i = 1; i < args.length (); ++i)
   10399      3085850 :         if (hash == args[i].second
   10400      4331388 :             && operand_equal_p (PHI_ARG_DEF_FROM_EDGE (phi, args[i - 1].first),
   10401      1245538 :                                 PHI_ARG_DEF_FROM_EDGE (phi, args[i].first)))
   10402              :           {
   10403      1244335 :             args[i].second = args[i - 1].second;
   10404      1244335 :             any_equal = true;
   10405              :           }
   10406              :         else
   10407              :           {
   10408      1841515 :             hash = args[i].second;
   10409      1841515 :             args[i].second = args[i].first->dest_idx;
   10410              :           }
   10411       772253 :       if (!any_equal)
   10412       294621 :         continue;
   10413       477632 :       if (need_resort)
   10414        26876 :         args.qsort (sort_phi_args);
   10415              : 
   10416              :       /* From the candidates vector now verify true candidates for
   10417              :          forwarders and create them.  */
   10418       477632 :       gphi *vphi = get_virtual_phi (bb);
   10419       477632 :       unsigned start = 0;
   10420      3922548 :       while (start < args.length () - 1)
   10421              :         {
   10422      1246897 :           unsigned i;
   10423      3613145 :           for (i = start + 1; i < args.length (); ++i)
   10424      3363046 :             if (args[start].second != args[i].second)
   10425              :               break;
   10426              :           /* args[start]..args[i-1] are equal.  */
   10427      1246897 :           if (start != i - 1)
   10428              :             {
   10429              :               /* Check all PHI nodes for argument equality
   10430              :                  except for vops.  */
   10431       723549 :               bool equal = true;
   10432       723549 :               gphi_iterator gsi2 = gsi;
   10433       723549 :               gsi_next (&gsi2);
   10434      1549189 :               for (; !gsi_end_p (gsi2); gsi_next (&gsi2))
   10435              :                 {
   10436      1061155 :                   gphi *phi2 = gsi2.phi ();
   10437      2122310 :                   if (virtual_operand_p (gimple_phi_result (phi2)))
   10438       257650 :                     continue;
   10439       803505 :                   tree start_arg
   10440       803505 :                     = PHI_ARG_DEF_FROM_EDGE (phi2, args[start].first);
   10441      4101849 :                   for (unsigned j = start + 1; j < i; ++j)
   10442              :                     {
   10443      5567960 :                       if (!operand_equal_p (start_arg,
   10444      2783980 :                                             PHI_ARG_DEF_FROM_EDGE
   10445              :                                               (phi2, args[j].first)))
   10446              :                         {
   10447              :                           /* Another PHI might have a shorter set of
   10448              :                              equivalent args.  Go for that.  */
   10449       289141 :                           i = j;
   10450       289141 :                           if (j == start + 1)
   10451              :                             equal = false;
   10452              :                           break;
   10453              :                         }
   10454              :                     }
   10455              :                   if (!equal)
   10456              :                     break;
   10457              :                 }
   10458       723549 :               if (equal)
   10459              :                 {
   10460              :                   /* If we are asked to forward all edges the block
   10461              :                      has all degenerate PHIs.  Do nothing in that case.  */
   10462       488034 :                   if (start == 0
   10463       230299 :                       && i == args.length ()
   10464       491751 :                       && args.length () == gimple_phi_num_args (phi))
   10465              :                     break;
   10466              :                   /* Instead of using make_forwarder_block we are
   10467              :                      rolling our own variant knowing that the forwarder
   10468              :                      does not need PHI nodes apart from eventually
   10469              :                      a virtual one.  */
   10470       485963 :                   auto_vec<tree, 8> vphi_args;
   10471       485963 :                   if (vphi)
   10472              :                     {
   10473       327298 :                       vphi_args.reserve_exact (i - start);
   10474      1650442 :                       for (unsigned j = start; j < i; ++j)
   10475       995846 :                         vphi_args.quick_push
   10476       995846 :                           (PHI_ARG_DEF_FROM_EDGE (vphi, args[j].first));
   10477              :                     }
   10478       485963 :                   free_dominance_info (fn, CDI_DOMINATORS);
   10479       485963 :                   if (dump_file && (dump_flags & TDF_DETAILS))
   10480              :                     {
   10481            1 :                       fprintf (dump_file, "New forwarder block for edge ");
   10482            1 :                       fprintf (dump_file, "%d -> %d.\n",
   10483            1 :                                args[start].first->src->index,
   10484            1 :                                args[start].first->dest->index);
   10485              :                     }
   10486       485963 :                   basic_block forwarder = split_edge (args[start].first);
   10487       485963 :                   profile_count count = forwarder->count;
   10488       485963 :                   bool irr = false;
   10489      1477776 :                   for (unsigned j = start + 1; j < i; ++j)
   10490              :                     {
   10491       991813 :                       edge e = args[j].first;
   10492       991813 :                       if (e->flags & EDGE_IRREDUCIBLE_LOOP)
   10493         4443 :                         irr = true;
   10494       991813 :                       redirect_edge_and_branch_force (e, forwarder);
   10495       991813 :                       redirect_edge_var_map_clear (e);
   10496       991813 :                       count += e->count ();
   10497              :                     }
   10498       485963 :                   forwarder->count = count;
   10499       485963 :                   if (irr)
   10500              :                     {
   10501         2916 :                       forwarder->flags |= BB_IRREDUCIBLE_LOOP;
   10502         2916 :                       single_succ_edge (forwarder)->flags
   10503         2916 :                         |= EDGE_IRREDUCIBLE_LOOP;
   10504              :                     }
   10505              : 
   10506       485963 :                   if (vphi)
   10507              :                     {
   10508       327298 :                       tree def = copy_ssa_name (vphi_args[0]);
   10509       327298 :                       gphi *vphi_copy = create_phi_node (def, forwarder);
   10510      1650442 :                       for (unsigned j = start; j < i; ++j)
   10511      1991692 :                         add_phi_arg (vphi_copy, vphi_args[j - start],
   10512       995846 :                                      args[j].first, UNKNOWN_LOCATION);
   10513       327298 :                       SET_PHI_ARG_DEF
   10514              :                         (vphi, single_succ_edge (forwarder)->dest_idx, def);
   10515              :                     }
   10516       485963 :                   didsomething = true;
   10517       485963 :                 }
   10518              :             }
   10519              :           /* Continue searching for more opportunities.  */
   10520      1244826 :           start = i;
   10521              :         }
   10522       785406 :     }
   10523      4693664 :   return didsomething;
   10524              : }
   10525              : 
   10526              : /* Garbage collection support for edge_def.  */
   10527              : 
   10528              : extern void gt_ggc_mx (tree&);
   10529              : extern void gt_ggc_mx (gimple *&);
   10530              : extern void gt_ggc_mx (rtx&);
   10531              : extern void gt_ggc_mx (basic_block&);
   10532              : 
   10533              : static void
   10534     50889720 : gt_ggc_mx (rtx_insn *& x)
   10535              : {
   10536            0 :   if (x)
   10537            0 :     gt_ggc_mx_rtx_def ((void *) x);
   10538            0 : }
   10539              : 
   10540              : void
   10541    169243829 : gt_ggc_mx (edge_def *e)
   10542              : {
   10543    169243829 :   tree block = LOCATION_BLOCK (e->goto_locus);
   10544    169243829 :   gt_ggc_mx (e->src);
   10545    169243829 :   gt_ggc_mx (e->dest);
   10546    169243829 :   if (current_ir_type () == IR_GIMPLE)
   10547    118354109 :     gt_ggc_mx (e->insns.g);
   10548              :   else
   10549     50889720 :     gt_ggc_mx (e->insns.r);
   10550    169243829 :   gt_ggc_mx (block);
   10551    169243829 : }
   10552              : 
   10553              : /* PCH support for edge_def.  */
   10554              : 
   10555              : extern void gt_pch_nx (tree&);
   10556              : extern void gt_pch_nx (gimple *&);
   10557              : extern void gt_pch_nx (rtx&);
   10558              : extern void gt_pch_nx (basic_block&);
   10559              : 
   10560              : static void
   10561            0 : gt_pch_nx (rtx_insn *& x)
   10562              : {
   10563            0 :   if (x)
   10564            0 :     gt_pch_nx_rtx_def ((void *) x);
   10565            0 : }
   10566              : 
   10567              : void
   10568            0 : gt_pch_nx (edge_def *e)
   10569              : {
   10570            0 :   tree block = LOCATION_BLOCK (e->goto_locus);
   10571            0 :   gt_pch_nx (e->src);
   10572            0 :   gt_pch_nx (e->dest);
   10573            0 :   if (current_ir_type () == IR_GIMPLE)
   10574            0 :     gt_pch_nx (e->insns.g);
   10575              :   else
   10576            0 :     gt_pch_nx (e->insns.r);
   10577            0 :   gt_pch_nx (block);
   10578            0 : }
   10579              : 
   10580              : void
   10581            0 : gt_pch_nx (edge_def *e, gt_pointer_operator op, void *cookie)
   10582              : {
   10583            0 :   tree block = LOCATION_BLOCK (e->goto_locus);
   10584            0 :   op (&(e->src), NULL, cookie);
   10585            0 :   op (&(e->dest), NULL, cookie);
   10586            0 :   if (current_ir_type () == IR_GIMPLE)
   10587            0 :     op (&(e->insns.g), NULL, cookie);
   10588              :   else
   10589            0 :     op (&(e->insns.r), NULL, cookie);
   10590            0 :   op (&(block), &(block), cookie);
   10591            0 : }
   10592              : 
   10593              : #if CHECKING_P
   10594              : 
   10595              : namespace selftest {
   10596              : 
   10597              : /* Helper function for CFG selftests: create a dummy function decl
   10598              :    and push it as cfun.  */
   10599              : 
   10600              : static tree
   10601           12 : push_fndecl (const char *name)
   10602              : {
   10603           12 :   tree fn_type = build_function_type_array (integer_type_node, 0, NULL);
   10604              :   /* FIXME: this uses input_location: */
   10605           12 :   tree fndecl = build_fn_decl (name, fn_type);
   10606           12 :   tree retval = build_decl (UNKNOWN_LOCATION, RESULT_DECL,
   10607              :                             NULL_TREE, integer_type_node);
   10608           12 :   DECL_RESULT (fndecl) = retval;
   10609           12 :   push_struct_function (fndecl);
   10610           12 :   function *fun = DECL_STRUCT_FUNCTION (fndecl);
   10611           12 :   ASSERT_TRUE (fun != NULL);
   10612           12 :   init_empty_tree_cfg_for_function (fun);
   10613           12 :   ASSERT_EQ (2, n_basic_blocks_for_fn (fun));
   10614           12 :   ASSERT_EQ (0, n_edges_for_fn (fun));
   10615           12 :   return fndecl;
   10616              : }
   10617              : 
   10618              : /* These tests directly create CFGs.
   10619              :    Compare with the static fns within tree-cfg.cc:
   10620              :      - build_gimple_cfg
   10621              :      - make_blocks: calls create_basic_block (seq, bb);
   10622              :      - make_edges.   */
   10623              : 
   10624              : /* Verify a simple cfg of the form:
   10625              :      ENTRY -> A -> B -> C -> EXIT.  */
   10626              : 
   10627              : static void
   10628            4 : test_linear_chain ()
   10629              : {
   10630            4 :   gimple_register_cfg_hooks ();
   10631              : 
   10632            4 :   tree fndecl = push_fndecl ("cfg_test_linear_chain");
   10633            4 :   function *fun = DECL_STRUCT_FUNCTION (fndecl);
   10634              : 
   10635              :   /* Create some empty blocks.  */
   10636            4 :   basic_block bb_a = create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
   10637            4 :   basic_block bb_b = create_empty_bb (bb_a);
   10638            4 :   basic_block bb_c = create_empty_bb (bb_b);
   10639              : 
   10640            4 :   ASSERT_EQ (5, n_basic_blocks_for_fn (fun));
   10641            4 :   ASSERT_EQ (0, n_edges_for_fn (fun));
   10642              : 
   10643              :   /* Create some edges: a simple linear chain of BBs.  */
   10644            4 :   make_edge (ENTRY_BLOCK_PTR_FOR_FN (fun), bb_a, EDGE_FALLTHRU);
   10645            4 :   make_edge (bb_a, bb_b, 0);
   10646            4 :   make_edge (bb_b, bb_c, 0);
   10647            4 :   make_edge (bb_c, EXIT_BLOCK_PTR_FOR_FN (fun), 0);
   10648              : 
   10649              :   /* Verify the edges.  */
   10650            4 :   ASSERT_EQ (4, n_edges_for_fn (fun));
   10651            4 :   ASSERT_EQ (NULL, ENTRY_BLOCK_PTR_FOR_FN (fun)->preds);
   10652            4 :   ASSERT_EQ (1, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs->length ());
   10653            4 :   ASSERT_EQ (1, bb_a->preds->length ());
   10654            4 :   ASSERT_EQ (1, bb_a->succs->length ());
   10655            4 :   ASSERT_EQ (1, bb_b->preds->length ());
   10656            4 :   ASSERT_EQ (1, bb_b->succs->length ());
   10657            4 :   ASSERT_EQ (1, bb_c->preds->length ());
   10658            4 :   ASSERT_EQ (1, bb_c->succs->length ());
   10659            4 :   ASSERT_EQ (1, EXIT_BLOCK_PTR_FOR_FN (fun)->preds->length ());
   10660            4 :   ASSERT_EQ (NULL, EXIT_BLOCK_PTR_FOR_FN (fun)->succs);
   10661              : 
   10662              :   /* Verify the dominance information
   10663              :      Each BB in our simple chain should be dominated by the one before
   10664              :      it.  */
   10665            4 :   calculate_dominance_info (CDI_DOMINATORS);
   10666            4 :   ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_b));
   10667            4 :   ASSERT_EQ (bb_b, get_immediate_dominator (CDI_DOMINATORS, bb_c));
   10668            4 :   auto_vec<basic_block> dom_by_b = get_dominated_by (CDI_DOMINATORS, bb_b);
   10669            4 :   ASSERT_EQ (1, dom_by_b.length ());
   10670            4 :   ASSERT_EQ (bb_c, dom_by_b[0]);
   10671            4 :   free_dominance_info (CDI_DOMINATORS);
   10672              : 
   10673              :   /* Similarly for post-dominance: each BB in our chain is post-dominated
   10674              :      by the one after it.  */
   10675            4 :   calculate_dominance_info (CDI_POST_DOMINATORS);
   10676            4 :   ASSERT_EQ (bb_b, get_immediate_dominator (CDI_POST_DOMINATORS, bb_a));
   10677            4 :   ASSERT_EQ (bb_c, get_immediate_dominator (CDI_POST_DOMINATORS, bb_b));
   10678            4 :   auto_vec<basic_block> postdom_by_b = get_dominated_by (CDI_POST_DOMINATORS, bb_b);
   10679            4 :   ASSERT_EQ (1, postdom_by_b.length ());
   10680            4 :   ASSERT_EQ (bb_a, postdom_by_b[0]);
   10681            4 :   free_dominance_info (CDI_POST_DOMINATORS);
   10682              : 
   10683            4 :   pop_cfun ();
   10684            4 : }
   10685              : 
   10686              : /* Verify a simple CFG of the form:
   10687              :      ENTRY
   10688              :        |
   10689              :        A
   10690              :       / \
   10691              :      /t  \f
   10692              :     B     C
   10693              :      \   /
   10694              :       \ /
   10695              :        D
   10696              :        |
   10697              :       EXIT.  */
   10698              : 
   10699              : static void
   10700            4 : test_diamond ()
   10701              : {
   10702            4 :   gimple_register_cfg_hooks ();
   10703              : 
   10704            4 :   tree fndecl = push_fndecl ("cfg_test_diamond");
   10705            4 :   function *fun = DECL_STRUCT_FUNCTION (fndecl);
   10706              : 
   10707              :   /* Create some empty blocks.  */
   10708            4 :   basic_block bb_a = create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
   10709            4 :   basic_block bb_b = create_empty_bb (bb_a);
   10710            4 :   basic_block bb_c = create_empty_bb (bb_a);
   10711            4 :   basic_block bb_d = create_empty_bb (bb_b);
   10712              : 
   10713            4 :   ASSERT_EQ (6, n_basic_blocks_for_fn (fun));
   10714            4 :   ASSERT_EQ (0, n_edges_for_fn (fun));
   10715              : 
   10716              :   /* Create the edges.  */
   10717            4 :   make_edge (ENTRY_BLOCK_PTR_FOR_FN (fun), bb_a, EDGE_FALLTHRU);
   10718            4 :   make_edge (bb_a, bb_b, EDGE_TRUE_VALUE);
   10719            4 :   make_edge (bb_a, bb_c, EDGE_FALSE_VALUE);
   10720            4 :   make_edge (bb_b, bb_d, 0);
   10721            4 :   make_edge (bb_c, bb_d, 0);
   10722            4 :   make_edge (bb_d, EXIT_BLOCK_PTR_FOR_FN (fun), 0);
   10723              : 
   10724              :   /* Verify the edges.  */
   10725            4 :   ASSERT_EQ (6, n_edges_for_fn (fun));
   10726            4 :   ASSERT_EQ (1, bb_a->preds->length ());
   10727            4 :   ASSERT_EQ (2, bb_a->succs->length ());
   10728            4 :   ASSERT_EQ (1, bb_b->preds->length ());
   10729            4 :   ASSERT_EQ (1, bb_b->succs->length ());
   10730            4 :   ASSERT_EQ (1, bb_c->preds->length ());
   10731            4 :   ASSERT_EQ (1, bb_c->succs->length ());
   10732            4 :   ASSERT_EQ (2, bb_d->preds->length ());
   10733            4 :   ASSERT_EQ (1, bb_d->succs->length ());
   10734              : 
   10735              :   /* Verify the dominance information.  */
   10736            4 :   calculate_dominance_info (CDI_DOMINATORS);
   10737            4 :   ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_b));
   10738            4 :   ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_c));
   10739            4 :   ASSERT_EQ (bb_a, get_immediate_dominator (CDI_DOMINATORS, bb_d));
   10740            4 :   auto_vec<basic_block> dom_by_a = get_dominated_by (CDI_DOMINATORS, bb_a);
   10741            4 :   ASSERT_EQ (3, dom_by_a.length ()); /* B, C, D, in some order.  */
   10742            4 :   dom_by_a.release ();
   10743            4 :   auto_vec<basic_block> dom_by_b = get_dominated_by (CDI_DOMINATORS, bb_b);
   10744            4 :   ASSERT_EQ (0, dom_by_b.length ());
   10745            4 :   dom_by_b.release ();
   10746            4 :   free_dominance_info (CDI_DOMINATORS);
   10747              : 
   10748              :   /* Similarly for post-dominance.  */
   10749            4 :   calculate_dominance_info (CDI_POST_DOMINATORS);
   10750            4 :   ASSERT_EQ (bb_d, get_immediate_dominator (CDI_POST_DOMINATORS, bb_a));
   10751            4 :   ASSERT_EQ (bb_d, get_immediate_dominator (CDI_POST_DOMINATORS, bb_b));
   10752            4 :   ASSERT_EQ (bb_d, get_immediate_dominator (CDI_POST_DOMINATORS, bb_c));
   10753            4 :   auto_vec<basic_block> postdom_by_d = get_dominated_by (CDI_POST_DOMINATORS, bb_d);
   10754            4 :   ASSERT_EQ (3, postdom_by_d.length ()); /* A, B, C in some order.  */
   10755            4 :   postdom_by_d.release ();
   10756            4 :   auto_vec<basic_block> postdom_by_b = get_dominated_by (CDI_POST_DOMINATORS, bb_b);
   10757            4 :   ASSERT_EQ (0, postdom_by_b.length ());
   10758            4 :   postdom_by_b.release ();
   10759            4 :   free_dominance_info (CDI_POST_DOMINATORS);
   10760              : 
   10761            4 :   pop_cfun ();
   10762            4 : }
   10763              : 
   10764              : /* Verify that we can handle a CFG containing a "complete" aka
   10765              :    fully-connected subgraph (where A B C D below all have edges
   10766              :    pointing to each other node, also to themselves).
   10767              :    e.g.:
   10768              :      ENTRY  EXIT
   10769              :        |    ^
   10770              :        |   /
   10771              :        |  /
   10772              :        | /
   10773              :        V/
   10774              :        A<--->B
   10775              :        ^^   ^^
   10776              :        | \ / |
   10777              :        |  X  |
   10778              :        | / \ |
   10779              :        VV   VV
   10780              :        C<--->D
   10781              : */
   10782              : 
   10783              : static void
   10784            4 : test_fully_connected ()
   10785              : {
   10786            4 :   gimple_register_cfg_hooks ();
   10787              : 
   10788            4 :   tree fndecl = push_fndecl ("cfg_fully_connected");
   10789            4 :   function *fun = DECL_STRUCT_FUNCTION (fndecl);
   10790              : 
   10791            4 :   const int n = 4;
   10792              : 
   10793              :   /* Create some empty blocks.  */
   10794            4 :   auto_vec <basic_block> subgraph_nodes;
   10795           20 :   for (int i = 0; i < n; i++)
   10796           16 :     subgraph_nodes.safe_push (create_empty_bb (ENTRY_BLOCK_PTR_FOR_FN (fun)));
   10797              : 
   10798            4 :   ASSERT_EQ (n + 2, n_basic_blocks_for_fn (fun));
   10799            4 :   ASSERT_EQ (0, n_edges_for_fn (fun));
   10800              : 
   10801              :   /* Create the edges.  */
   10802            4 :   make_edge (ENTRY_BLOCK_PTR_FOR_FN (fun), subgraph_nodes[0], EDGE_FALLTHRU);
   10803            4 :   make_edge (subgraph_nodes[0], EXIT_BLOCK_PTR_FOR_FN (fun), 0);
   10804           24 :   for (int i = 0; i < n; i++)
   10805           80 :     for (int j = 0; j < n; j++)
   10806           64 :       make_edge (subgraph_nodes[i], subgraph_nodes[j], 0);
   10807              : 
   10808              :   /* Verify the edges.  */
   10809            4 :   ASSERT_EQ (2 + (n * n), n_edges_for_fn (fun));
   10810              :   /* The first one is linked to ENTRY/EXIT as well as itself and
   10811              :      everything else.  */
   10812            4 :   ASSERT_EQ (n + 1, subgraph_nodes[0]->preds->length ());
   10813            4 :   ASSERT_EQ (n + 1, subgraph_nodes[0]->succs->length ());
   10814              :   /* The other ones in the subgraph are linked to everything in
   10815              :      the subgraph (including themselves).  */
   10816           16 :   for (int i = 1; i < n; i++)
   10817              :     {
   10818           12 :       ASSERT_EQ (n, subgraph_nodes[i]->preds->length ());
   10819           12 :       ASSERT_EQ (n, subgraph_nodes[i]->succs->length ());
   10820              :     }
   10821              : 
   10822              :   /* Verify the dominance information.  */
   10823            4 :   calculate_dominance_info (CDI_DOMINATORS);
   10824              :   /* The initial block in the subgraph should be dominated by ENTRY.  */
   10825            4 :   ASSERT_EQ (ENTRY_BLOCK_PTR_FOR_FN (fun),
   10826              :              get_immediate_dominator (CDI_DOMINATORS,
   10827              :                                       subgraph_nodes[0]));
   10828              :   /* Every other block in the subgraph should be dominated by the
   10829              :      initial block.  */
   10830           16 :   for (int i = 1; i < n; i++)
   10831           12 :     ASSERT_EQ (subgraph_nodes[0],
   10832              :                get_immediate_dominator (CDI_DOMINATORS,
   10833              :                                         subgraph_nodes[i]));
   10834            4 :   free_dominance_info (CDI_DOMINATORS);
   10835              : 
   10836              :   /* Similarly for post-dominance.  */
   10837            4 :   calculate_dominance_info (CDI_POST_DOMINATORS);
   10838              :   /* The initial block in the subgraph should be postdominated by EXIT.  */
   10839            4 :   ASSERT_EQ (EXIT_BLOCK_PTR_FOR_FN (fun),
   10840              :              get_immediate_dominator (CDI_POST_DOMINATORS,
   10841              :                                       subgraph_nodes[0]));
   10842              :   /* Every other block in the subgraph should be postdominated by the
   10843              :      initial block, since that leads to EXIT.  */
   10844           16 :   for (int i = 1; i < n; i++)
   10845           12 :     ASSERT_EQ (subgraph_nodes[0],
   10846              :                get_immediate_dominator (CDI_POST_DOMINATORS,
   10847              :                                         subgraph_nodes[i]));
   10848            4 :   free_dominance_info (CDI_POST_DOMINATORS);
   10849              : 
   10850            4 :   pop_cfun ();
   10851            4 : }
   10852              : 
   10853              : /* Run all of the selftests within this file.  */
   10854              : 
   10855              : void
   10856            4 : tree_cfg_cc_tests ()
   10857              : {
   10858            4 :   test_linear_chain ();
   10859            4 :   test_diamond ();
   10860            4 :   test_fully_connected ();
   10861            4 : }
   10862              : 
   10863              : } // namespace selftest
   10864              : 
   10865              : /* TODO: test the dominator/postdominator logic with various graphs/nodes:
   10866              :    - loop
   10867              :    - nested loops
   10868              :    - switch statement (a block with many out-edges)
   10869              :    - something that jumps to itself
   10870              :    - etc  */
   10871              : 
   10872              : #endif /* CHECKING_P */
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.