LCOV - code coverage report
Current view: top level - gcc - tree-ssa-reassoc.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 92.4 % 3646 3368
Test Date: 2026-11-14 16:39:50 Functions: 95.1 % 103 98
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Reassociation for trees.
       2              :    Copyright (C) 2005-2026 Free Software Foundation, Inc.
       3              :    Contributed by Daniel Berlin <dan@dberlin.org>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify
       8              : it under the terms of the GNU General Public License as published by
       9              : the Free Software Foundation; either version 3, or (at your option)
      10              : any later version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful,
      13              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      14              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15              : GNU General Public License for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "target.h"
      26              : #include "rtl.h"
      27              : #include "tree.h"
      28              : #include "gimple.h"
      29              : #include "cfghooks.h"
      30              : #include "alloc-pool.h"
      31              : #include "tree-pass.h"
      32              : #include "memmodel.h"
      33              : #include "tm_p.h"
      34              : #include "ssa.h"
      35              : #include "optabs-tree.h"
      36              : #include "gimple-pretty-print.h"
      37              : #include "diagnostic-core.h"
      38              : #include "fold-const.h"
      39              : #include "stor-layout.h"
      40              : #include "cfganal.h"
      41              : #include "gimple-iterator.h"
      42              : #include "gimple-fold.h"
      43              : #include "tree-eh.h"
      44              : #include "gimplify-me.h"
      45              : #include "tree-cfg.h"
      46              : #include "tree-ssa-loop.h"
      47              : #include "flags.h"
      48              : #include "tree-ssa.h"
      49              : #include "langhooks.h"
      50              : #include "cfgloop.h"
      51              : #include "builtins.h"
      52              : #include "gimplify.h"
      53              : #include "case-cfn-macros.h"
      54              : #include "tree-ssa-reassoc.h"
      55              : #include "tree-ssa-math-opts.h"
      56              : #include "gimple-range.h"
      57              : #include "internal-fn.h"
      58              : 
      59              : /*  This is a simple global reassociation pass.  It is, in part, based
      60              :     on the LLVM pass of the same name (They do some things more/less
      61              :     than we do, in different orders, etc).
      62              : 
      63              :     It consists of five steps:
      64              : 
      65              :     1. Breaking up subtract operations into addition + negate, where
      66              :     it would promote the reassociation of adds.
      67              : 
      68              :     2. Left linearization of the expression trees, so that (A+B)+(C+D)
      69              :     becomes (((A+B)+C)+D), which is easier for us to rewrite later.
      70              :     During linearization, we place the operands of the binary
      71              :     expressions into a vector of operand_entry_*
      72              : 
      73              :     3. Optimization of the operand lists, eliminating things like a +
      74              :     -a, a & a, etc.
      75              : 
      76              :     3a. Combine repeated factors with the same occurrence counts
      77              :     into a __builtin_powi call that will later be optimized into
      78              :     an optimal number of multiplies.
      79              : 
      80              :     4. Rewrite the expression trees we linearized and optimized so
      81              :     they are in proper rank order.
      82              : 
      83              :     5. Repropagate negates, as nothing else will clean it up ATM.
      84              : 
      85              :     A bit of theory on #4, since nobody seems to write anything down
      86              :     about why it makes sense to do it the way they do it:
      87              : 
      88              :     We could do this much nicer theoretically, but don't (for reasons
      89              :     explained after how to do it theoretically nice :P).
      90              : 
      91              :     In order to promote the most redundancy elimination, you want
      92              :     binary expressions whose operands are the same rank (or
      93              :     preferably, the same value) exposed to the redundancy eliminator,
      94              :     for possible elimination.
      95              : 
      96              :     So the way to do this if we really cared, is to build the new op
      97              :     tree from the leaves to the roots, merging as you go, and putting the
      98              :     new op on the end of the worklist, until you are left with one
      99              :     thing on the worklist.
     100              : 
     101              :     IE if you have to rewrite the following set of operands (listed with
     102              :     rank in parentheses), with opcode PLUS_EXPR:
     103              : 
     104              :     a (1),  b (1),  c (1),  d (2), e (2)
     105              : 
     106              : 
     107              :     We start with our merge worklist empty, and the ops list with all of
     108              :     those on it.
     109              : 
     110              :     You want to first merge all leaves of the same rank, as much as
     111              :     possible.
     112              : 
     113              :     So first build a binary op of
     114              : 
     115              :     mergetmp = a + b, and put "mergetmp" on the merge worklist.
     116              : 
     117              :     Because there is no three operand form of PLUS_EXPR, c is not going to
     118              :     be exposed to redundancy elimination as a rank 1 operand.
     119              : 
     120              :     So you might as well throw it on the merge worklist (you could also
     121              :     consider it to now be a rank two operand, and merge it with d and e,
     122              :     but in this case, you then have evicted e from a binary op. So at
     123              :     least in this situation, you can't win.)
     124              : 
     125              :     Then build a binary op of d + e
     126              :     mergetmp2 = d + e
     127              : 
     128              :     and put mergetmp2 on the merge worklist.
     129              : 
     130              :     so merge worklist = {mergetmp, c, mergetmp2}
     131              : 
     132              :     Continue building binary ops of these operations until you have only
     133              :     one operation left on the worklist.
     134              : 
     135              :     So we have
     136              : 
     137              :     build binary op
     138              :     mergetmp3 = mergetmp + c
     139              : 
     140              :     worklist = {mergetmp2, mergetmp3}
     141              : 
     142              :     mergetmp4 = mergetmp2 + mergetmp3
     143              : 
     144              :     worklist = {mergetmp4}
     145              : 
     146              :     because we have one operation left, we can now just set the original
     147              :     statement equal to the result of that operation.
     148              : 
     149              :     This will at least expose a + b  and d + e to redundancy elimination
     150              :     as binary operations.
     151              : 
     152              :     For extra points, you can reuse the old statements to build the
     153              :     mergetmps, since you shouldn't run out.
     154              : 
     155              :     So why don't we do this?
     156              : 
     157              :     Because it's expensive, and rarely will help.  Most trees we are
     158              :     reassociating have 3 or less ops.  If they have 2 ops, they already
     159              :     will be written into a nice single binary op.  If you have 3 ops, a
     160              :     single simple check suffices to tell you whether the first two are of the
     161              :     same rank.  If so, you know to order it
     162              : 
     163              :     mergetmp = op1 + op2
     164              :     newstmt = mergetmp + op3
     165              : 
     166              :     instead of
     167              :     mergetmp = op2 + op3
     168              :     newstmt = mergetmp + op1
     169              : 
     170              :     If all three are of the same rank, you can't expose them all in a
     171              :     single binary operator anyway, so the above is *still* the best you
     172              :     can do.
     173              : 
     174              :     Thus, this is what we do.  When we have three ops left, we check to see
     175              :     what order to put them in, and call it a day.  As a nod to vector sum
     176              :     reduction, we check if any of the ops are really a phi node that is a
     177              :     destructive update for the associating op, and keep the destructive
     178              :     update together for vector sum reduction recognition.  */
     179              : 
     180              : /* Enable insertion of __builtin_powi calls during execute_reassoc.  See
     181              :    point 3a in the pass header comment.  */
     182              : static bool reassoc_insert_powi_p;
     183              : 
     184              : /* Enable biasing ranks of loop accumulators.  We don't want this before
     185              :    vectorization, since it interferes with reduction chains.  */
     186              : static bool reassoc_bias_loop_carried_phi_ranks_p;
     187              : 
     188              : /* Statistics */
     189              : static struct
     190              : {
     191              :   int linearized;
     192              :   int constants_eliminated;
     193              :   int ops_eliminated;
     194              :   int rewritten;
     195              :   int pows_encountered;
     196              :   int pows_created;
     197              : } reassociate_stats;
     198              : 
     199              : 
     200              : static object_allocator<operand_entry> operand_entry_pool
     201              :   ("operand entry pool");
     202              : 
     203              : /* This is used to assign a unique ID to each struct operand_entry
     204              :    so that qsort results are identical on different hosts.  */
     205              : static unsigned int next_operand_entry_id;
     206              : 
     207              : /* Starting rank number for a given basic block, so that we can rank
     208              :    operations using unmovable instructions in that BB based on the bb
     209              :    depth.  */
     210              : static int64_t *bb_rank;
     211              : 
     212              : /* Operand->rank hashtable.  */
     213              : static hash_map<tree, int64_t> *operand_rank;
     214              : 
     215              : /* SSA_NAMEs that are forms of loop accumulators and whose ranks need to be
     216              :    biased.  */
     217              : static auto_bitmap biased_names;
     218              : 
     219              : /* Vector of SSA_NAMEs on which after reassociate_bb is done with
     220              :    all basic blocks the CFG should be adjusted - basic blocks
     221              :    split right after that SSA_NAME's definition statement and before
     222              :    the only use, which must be a bit ior.  */
     223              : static vec<tree> reassoc_branch_fixups;
     224              : 
     225              : /* Forward decls.  */
     226              : static int64_t get_rank (tree);
     227              : static bool reassoc_stmt_dominates_stmt_p (gimple *, gimple *);
     228              : 
     229              : /* Wrapper around gsi_remove, which adjusts gimple_uid of debug stmts
     230              :    possibly added by gsi_remove.  */
     231              : 
     232              : static bool
     233       168832 : reassoc_remove_stmt (gimple_stmt_iterator *gsi)
     234              : {
     235       168832 :   gimple *stmt = gsi_stmt (*gsi);
     236              : 
     237       168832 :   if (!MAY_HAVE_DEBUG_BIND_STMTS || gimple_code (stmt) == GIMPLE_PHI)
     238        68701 :     return gsi_remove (gsi, true);
     239              : 
     240       100131 :   gimple_stmt_iterator prev = *gsi;
     241       100131 :   gsi_prev (&prev);
     242       100131 :   unsigned uid = gimple_uid (stmt);
     243       100131 :   basic_block bb = gimple_bb (stmt);
     244       100131 :   bool ret = gsi_remove (gsi, true);
     245       100131 :   if (!gsi_end_p (prev))
     246        99921 :     gsi_next (&prev);
     247              :   else
     248          420 :     prev = gsi_start_bb (bb);
     249       100131 :   gimple *end_stmt = gsi_stmt (*gsi);
     250       104808 :   while ((stmt = gsi_stmt (prev)) != end_stmt)
     251              :     {
     252         4677 :       gcc_assert (stmt && is_gimple_debug (stmt) && gimple_uid (stmt) == 0);
     253         4677 :       gimple_set_uid (stmt, uid);
     254         4677 :       gsi_next (&prev);
     255              :     }
     256              :   return ret;
     257              : }
     258              : 
     259              : /* Bias amount for loop-carried phis.  We want this to be larger than
     260              :    the depth of any reassociation tree we can see, but not larger than
     261              :    the rank difference between two blocks.  */
     262              : #define PHI_LOOP_BIAS (1 << 15)
     263              : 
     264              : /* Return TRUE iff PHI_LOOP_BIAS should be propagated from one of the STMT's
     265              :    operands to the STMT's left-hand side.  The goal is to preserve bias in code
     266              :    like this:
     267              : 
     268              :      x_1 = phi(x_0, x_2)
     269              :      a = x_1 | 1
     270              :      b = a ^ 2
     271              :      .MEM = b
     272              :      c = b + d
     273              :      x_2 = c + e
     274              : 
     275              :    That is, we need to preserve bias along single-use chains originating from
     276              :    loop-carried phis.  Only GIMPLE_ASSIGNs to SSA_NAMEs are considered to be
     277              :    uses, because only they participate in rank propagation.  */
     278              : static bool
     279      7156496 : propagate_bias_p (gimple *stmt)
     280              : {
     281      7156496 :   use_operand_p use;
     282      7156496 :   imm_use_iterator use_iter;
     283      7156496 :   gimple *single_use_stmt = NULL;
     284              : 
     285      7156496 :   if (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) == tcc_reference)
     286              :     return false;
     287              : 
     288     12790609 :   FOR_EACH_IMM_USE_FAST (use, use_iter, gimple_assign_lhs (stmt))
     289              :     {
     290      8140029 :       gimple *current_use_stmt = USE_STMT (use);
     291              : 
     292      8140029 :       if (is_gimple_assign (current_use_stmt)
     293      8140029 :           && TREE_CODE (gimple_assign_lhs (current_use_stmt)) == SSA_NAME)
     294              :         {
     295      6139872 :           if (single_use_stmt != NULL && single_use_stmt != current_use_stmt)
     296       736480 :             return false;
     297              :           single_use_stmt = current_use_stmt;
     298              :         }
     299       736480 :     }
     300              : 
     301      4650580 :   if (single_use_stmt == NULL)
     302              :     return false;
     303              : 
     304      4650159 :   if (gimple_bb (stmt)->loop_father
     305      4650159 :       != gimple_bb (single_use_stmt)->loop_father)
     306       116579 :     return false;
     307              : 
     308              :   return true;
     309              : }
     310              : 
     311              : /* Rank assigned to a phi statement.  If STMT is a loop-carried phi of
     312              :    an innermost loop, and the phi has only a single use which is inside
     313              :    the loop, then the rank is the block rank of the loop latch plus an
     314              :    extra bias for the loop-carried dependence.  This causes expressions
     315              :    calculated into an accumulator variable to be independent for each
     316              :    iteration of the loop.  If STMT is some other phi, the rank is the
     317              :    block rank of its containing block.  */
     318              : static int64_t
     319      1491525 : phi_rank (gimple *stmt)
     320              : {
     321      1491525 :   basic_block bb = gimple_bb (stmt);
     322      1491525 :   class loop *father = bb->loop_father;
     323      1491525 :   tree res;
     324      1491525 :   unsigned i;
     325      1491525 :   use_operand_p use;
     326      1491525 :   gimple *use_stmt;
     327              : 
     328      1491525 :   if (!reassoc_bias_loop_carried_phi_ranks_p)
     329       567018 :     return bb_rank[bb->index];
     330              : 
     331              :   /* We only care about real loops (those with a latch).  */
     332       924507 :   if (!father->latch)
     333            1 :     return bb_rank[bb->index];
     334              : 
     335              :   /* Interesting phis must be in headers of innermost loops.  */
     336       924506 :   if (bb != father->header
     337       722068 :       || father->inner)
     338       368297 :     return bb_rank[bb->index];
     339              : 
     340              :   /* Ignore virtual SSA_NAMEs.  */
     341       556209 :   res = gimple_phi_result (stmt);
     342      1112418 :   if (virtual_operand_p (res))
     343            0 :     return bb_rank[bb->index];
     344              : 
     345              :   /* The phi definition must have a single use, and that use must be
     346              :      within the loop.  Otherwise this isn't an accumulator pattern.  */
     347       556209 :   if (!single_imm_use (res, &use, &use_stmt)
     348       556209 :       || gimple_bb (use_stmt)->loop_father != father)
     349       484716 :     return bb_rank[bb->index];
     350              : 
     351              :   /* Look for phi arguments from within the loop.  If found, bias this phi.  */
     352        81494 :   for (i = 0; i < gimple_phi_num_args (stmt); i++)
     353              :     {
     354        81221 :       tree arg = gimple_phi_arg_def (stmt, i);
     355        81221 :       if (TREE_CODE (arg) == SSA_NAME
     356        81221 :           && !SSA_NAME_IS_DEFAULT_DEF (arg))
     357              :         {
     358        75855 :           gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
     359        75855 :           if (gimple_bb (def_stmt)->loop_father == father)
     360        71220 :             return bb_rank[father->latch->index] + PHI_LOOP_BIAS;
     361              :         }
     362              :     }
     363              : 
     364              :   /* Must be an uninteresting phi.  */
     365          273 :   return bb_rank[bb->index];
     366              : }
     367              : 
     368              : /* Return the maximum of RANK and the rank that should be propagated
     369              :    from expression OP.  For most operands, this is just the rank of OP.
     370              :    For loop-carried phis, the value is zero to avoid undoing the bias
     371              :    in favor of the phi.  */
     372              : static int64_t
     373      7792331 : propagate_rank (int64_t rank, tree op, bool *maybe_biased_p)
     374              : {
     375      7792331 :   int64_t op_rank;
     376              : 
     377      7792331 :   op_rank = get_rank (op);
     378              : 
     379              :   /* Check whether op is biased after the get_rank () call, since it might have
     380              :      updated biased_names.  */
     381      7792331 :   if (TREE_CODE (op) == SSA_NAME
     382      7792331 :       && bitmap_bit_p (biased_names, SSA_NAME_VERSION (op)))
     383              :     {
     384        44713 :       if (maybe_biased_p == NULL)
     385              :         return rank;
     386        30411 :       *maybe_biased_p = true;
     387              :     }
     388              : 
     389      7778029 :   return MAX (rank, op_rank);
     390              : }
     391              : 
     392              : /* Look up the operand rank structure for expression E.  */
     393              : 
     394              : static inline int64_t
     395     14188467 : find_operand_rank (tree e)
     396              : {
     397     14188467 :   int64_t *slot = operand_rank->get (e);
     398     14188467 :   return slot ? *slot : -1;
     399              : }
     400              : 
     401              : /* Insert {E,RANK} into the operand rank hashtable.  */
     402              : 
     403              : static inline void
     404     15210809 : insert_operand_rank (tree e, int64_t rank)
     405              : {
     406     15210809 :   gcc_assert (rank > 0);
     407     15210809 :   bool existed = operand_rank->put (e, rank);
     408     15210809 :   gcc_assert (!existed);
     409     15210809 : }
     410              : 
     411              : /* Given an expression E, return the rank of the expression.  */
     412              : 
     413              : static int64_t
     414     17670981 : get_rank (tree e)
     415              : {
     416              :   /* SSA_NAME's have the rank of the expression they are the result
     417              :      of.
     418              :      For globals and uninitialized values, the rank is 0.
     419              :      For function arguments, use the pre-setup rank.
     420              :      For PHI nodes, stores, asm statements, etc, we use the rank of
     421              :      the BB.
     422              :      For simple operations, the rank is the maximum rank of any of
     423              :      its operands, or the bb_rank, whichever is less.
     424              :      I make no claims that this is optimal, however, it gives good
     425              :      results.  */
     426              : 
     427              :   /* We make an exception to the normal ranking system to break
     428              :      dependences of accumulator variables in loops.  Suppose we
     429              :      have a simple one-block loop containing:
     430              : 
     431              :        x_1 = phi(x_0, x_2)
     432              :        b = a + x_1
     433              :        c = b + d
     434              :        x_2 = c + e
     435              : 
     436              :      As shown, each iteration of the calculation into x is fully
     437              :      dependent upon the iteration before it.  We would prefer to
     438              :      see this in the form:
     439              : 
     440              :        x_1 = phi(x_0, x_2)
     441              :        b = a + d
     442              :        c = b + e
     443              :        x_2 = c + x_1
     444              : 
     445              :      If the loop is unrolled, the calculations of b and c from
     446              :      different iterations can be interleaved.
     447              : 
     448              :      To obtain this result during reassociation, we bias the rank
     449              :      of the phi definition x_1 upward, when it is recognized as an
     450              :      accumulator pattern.  The artificial rank causes it to be
     451              :      added last, providing the desired independence.  */
     452              : 
     453     17670981 :   if (TREE_CODE (e) == SSA_NAME)
     454              :     {
     455     14188467 :       ssa_op_iter iter;
     456     14188467 :       gimple *stmt;
     457     14188467 :       int64_t rank;
     458     14188467 :       tree op;
     459              : 
     460              :       /* If we already have a rank for this expression, use that.  */
     461     14188467 :       rank = find_operand_rank (e);
     462     14188467 :       if (rank != -1)
     463              :         return rank;
     464              : 
     465      8901907 :       stmt = SSA_NAME_DEF_STMT (e);
     466      8901907 :       if (gimple_code (stmt) == GIMPLE_PHI)
     467              :         {
     468      1491525 :           rank = phi_rank (stmt);
     469      1491525 :           if (rank != bb_rank[gimple_bb (stmt)->index])
     470        71220 :             bitmap_set_bit (biased_names, SSA_NAME_VERSION (e));
     471              :         }
     472              : 
     473      7410382 :       else if (!is_gimple_assign (stmt))
     474       253886 :         rank = bb_rank[gimple_bb (stmt)->index];
     475              : 
     476              :       else
     477              :         {
     478      7156496 :           bool biased_p = false;
     479      7156496 :           bool *maybe_biased_p = propagate_bias_p (stmt) ? &biased_p : NULL;
     480              : 
     481              :           /* Otherwise, find the maximum rank for the operands.  As an
     482              :              exception, remove the bias from loop-carried phis when propagating
     483              :              the rank so that dependent operations are not also biased.  */
     484              :           /* Simply walk over all SSA uses - this takes advantage of the
     485              :              fact that non-SSA operands are is_gimple_min_invariant and
     486              :              thus have rank 0.  */
     487      7156496 :           rank = 0;
     488     14948827 :           FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
     489      7792331 :             rank = propagate_rank (rank, op, maybe_biased_p);
     490              : 
     491      7156496 :           rank += 1;
     492      7156496 :           if (biased_p)
     493        29116 :             bitmap_set_bit (biased_names, SSA_NAME_VERSION (e));
     494              :         }
     495              : 
     496      8901907 :       if (dump_file && (dump_flags & TDF_DETAILS))
     497              :         {
     498          205 :           fprintf (dump_file, "Rank for ");
     499          205 :           print_generic_expr (dump_file, e);
     500          205 :           fprintf (dump_file, " is %" PRId64 "\n", rank);
     501              :         }
     502              : 
     503              :       /* Note the rank in the hashtable so we don't recompute it.  */
     504      8901907 :       insert_operand_rank (e, rank);
     505      8901907 :       return rank;
     506              :     }
     507              : 
     508              :   /* Constants, globals, etc., are rank 0 */
     509              :   return 0;
     510              : }
     511              : 
     512              : 
     513              : /* We want integer ones to end up last no matter what, since they are
     514              :    the ones we can do the most with.  */
     515              : #define INTEGER_CONST_TYPE 1 << 4
     516              : #define FLOAT_ONE_CONST_TYPE 1 << 3
     517              : #define FLOAT_CONST_TYPE 1 << 2
     518              : #define OTHER_CONST_TYPE 1 << 1
     519              : 
     520              : /* Classify an invariant tree into integer, float, or other, so that
     521              :    we can sort them to be near other constants of the same type.  */
     522              : static inline int
     523       323342 : constant_type (tree t)
     524              : {
     525       323342 :   if (INTEGRAL_TYPE_P (TREE_TYPE (t)))
     526              :     return INTEGER_CONST_TYPE;
     527        10598 :   else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (t)))
     528              :     {
     529              :       /* Sort -1.0 and 1.0 constants last, while in some cases
     530              :          const_binop can't optimize some inexact operations, multiplication
     531              :          by -1.0 or 1.0 can be always merged with others.  */
     532         6828 :       if (real_onep (t) || real_minus_onep (t))
     533          816 :         return FLOAT_ONE_CONST_TYPE;
     534              :       return FLOAT_CONST_TYPE;
     535              :     }
     536              :   else
     537              :     return OTHER_CONST_TYPE;
     538              : }
     539              : 
     540              : /* qsort comparison function to sort operand entries PA and PB by rank
     541              :    so that the sorted array is ordered by rank in decreasing order.  */
     542              : static int
     543     23694779 : sort_by_operand_rank (const void *pa, const void *pb)
     544              : {
     545     23694779 :   const operand_entry *oea = *(const operand_entry *const *)pa;
     546     23694779 :   const operand_entry *oeb = *(const operand_entry *const *)pb;
     547              : 
     548     23694779 :   if (oeb->rank != oea->rank)
     549     21271160 :     return oeb->rank > oea->rank ? 1 : -1;
     550              : 
     551              :   /* It's nicer for optimize_expression if constants that are likely
     552              :      to fold when added/multiplied/whatever are put next to each
     553              :      other.  Since all constants have rank 0, order them by type.  */
     554      2423619 :   if (oea->rank == 0)
     555              :     {
     556       161613 :       if (constant_type (oeb->op) != constant_type (oea->op))
     557           58 :         return constant_type (oea->op) - constant_type (oeb->op);
     558              :       else
     559              :         /* To make sorting result stable, we use unique IDs to determine
     560              :            order.  */
     561       161555 :         return oeb->id > oea->id ? 1 : -1;
     562              :     }
     563              : 
     564      2262006 :   if (TREE_CODE (oea->op) != SSA_NAME)
     565              :     {
     566            0 :       if (TREE_CODE (oeb->op) != SSA_NAME)
     567            0 :         return oeb->id > oea->id ? 1 : -1;
     568              :       else
     569              :         return 1;
     570              :     }
     571      2262006 :   else if (TREE_CODE (oeb->op) != SSA_NAME)
     572              :     return -1;
     573              : 
     574              :   /* Lastly, make sure the versions that are the same go next to each
     575              :      other.  */
     576      2262006 :   if (SSA_NAME_VERSION (oeb->op) != SSA_NAME_VERSION (oea->op))
     577              :     {
     578              :       /* As SSA_NAME_VERSION is assigned pretty randomly, because we reuse
     579              :          versions of removed SSA_NAMEs, so if possible, prefer to sort
     580              :          based on basic block and gimple_uid of the SSA_NAME_DEF_STMT.
     581              :          See PR60418.  */
     582      2206295 :       gimple *stmta = SSA_NAME_DEF_STMT (oea->op);
     583      2206295 :       gimple *stmtb = SSA_NAME_DEF_STMT (oeb->op);
     584      2206295 :       basic_block bba = gimple_bb (stmta);
     585      2206295 :       basic_block bbb = gimple_bb (stmtb);
     586      2206295 :       if (bbb != bba)
     587              :         {
     588              :           /* One of the SSA_NAMEs can be defined in oeN->stmt_to_insert
     589              :              but the other might not.  */
     590       143751 :           if (!bba)
     591              :             return 1;
     592       139424 :           if (!bbb)
     593              :             return -1;
     594              :           /* If neither is, compare bb_rank.  */
     595       133442 :           if (bb_rank[bbb->index] != bb_rank[bba->index])
     596       133442 :             return (bb_rank[bbb->index] >> 16) - (bb_rank[bba->index] >> 16);
     597              :         }
     598              : 
     599      2062544 :       bool da = reassoc_stmt_dominates_stmt_p (stmta, stmtb);
     600      2062544 :       bool db = reassoc_stmt_dominates_stmt_p (stmtb, stmta);
     601      2062544 :       if (da != db)
     602      2007151 :         return da ? 1 : -1;
     603              : 
     604        55393 :       return SSA_NAME_VERSION (oeb->op) > SSA_NAME_VERSION (oea->op) ? 1 : -1;
     605              :     }
     606              : 
     607        55711 :   return oeb->id > oea->id ? 1 : -1;
     608              : }
     609              : 
     610              : /* Add an operand entry to *OPS for the tree operand OP.  */
     611              : 
     612              : static void
     613      9878158 : add_to_ops_vec (vec<operand_entry *> *ops, tree op, gimple *stmt_to_insert = NULL)
     614              : {
     615      9878158 :   operand_entry *oe = operand_entry_pool.allocate ();
     616              : 
     617      9878158 :   oe->op = op;
     618      9878158 :   oe->rank = get_rank (op);
     619      9878158 :   oe->id = next_operand_entry_id++;
     620      9878158 :   oe->count = 1;
     621      9878158 :   oe->stmt_to_insert = stmt_to_insert;
     622      9878158 :   ops->safe_push (oe);
     623      9878158 : }
     624              : 
     625              : /* Add an operand entry to *OPS for the tree operand OP with repeat
     626              :    count REPEAT.  */
     627              : 
     628              : static void
     629           12 : add_repeat_to_ops_vec (vec<operand_entry *> *ops, tree op,
     630              :                        HOST_WIDE_INT repeat)
     631              : {
     632           12 :   operand_entry *oe = operand_entry_pool.allocate ();
     633              : 
     634           12 :   oe->op = op;
     635           12 :   oe->rank = get_rank (op);
     636           12 :   oe->id = next_operand_entry_id++;
     637           12 :   oe->count = repeat;
     638           12 :   oe->stmt_to_insert = NULL;
     639           12 :   ops->safe_push (oe);
     640              : 
     641           12 :   reassociate_stats.pows_encountered++;
     642           12 : }
     643              : 
     644              : /* Returns true if we can associate the SSA def OP.  */
     645              : 
     646              : static bool
     647     32334253 : can_reassociate_op_p (tree op)
     648              : {
     649     32334253 :   if (TREE_CODE (op) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op))
     650              :     return false;
     651              :   /* Uninitialized variables can't participate in reassociation. */
     652     32333357 :   if (TREE_CODE (op) == SSA_NAME && ssa_name_maybe_undef_p (op))
     653              :     return false;
     654              :   /* Make sure asm goto outputs do not participate in reassociation since
     655              :      we have no way to find an insertion place after asm goto.  */
     656     32328489 :   if (TREE_CODE (op) == SSA_NAME
     657     23732333 :       && gimple_code (SSA_NAME_DEF_STMT (op)) == GIMPLE_ASM
     658     32351164 :       && gimple_asm_nlabels (as_a <gasm *> (SSA_NAME_DEF_STMT (op))) != 0)
     659          100 :     return false;
     660              :   return true;
     661              : }
     662              : 
     663              : /* Returns true if we can reassociate operations of TYPE.
     664              :    That is for integral or non-saturating fixed-point types, and for
     665              :    floating point type when associative-math is enabled.  */
     666              : 
     667              : static bool
     668     59303329 : can_reassociate_type_p (tree type)
     669              : {
     670     59303329 :   if ((ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_WRAPS (type))
     671     36807406 :       || NON_SAT_FIXED_POINT_TYPE_P (type)
     672     96110735 :       || (flag_associative_math && FLOAT_TYPE_P (type)))
     673     22887942 :     return true;
     674              :   return false;
     675              : }
     676              : 
     677              : /* Return true if STMT is reassociable operation containing a binary
     678              :    operation with tree code CODE, and is inside LOOP.  */
     679              : 
     680              : static bool
     681      7859404 : is_reassociable_op (gimple *stmt, enum tree_code code, class loop *loop)
     682              : {
     683      7859404 :   basic_block bb = gimple_bb (stmt);
     684              : 
     685      7859404 :   if (gimple_bb (stmt) == NULL)
     686              :     return false;
     687              : 
     688      7671410 :   if (!flow_bb_inside_loop_p (loop, bb))
     689              :     return false;
     690              : 
     691      7449817 :   if (is_gimple_assign (stmt)
     692      5922122 :       && gimple_assign_rhs_code (stmt) == code
     693      8280541 :       && has_single_use (gimple_assign_lhs (stmt)))
     694              :     {
     695       596116 :       tree rhs1 = gimple_assign_rhs1 (stmt);
     696       596116 :       tree rhs2 = gimple_assign_rhs2 (stmt);
     697       596116 :       if (!can_reassociate_op_p (rhs1)
     698       596116 :           || (rhs2 && !can_reassociate_op_p (rhs2)))
     699          526 :         return false;
     700              :       return true;
     701              :     }
     702              : 
     703              :   return false;
     704              : }
     705              : 
     706              : 
     707              : /* Return true if STMT is a nop-conversion.  */
     708              : 
     709              : static bool
     710      7792835 : gimple_nop_conversion_p (gimple *stmt)
     711              : {
     712      7792835 :   if (gassign *ass = dyn_cast <gassign *> (stmt))
     713              :     {
     714      9146294 :       if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (ass))
     715      6880990 :           && tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (ass)),
     716      1538562 :                                     TREE_TYPE (gimple_assign_rhs1 (ass))))
     717              :         return true;
     718              :     }
     719              :   return false;
     720              : }
     721              : 
     722              : /* Given NAME, if NAME is defined by a unary operation OPCODE, return the
     723              :    operand of the negate operation.  Otherwise, return NULL.  */
     724              : 
     725              : static tree
     726      7701634 : get_unary_op (tree name, enum tree_code opcode)
     727              : {
     728      7701634 :   gimple *stmt = SSA_NAME_DEF_STMT (name);
     729              : 
     730              :   /* Look through nop conversions (sign changes).  */
     731      7701634 :   if (gimple_nop_conversion_p (stmt)
     732      7701634 :       && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
     733       920042 :     stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
     734              : 
     735      7701634 :   if (!is_gimple_assign (stmt))
     736              :     return NULL_TREE;
     737              : 
     738      4891263 :   if (gimple_assign_rhs_code (stmt) == opcode)
     739       126064 :     return gimple_assign_rhs1 (stmt);
     740              :   return NULL_TREE;
     741              : }
     742              : 
     743              : /* Return true if OP1 and OP2 have the same value if casted to either type.  */
     744              : 
     745              : static bool
     746        46794 : ops_equal_values_p (tree op1, tree op2)
     747              : {
     748        46794 :   if (op1 == op2)
     749              :     return true;
     750              : 
     751        46563 :   tree orig_op1 = op1;
     752        46563 :   if (TREE_CODE (op1) == SSA_NAME)
     753              :     {
     754        46563 :       gimple *stmt = SSA_NAME_DEF_STMT (op1);
     755        46563 :       if (gimple_nop_conversion_p (stmt))
     756              :         {
     757        19086 :           op1 = gimple_assign_rhs1 (stmt);
     758        19086 :           if (op1 == op2)
     759              :             return true;
     760              :         }
     761              :     }
     762              : 
     763        44638 :   if (TREE_CODE (op2) == SSA_NAME)
     764              :     {
     765        44638 :       gimple *stmt = SSA_NAME_DEF_STMT (op2);
     766        44638 :       if (gimple_nop_conversion_p (stmt))
     767              :         {
     768        17277 :           op2 = gimple_assign_rhs1 (stmt);
     769        17277 :           if (op1 == op2
     770        17277 :               || orig_op1 == op2)
     771              :             return true;
     772              :         }
     773              :     }
     774              : 
     775              :   return false;
     776              : }
     777              : 
     778              : 
     779              : /* If CURR and LAST are a pair of ops that OPCODE allows us to
     780              :    eliminate through equivalences, do so, remove them from OPS, and
     781              :    return true.  Otherwise, return false.  */
     782              : 
     783              : static bool
     784      9770324 : eliminate_duplicate_pair (enum tree_code opcode,
     785              :                           vec<operand_entry *> *ops,
     786              :                           bool *all_done,
     787              :                           unsigned int i,
     788              :                           operand_entry *curr,
     789              :                           operand_entry *last)
     790              : {
     791              : 
     792              :   /* If we have two of the same op, and the opcode is & |, min, or max,
     793              :      we can eliminate one of them.
     794              :      If we have two of the same op, and the opcode is ^, we can
     795              :      eliminate both of them.  */
     796              : 
     797      9770324 :   if (last && last->op == curr->op)
     798              :     {
     799         5591 :       switch (opcode)
     800              :         {
     801           27 :         case MAX_EXPR:
     802           27 :         case MIN_EXPR:
     803           27 :         case BIT_IOR_EXPR:
     804           27 :         case BIT_AND_EXPR:
     805           27 :           if (dump_file && (dump_flags & TDF_DETAILS))
     806              :             {
     807            1 :               fprintf (dump_file, "Equivalence: ");
     808            1 :               print_generic_expr (dump_file, curr->op);
     809            1 :               fprintf (dump_file, " [&|minmax] ");
     810            1 :               print_generic_expr (dump_file, last->op);
     811            1 :               fprintf (dump_file, " -> ");
     812            1 :               print_generic_stmt (dump_file, last->op);
     813              :             }
     814              : 
     815           27 :           ops->ordered_remove (i);
     816           27 :           reassociate_stats.ops_eliminated ++;
     817              : 
     818           27 :           return true;
     819              : 
     820          122 :         case BIT_XOR_EXPR:
     821          122 :           if (dump_file && (dump_flags & TDF_DETAILS))
     822              :             {
     823            0 :               fprintf (dump_file, "Equivalence: ");
     824            0 :               print_generic_expr (dump_file, curr->op);
     825            0 :               fprintf (dump_file, " ^ ");
     826            0 :               print_generic_expr (dump_file, last->op);
     827            0 :               fprintf (dump_file, " -> nothing\n");
     828              :             }
     829              : 
     830          122 :           reassociate_stats.ops_eliminated += 2;
     831              : 
     832          122 :           if (ops->length () == 2)
     833              :             {
     834            1 :               ops->truncate (0);
     835            1 :               add_to_ops_vec (ops, build_zero_cst (TREE_TYPE (last->op)));
     836            1 :               *all_done = true;
     837              :             }
     838              :           else
     839              :             {
     840          121 :               ops->ordered_remove (i-1);
     841          121 :               ops->ordered_remove (i-1);
     842              :             }
     843              : 
     844              :           return true;
     845              : 
     846              :         default:
     847              :           break;
     848              :         }
     849              :     }
     850              :   return false;
     851              : }
     852              : 
     853              : static vec<tree> plus_negates;
     854              : 
     855              : /* If OPCODE is PLUS_EXPR, CURR->OP is a negate expression or a bitwise not
     856              :    expression, look in OPS for a corresponding positive operation to cancel
     857              :    it out.  If we find one, remove the other from OPS, replace
     858              :    OPS[CURRINDEX] with 0 or -1, respectively, and return true.  Otherwise,
     859              :    return false. */
     860              : 
     861              : static bool
     862      9770175 : eliminate_plus_minus_pair (enum tree_code opcode,
     863              :                            vec<operand_entry *> *ops,
     864              :                            unsigned int currindex,
     865              :                            operand_entry *curr)
     866              : {
     867      9770175 :   tree negateop;
     868      9770175 :   tree notop;
     869      9770175 :   unsigned int i;
     870      9770175 :   operand_entry *oe;
     871              : 
     872      9770175 :   if (opcode != PLUS_EXPR || TREE_CODE (curr->op) != SSA_NAME)
     873              :     return false;
     874              : 
     875      3079315 :   negateop = get_unary_op (curr->op, NEGATE_EXPR);
     876      3079315 :   notop = get_unary_op (curr->op, BIT_NOT_EXPR);
     877      3079315 :   if (negateop == NULL_TREE && notop == NULL_TREE)
     878              :     return false;
     879              : 
     880              :   /* Any non-negated version will have a rank that is one less than
     881              :      the current rank.  So once we hit those ranks, if we don't find
     882              :      one, we can stop.  */
     883              : 
     884       133655 :   for (i = currindex + 1;
     885       196512 :        ops->iterate (i, &oe)
     886       243306 :        && oe->rank >= curr->rank - 1 ;
     887              :        i++)
     888              :     {
     889        46794 :       if (negateop
     890        46794 :           && ops_equal_values_p (oe->op, negateop))
     891              :         {
     892         1579 :           if (dump_file && (dump_flags & TDF_DETAILS))
     893              :             {
     894            0 :               fprintf (dump_file, "Equivalence: ");
     895            0 :               print_generic_expr (dump_file, negateop);
     896            0 :               fprintf (dump_file, " + -");
     897            0 :               print_generic_expr (dump_file, oe->op);
     898            0 :               fprintf (dump_file, " -> 0\n");
     899              :             }
     900              : 
     901         1579 :           ops->ordered_remove (i);
     902         1579 :           add_to_ops_vec (ops, build_zero_cst (TREE_TYPE (oe->op)));
     903         1579 :           ops->ordered_remove (currindex);
     904         1579 :           reassociate_stats.ops_eliminated ++;
     905              : 
     906         1579 :           return true;
     907              :         }
     908        45215 :       else if (notop
     909        45215 :                && ops_equal_values_p (oe->op, notop))
     910              :         {
     911         1917 :           tree op_type = TREE_TYPE (oe->op);
     912              : 
     913         1917 :           if (dump_file && (dump_flags & TDF_DETAILS))
     914              :             {
     915            0 :               fprintf (dump_file, "Equivalence: ");
     916            0 :               print_generic_expr (dump_file, notop);
     917            0 :               fprintf (dump_file, " + ~");
     918            0 :               print_generic_expr (dump_file, oe->op);
     919            0 :               fprintf (dump_file, " -> -1\n");
     920              :             }
     921              : 
     922         1917 :           ops->ordered_remove (i);
     923         1917 :           add_to_ops_vec (ops, build_all_ones_cst (op_type));
     924         1917 :           ops->ordered_remove (currindex);
     925         1917 :           reassociate_stats.ops_eliminated ++;
     926              : 
     927         1917 :           return true;
     928              :         }
     929              :     }
     930              : 
     931              :   /* If CURR->OP is a negate expr without nop conversion in a plus expr:
     932              :      save it for later inspection in repropagate_negates().  */
     933        86861 :   if (negateop != NULL_TREE
     934        86861 :       && gimple_assign_rhs_code (SSA_NAME_DEF_STMT (curr->op)) == NEGATE_EXPR)
     935        86352 :     plus_negates.safe_push (curr->op);
     936              : 
     937              :   return false;
     938              : }
     939              : 
     940              : /* If OPCODE is BIT_IOR_EXPR, BIT_AND_EXPR, and, CURR->OP is really a
     941              :    bitwise not expression, look in OPS for a corresponding operand to
     942              :    cancel it out.  If we find one, remove the other from OPS, replace
     943              :    OPS[CURRINDEX] with 0, and return true.  Otherwise, return
     944              :    false. */
     945              : 
     946              : static bool
     947      9770325 : eliminate_not_pairs (enum tree_code opcode,
     948              :                      vec<operand_entry *> *ops,
     949              :                      unsigned int currindex,
     950              :                      operand_entry *curr)
     951              : {
     952      9770325 :   tree notop;
     953      9770325 :   unsigned int i;
     954      9770325 :   operand_entry *oe;
     955              : 
     956      9770325 :   if ((opcode != BIT_IOR_EXPR && opcode != BIT_AND_EXPR)
     957      2047180 :       || TREE_CODE (curr->op) != SSA_NAME)
     958              :     return false;
     959              : 
     960      1543004 :   notop = get_unary_op (curr->op, BIT_NOT_EXPR);
     961      1543004 :   if (notop == NULL_TREE)
     962              :     return false;
     963              : 
     964              :   /* Any non-not version will have a rank that is one less than
     965              :      the current rank.  So once we hit those ranks, if we don't find
     966              :      one, we can stop.  */
     967              : 
     968        35707 :   for (i = currindex + 1;
     969      9798602 :        ops->iterate (i, &oe)
     970        71474 :        && oe->rank >= curr->rank - 1;
     971              :        i++)
     972              :     {
     973         7491 :       if (oe->op == notop)
     974              :         {
     975            1 :           if (dump_file && (dump_flags & TDF_DETAILS))
     976              :             {
     977            0 :               fprintf (dump_file, "Equivalence: ");
     978            0 :               print_generic_expr (dump_file, notop);
     979            0 :               if (opcode == BIT_AND_EXPR)
     980            0 :                 fprintf (dump_file, " & ~");
     981            0 :               else if (opcode == BIT_IOR_EXPR)
     982            0 :                 fprintf (dump_file, " | ~");
     983            0 :               print_generic_expr (dump_file, oe->op);
     984            0 :               if (opcode == BIT_AND_EXPR)
     985            0 :                 fprintf (dump_file, " -> 0\n");
     986            0 :               else if (opcode == BIT_IOR_EXPR)
     987            0 :                 fprintf (dump_file, " -> -1\n");
     988              :             }
     989              : 
     990            1 :           if (opcode == BIT_AND_EXPR)
     991            1 :             oe->op = build_zero_cst (TREE_TYPE (oe->op));
     992            0 :           else if (opcode == BIT_IOR_EXPR)
     993            0 :             oe->op = build_all_ones_cst (TREE_TYPE (oe->op));
     994              : 
     995            1 :           reassociate_stats.ops_eliminated += ops->length () - 1;
     996            1 :           ops->truncate (0);
     997            1 :           ops->quick_push (oe);
     998            1 :           return true;
     999              :         }
    1000              :     }
    1001              : 
    1002              :   return false;
    1003              : }
    1004              : 
    1005              : /* Use constant value that may be present in OPS to try to eliminate
    1006              :    operands.  Note that this function is only really used when we've
    1007              :    eliminated ops for other reasons, or merged constants.  Across
    1008              :    single statements, fold already does all of this, plus more.  There
    1009              :    is little point in duplicating logic, so I've only included the
    1010              :    identities that I could ever construct testcases to trigger.  */
    1011              : 
    1012              : static void
    1013      4762234 : eliminate_using_constants (enum tree_code opcode,
    1014              :                            vec<operand_entry *> *ops)
    1015              : {
    1016      4762234 :   operand_entry *oelast = ops->last ();
    1017      4762234 :   tree type = TREE_TYPE (oelast->op);
    1018              : 
    1019      4762234 :   if (oelast->rank == 0
    1020      4762234 :       && (ANY_INTEGRAL_TYPE_P (type) || FLOAT_TYPE_P (type)))
    1021              :     {
    1022      3440857 :       switch (opcode)
    1023              :         {
    1024       430903 :         case BIT_AND_EXPR:
    1025       430903 :           if (integer_zerop (oelast->op))
    1026              :             {
    1027            0 :               if (ops->length () != 1)
    1028              :                 {
    1029            0 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1030            0 :                     fprintf (dump_file, "Found & 0, removing all other ops\n");
    1031              : 
    1032            0 :                   reassociate_stats.ops_eliminated += ops->length () - 1;
    1033              : 
    1034            0 :                   ops->truncate (0);
    1035            0 :                   ops->quick_push (oelast);
    1036         1839 :                   return;
    1037              :                 }
    1038              :             }
    1039       430903 :           else if (integer_all_onesp (oelast->op))
    1040              :             {
    1041            4 :               if (ops->length () != 1)
    1042              :                 {
    1043            4 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1044            0 :                     fprintf (dump_file, "Found & -1, removing\n");
    1045            4 :                   ops->pop ();
    1046            4 :                   reassociate_stats.ops_eliminated++;
    1047              :                 }
    1048              :             }
    1049              :           break;
    1050        72740 :         case BIT_IOR_EXPR:
    1051        72740 :           if (integer_all_onesp (oelast->op))
    1052              :             {
    1053            0 :               if (ops->length () != 1)
    1054              :                 {
    1055            0 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1056            0 :                     fprintf (dump_file, "Found | -1, removing all other ops\n");
    1057              : 
    1058            0 :                   reassociate_stats.ops_eliminated += ops->length () - 1;
    1059              : 
    1060            0 :                   ops->truncate (0);
    1061            0 :                   ops->quick_push (oelast);
    1062            0 :                   return;
    1063              :                 }
    1064              :             }
    1065        72740 :           else if (integer_zerop (oelast->op))
    1066              :             {
    1067            4 :               if (ops->length () != 1)
    1068              :                 {
    1069            4 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1070            0 :                     fprintf (dump_file, "Found | 0, removing\n");
    1071            4 :                   ops->pop ();
    1072            4 :                   reassociate_stats.ops_eliminated++;
    1073              :                 }
    1074              :             }
    1075              :           break;
    1076       962918 :         case MULT_EXPR:
    1077       962918 :           if (integer_zerop (oelast->op)
    1078       962918 :               || (FLOAT_TYPE_P (type)
    1079         1438 :                   && !HONOR_NANS (type)
    1080         1343 :                   && !HONOR_SIGNED_ZEROS (type)
    1081         1343 :                   && real_zerop (oelast->op)))
    1082              :             {
    1083            0 :               if (ops->length () != 1)
    1084              :                 {
    1085            0 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1086            0 :                     fprintf (dump_file, "Found * 0, removing all other ops\n");
    1087              : 
    1088            0 :                   reassociate_stats.ops_eliminated += ops->length () - 1;
    1089            0 :                   ops->truncate (0);
    1090            0 :                   ops->quick_push (oelast);
    1091            0 :                   return;
    1092              :                 }
    1093              :             }
    1094       962918 :           else if (integer_onep (oelast->op)
    1095       962918 :                    || (FLOAT_TYPE_P (type)
    1096         1438 :                        && !HONOR_SNANS (type)
    1097         1438 :                        && real_onep (oelast->op)))
    1098              :             {
    1099            4 :               if (ops->length () != 1)
    1100              :                 {
    1101            4 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1102            0 :                     fprintf (dump_file, "Found * 1, removing\n");
    1103            4 :                   ops->pop ();
    1104            4 :                   reassociate_stats.ops_eliminated++;
    1105            4 :                   return;
    1106              :                 }
    1107              :             }
    1108              :           break;
    1109      1800623 :         case BIT_XOR_EXPR:
    1110      1800623 :         case PLUS_EXPR:
    1111      1800623 :         case MINUS_EXPR:
    1112      1800623 :           if (integer_zerop (oelast->op)
    1113      1800623 :               || (FLOAT_TYPE_P (type)
    1114          849 :                   && (opcode == PLUS_EXPR || opcode == MINUS_EXPR)
    1115          849 :                   && fold_real_zero_addition_p (type, 0, oelast->op,
    1116              :                                                 opcode == MINUS_EXPR)))
    1117              :             {
    1118         1835 :               if (ops->length () != 1)
    1119              :                 {
    1120         1835 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    1121            0 :                     fprintf (dump_file, "Found [|^+] 0, removing\n");
    1122         1835 :                   ops->pop ();
    1123         1835 :                   reassociate_stats.ops_eliminated++;
    1124         1835 :                   return;
    1125              :                 }
    1126              :             }
    1127              :           break;
    1128              :         default:
    1129              :           break;
    1130              :         }
    1131              :     }
    1132              : }
    1133              : 
    1134              : 
    1135              : static void linearize_expr_tree (vec<operand_entry *> *, gimple *,
    1136              :                                  bool, bool);
    1137              : 
    1138              : /* Structure for tracking and counting operands.  */
    1139              : struct oecount {
    1140              :   unsigned int cnt;
    1141              :   unsigned int id;
    1142              :   enum tree_code oecode;
    1143              :   tree op;
    1144              : };
    1145              : 
    1146              : 
    1147              : /* The heap for the oecount hashtable and the sorted list of operands.  */
    1148              : static vec<oecount> cvec;
    1149              : 
    1150              : 
    1151              : /* Oecount hashtable helpers.  */
    1152              : 
    1153              : struct oecount_hasher : int_hash <int, 0, 1>
    1154              : {
    1155              :   static inline hashval_t hash (int);
    1156              :   static inline bool equal (int, int);
    1157              : };
    1158              : 
    1159              : /* Hash function for oecount.  */
    1160              : 
    1161              : inline hashval_t
    1162       155839 : oecount_hasher::hash (int p)
    1163              : {
    1164       155839 :   const oecount *c = &cvec[p - 42];
    1165       155839 :   return htab_hash_pointer (c->op) ^ (hashval_t)c->oecode;
    1166              : }
    1167              : 
    1168              : /* Comparison function for oecount.  */
    1169              : 
    1170              : inline bool
    1171        84729 : oecount_hasher::equal (int p1, int p2)
    1172              : {
    1173        84729 :   const oecount *c1 = &cvec[p1 - 42];
    1174        84729 :   const oecount *c2 = &cvec[p2 - 42];
    1175        84729 :   return c1->oecode == c2->oecode && c1->op == c2->op;
    1176              : }
    1177              : 
    1178              : /* Comparison function for qsort sorting oecount elements by count.  */
    1179              : 
    1180              : static int
    1181       601846 : oecount_cmp (const void *p1, const void *p2)
    1182              : {
    1183       601846 :   const oecount *c1 = (const oecount *)p1;
    1184       601846 :   const oecount *c2 = (const oecount *)p2;
    1185       601846 :   if (c1->cnt != c2->cnt)
    1186         8974 :     return c1->cnt > c2->cnt ? 1 : -1;
    1187              :   else
    1188              :     /* If counts are identical, use unique IDs to stabilize qsort.  */
    1189       592872 :     return c1->id > c2->id ? 1 : -1;
    1190              : }
    1191              : 
    1192              : /* Return TRUE iff STMT represents a builtin call that raises OP
    1193              :    to some constant integer exponent, as required by decrement_power.  */
    1194              : 
    1195              : static bool
    1196         1280 : stmt_is_power_of_op (gimple *stmt, tree op)
    1197              : {
    1198         1280 :   if (!is_gimple_call (stmt))
    1199              :     return false;
    1200              : 
    1201            8 :   switch (gimple_call_combined_fn (stmt))
    1202              :     {
    1203            2 :     CASE_CFN_POW:
    1204            2 :       {
    1205            2 :         tree arg1 = gimple_call_arg (stmt, 1);
    1206            2 :         if (TREE_CODE (arg1) != REAL_CST)
    1207              :           return false;
    1208            1 :         REAL_VALUE_TYPE c = TREE_REAL_CST (arg1);
    1209            1 :         REAL_VALUE_TYPE cint;
    1210            1 :         real_from_integer (&cint, VOIDmode, real_to_integer (&c), SIGNED);
    1211            1 :         if (!real_identical (&c, &cint))
    1212              :           return false;
    1213            0 :         return operand_equal_p (gimple_call_arg (stmt, 0), op, 0);
    1214              :       }
    1215              : 
    1216            1 :     CASE_CFN_POWI:
    1217            1 :       if (!tree_fits_shwi_p (gimple_call_arg (stmt, 1)))
    1218              :         return false;
    1219            0 :       return operand_equal_p (gimple_call_arg (stmt, 0), op, 0);
    1220              : 
    1221              :     default:
    1222              :       return false;
    1223              :     }
    1224              : }
    1225              : 
    1226              : /* Given STMT which is a __builtin_pow* call, decrement its exponent
    1227              :    in place and return the result.  Assumes that stmt_is_power_of_op
    1228              :    was previously called for STMT and returned TRUE.  */
    1229              : 
    1230              : static HOST_WIDE_INT
    1231            0 : decrement_power (gimple *stmt)
    1232              : {
    1233            0 :   REAL_VALUE_TYPE c, cint;
    1234            0 :   HOST_WIDE_INT power;
    1235            0 :   tree arg1;
    1236              : 
    1237            0 :   switch (gimple_call_combined_fn (stmt))
    1238              :     {
    1239            0 :     CASE_CFN_POW:
    1240            0 :       arg1 = gimple_call_arg (stmt, 1);
    1241            0 :       c = TREE_REAL_CST (arg1);
    1242            0 :       power = real_to_integer (&c) - 1;
    1243            0 :       real_from_integer (&cint, VOIDmode, power, SIGNED);
    1244            0 :       gimple_call_set_arg (stmt, 1, build_real (TREE_TYPE (arg1), cint));
    1245            0 :       return power;
    1246              : 
    1247            0 :     CASE_CFN_POWI:
    1248            0 :       arg1 = gimple_call_arg (stmt, 1);
    1249            0 :       power = TREE_INT_CST_LOW (arg1) - 1;
    1250            0 :       gimple_call_set_arg (stmt, 1, build_int_cst (TREE_TYPE (arg1), power));
    1251            0 :       return power;
    1252              : 
    1253            0 :     default:
    1254            0 :       gcc_unreachable ();
    1255              :     }
    1256              : }
    1257              : 
    1258              : /* Replace SSA defined by STMT and replace all its uses with new
    1259              :    SSA.  Also return the new SSA.  */
    1260              : 
    1261              : static tree
    1262          344 : make_new_ssa_for_def (gimple *stmt, enum tree_code opcode, tree op)
    1263              : {
    1264          344 :   gimple *use_stmt;
    1265          344 :   use_operand_p use;
    1266          344 :   imm_use_iterator iter;
    1267          344 :   tree new_lhs, new_debug_lhs = NULL_TREE;
    1268          344 :   tree lhs = gimple_get_lhs (stmt);
    1269              : 
    1270          344 :   new_lhs = make_ssa_name (TREE_TYPE (lhs));
    1271          344 :   gimple_set_lhs (stmt, new_lhs);
    1272              : 
    1273              :   /* Also need to update GIMPLE_DEBUGs.  */
    1274          745 :   FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    1275              :     {
    1276          401 :       tree repl = new_lhs;
    1277          401 :       if (is_gimple_debug (use_stmt))
    1278              :         {
    1279           57 :           if (new_debug_lhs == NULL_TREE)
    1280              :             {
    1281           21 :               new_debug_lhs = build_debug_expr_decl (TREE_TYPE (lhs));
    1282           21 :               gdebug *def_temp
    1283           21 :                 = gimple_build_debug_bind (new_debug_lhs,
    1284           21 :                                            build2 (opcode, TREE_TYPE (lhs),
    1285              :                                                    new_lhs, op),
    1286              :                                            stmt);
    1287           21 :               gimple_set_uid (def_temp, gimple_uid (stmt));
    1288           21 :               gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    1289           21 :               gsi_insert_after (&gsi, def_temp, GSI_SAME_STMT);
    1290              :             }
    1291              :           repl = new_debug_lhs;
    1292              :         }
    1293          802 :       FOR_EACH_IMM_USE_ON_STMT (use, iter)
    1294          401 :         SET_USE (use, repl);
    1295          401 :       update_stmt (use_stmt);
    1296          344 :     }
    1297          344 :   return new_lhs;
    1298              : }
    1299              : 
    1300              : /* Replace all SSAs defined in STMTS_TO_FIX and replace its
    1301              :    uses with new SSAs.  Also do this for the stmt that defines DEF
    1302              :    if *DEF is not OP.  */
    1303              : 
    1304              : static void
    1305          288 : make_new_ssa_for_all_defs (tree *def, enum tree_code opcode, tree op,
    1306              :                            vec<gimple *> &stmts_to_fix)
    1307              : {
    1308          288 :   unsigned i;
    1309          288 :   gimple *stmt;
    1310              : 
    1311          288 :   if (*def != op
    1312          288 :       && TREE_CODE (*def) == SSA_NAME
    1313          288 :       && (stmt = SSA_NAME_DEF_STMT (*def))
    1314          576 :       && gimple_code (stmt) != GIMPLE_NOP)
    1315          288 :     *def = make_new_ssa_for_def (stmt, opcode, op);
    1316              : 
    1317          344 :   FOR_EACH_VEC_ELT (stmts_to_fix, i, stmt)
    1318           56 :     make_new_ssa_for_def (stmt, opcode, op);
    1319          288 : }
    1320              : 
    1321              : /* Find the single immediate use of STMT's LHS, and replace it
    1322              :    with OP.  Remove STMT.  If STMT's LHS is the same as *DEF,
    1323              :    replace *DEF with OP as well.  */
    1324              : 
    1325              : static void
    1326          886 : propagate_op_to_single_use (tree op, gimple *stmt, tree *def)
    1327              : {
    1328          886 :   tree lhs;
    1329          886 :   gimple *use_stmt;
    1330          886 :   use_operand_p use;
    1331          886 :   gimple_stmt_iterator gsi;
    1332              : 
    1333          886 :   if (is_gimple_call (stmt))
    1334            0 :     lhs = gimple_call_lhs (stmt);
    1335              :   else
    1336          886 :     lhs = gimple_assign_lhs (stmt);
    1337              : 
    1338          886 :   gcc_assert (has_single_use (lhs));
    1339          886 :   single_imm_use (lhs, &use, &use_stmt);
    1340          886 :   if (lhs == *def)
    1341          602 :     *def = op;
    1342          886 :   SET_USE (use, op);
    1343          886 :   if (TREE_CODE (op) != SSA_NAME)
    1344           40 :     update_stmt (use_stmt);
    1345          886 :   gsi = gsi_for_stmt (stmt);
    1346          886 :   unlink_stmt_vdef (stmt);
    1347          886 :   reassoc_remove_stmt (&gsi);
    1348          886 :   release_defs (stmt);
    1349          886 : }
    1350              : 
    1351              : /* Walks the linear chain with result *DEF searching for an operation
    1352              :    with operand OP and code OPCODE removing that from the chain.  *DEF
    1353              :    is updated if there is only one operand but no operation left.  */
    1354              : 
    1355              : static void
    1356          890 : zero_one_operation (tree *def, enum tree_code opcode, tree op)
    1357              : {
    1358          890 :   tree orig_def = *def;
    1359          890 :   gimple *stmt = SSA_NAME_DEF_STMT (*def);
    1360              :   /* PR72835 - Record the stmt chain that has to be updated such that
    1361              :      we dont use the same LHS when the values computed are different.  */
    1362          890 :   auto_vec<gimple *, 64> stmts_to_fix;
    1363              : 
    1364         1570 :   do
    1365              :     {
    1366         1230 :       tree name;
    1367              : 
    1368         1230 :       if (opcode == MULT_EXPR)
    1369              :         {
    1370         1228 :           if (stmt_is_power_of_op (stmt, op))
    1371              :             {
    1372            0 :               if (decrement_power (stmt) == 1)
    1373              :                 {
    1374            0 :                   if (stmts_to_fix.length () > 0)
    1375            0 :                     stmts_to_fix.pop ();
    1376            0 :                   propagate_op_to_single_use (op, stmt, def);
    1377              :                 }
    1378              :               break;
    1379              :             }
    1380         1228 :           else if (gimple_assign_rhs_code (stmt) == NEGATE_EXPR)
    1381              :             {
    1382           15 :               if (gimple_assign_rhs1 (stmt) == op)
    1383              :                 {
    1384           11 :                   tree cst = build_minus_one_cst (TREE_TYPE (op));
    1385           11 :                   if (stmts_to_fix.length () > 0)
    1386           11 :                     stmts_to_fix.pop ();
    1387           11 :                   propagate_op_to_single_use (cst, stmt, def);
    1388           11 :                   break;
    1389              :                 }
    1390            4 :               else if (integer_minus_onep (op)
    1391            4 :                        || real_minus_onep (op))
    1392              :                 {
    1393            4 :                   gimple_assign_set_rhs_code
    1394            4 :                     (stmt, TREE_CODE (gimple_assign_rhs1 (stmt)));
    1395            4 :                   break;
    1396              :                 }
    1397              :             }
    1398              :         }
    1399              : 
    1400         1215 :       name = gimple_assign_rhs1 (stmt);
    1401              : 
    1402              :       /* If this is the operation we look for and one of the operands
    1403              :          is ours simply propagate the other operand into the stmts
    1404              :          single use.  */
    1405         1215 :       if (gimple_assign_rhs_code (stmt) == opcode
    1406         1215 :           && (name == op
    1407          949 :               || gimple_assign_rhs2 (stmt) == op))
    1408              :         {
    1409          875 :           if (name == op)
    1410          266 :             name = gimple_assign_rhs2 (stmt);
    1411          875 :           if (stmts_to_fix.length () > 0)
    1412          273 :             stmts_to_fix.pop ();
    1413          875 :           propagate_op_to_single_use (name, stmt, def);
    1414          875 :           break;
    1415              :         }
    1416              : 
    1417              :       /* We might have a multiply of two __builtin_pow* calls, and
    1418              :          the operand might be hiding in the rightmost one.  Likewise
    1419              :          this can happen for a negate.  */
    1420          340 :       if (opcode == MULT_EXPR
    1421          340 :           && gimple_assign_rhs_code (stmt) == opcode
    1422          340 :           && TREE_CODE (gimple_assign_rhs2 (stmt)) == SSA_NAME
    1423          615 :           && has_single_use (gimple_assign_rhs2 (stmt)))
    1424              :         {
    1425           52 :           gimple *stmt2 = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
    1426           52 :           if (stmt_is_power_of_op (stmt2, op))
    1427              :             {
    1428            0 :               if (decrement_power (stmt2) == 1)
    1429            0 :                 propagate_op_to_single_use (op, stmt2, def);
    1430              :               else
    1431            0 :                 stmts_to_fix.safe_push (stmt2);
    1432            0 :               break;
    1433              :             }
    1434           52 :           else if (is_gimple_assign (stmt2)
    1435           52 :                    && gimple_assign_rhs_code (stmt2) == NEGATE_EXPR)
    1436              :             {
    1437            0 :               if (gimple_assign_rhs1 (stmt2) == op)
    1438              :                 {
    1439            0 :                   tree cst = build_minus_one_cst (TREE_TYPE (op));
    1440            0 :                   propagate_op_to_single_use (cst, stmt2, def);
    1441            0 :                   break;
    1442              :                 }
    1443            0 :               else if (integer_minus_onep (op)
    1444            0 :                        || real_minus_onep (op))
    1445              :                 {
    1446            0 :                   stmts_to_fix.safe_push (stmt2);
    1447            0 :                   gimple_assign_set_rhs_code
    1448            0 :                     (stmt2, TREE_CODE (gimple_assign_rhs1 (stmt2)));
    1449            0 :                   break;
    1450              :                 }
    1451              :             }
    1452              :         }
    1453              : 
    1454              :       /* Continue walking the chain.  */
    1455          340 :       gcc_assert (name != op
    1456              :                   && TREE_CODE (name) == SSA_NAME);
    1457          340 :       stmt = SSA_NAME_DEF_STMT (name);
    1458          340 :       stmts_to_fix.safe_push (stmt);
    1459          340 :     }
    1460              :   while (1);
    1461              : 
    1462          890 :   if (stmts_to_fix.length () > 0 || *def == orig_def)
    1463          288 :     make_new_ssa_for_all_defs (def, opcode, op, stmts_to_fix);
    1464          890 : }
    1465              : 
    1466              : /* Returns true if statement S1 dominates statement S2.  Like
    1467              :    stmt_dominates_stmt_p, but uses stmt UIDs to optimize.  */
    1468              : 
    1469              : static bool
    1470      4822474 : reassoc_stmt_dominates_stmt_p (gimple *s1, gimple *s2)
    1471              : {
    1472      4822474 :   basic_block bb1 = gimple_bb (s1), bb2 = gimple_bb (s2);
    1473              : 
    1474              :   /* If bb1 is NULL, it should be a GIMPLE_NOP def stmt of an (D)
    1475              :      SSA_NAME.  Assume it lives at the beginning of function and
    1476              :      thus dominates everything.  */
    1477      4822474 :   if (!bb1 || s1 == s2)
    1478              :     return true;
    1479              : 
    1480              :   /* If bb2 is NULL, it doesn't dominate any stmt with a bb.  */
    1481      4819301 :   if (!bb2)
    1482              :     return false;
    1483              : 
    1484      4811774 :   if (bb1 == bb2)
    1485              :     {
    1486              :       /* PHIs in the same basic block are assumed to be
    1487              :          executed all in parallel, if only one stmt is a PHI,
    1488              :          it dominates the other stmt in the same basic block.  */
    1489      4671457 :       if (gimple_code (s1) == GIMPLE_PHI)
    1490              :         return true;
    1491              : 
    1492      4556753 :       if (gimple_code (s2) == GIMPLE_PHI)
    1493              :         return false;
    1494              : 
    1495      4505018 :       gcc_assert (gimple_uid (s1) && gimple_uid (s2));
    1496              : 
    1497      4505018 :       if (gimple_uid (s1) < gimple_uid (s2))
    1498              :         return true;
    1499              : 
    1500      2472049 :       if (gimple_uid (s1) > gimple_uid (s2))
    1501              :         return false;
    1502              : 
    1503        36262 :       gimple_stmt_iterator gsi = gsi_for_stmt (s1);
    1504        36262 :       unsigned int uid = gimple_uid (s1);
    1505        78074 :       for (gsi_next (&gsi); !gsi_end_p (gsi); gsi_next (&gsi))
    1506              :         {
    1507        76212 :           gimple *s = gsi_stmt (gsi);
    1508        76212 :           if (gimple_uid (s) != uid)
    1509              :             break;
    1510        45071 :           if (s == s2)
    1511              :             return true;
    1512              :         }
    1513              : 
    1514              :       return false;
    1515              :     }
    1516              : 
    1517       140317 :   return dominated_by_p (CDI_DOMINATORS, bb2, bb1);
    1518              : }
    1519              : 
    1520              : /* Insert STMT after INSERT_POINT.  */
    1521              : 
    1522              : static void
    1523        38837 : insert_stmt_after (gimple *stmt, gimple *insert_point)
    1524              : {
    1525        38837 :   gimple_stmt_iterator gsi;
    1526        38837 :   basic_block bb;
    1527              : 
    1528        38837 :   if (gimple_code (insert_point) == GIMPLE_PHI)
    1529           30 :     bb = gimple_bb (insert_point);
    1530        38807 :   else if (!stmt_ends_bb_p (insert_point))
    1531              :     {
    1532        38799 :       gsi = gsi_for_stmt (insert_point);
    1533        38799 :       gimple_set_uid (stmt, gimple_uid (insert_point));
    1534        38799 :       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
    1535        38799 :       return;
    1536              :     }
    1537            8 :   else if (gimple_code (insert_point) == GIMPLE_ASM
    1538            5 :            && gimple_asm_nlabels (as_a <gasm *> (insert_point)) != 0
    1539           13 :            && !single_succ_p (gimple_bb (insert_point)))
    1540              :     /* We have no idea where to insert - it depends on where the
    1541              :        uses will be placed.  */
    1542            0 :     gcc_unreachable ();
    1543              :   else
    1544              :     /* We assume INSERT_POINT is a SSA_NAME_DEF_STMT of some SSA_NAME,
    1545              :        thus if it must end a basic block, it should be a call that can
    1546              :        throw, or some assignment that can throw.  If it throws, the LHS
    1547              :        of it will not be initialized though, so only valid places using
    1548              :        the SSA_NAME should be dominated by the fallthru edge.  */
    1549            8 :     bb = find_fallthru_edge (gimple_bb (insert_point)->succs)->dest;
    1550           38 :   gsi = gsi_after_labels (bb);
    1551           38 :   if (gsi_end_p (gsi))
    1552              :     {
    1553            0 :       gimple_stmt_iterator gsi2 = gsi_last_bb (bb);
    1554            0 :       gimple_set_uid (stmt,
    1555            0 :                       gsi_end_p (gsi2) ? 1 : gimple_uid (gsi_stmt (gsi2)));
    1556              :     }
    1557              :   else
    1558           38 :     gimple_set_uid (stmt, gimple_uid (gsi_stmt (gsi)));
    1559           38 :   gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
    1560              : }
    1561              : 
    1562              : /* Builds one statement performing OP1 OPCODE OP2 using TMPVAR for
    1563              :    the result.  Places the statement after the definition of either
    1564              :    OP1 or OP2.  Returns the new statement.  */
    1565              : 
    1566              : static gimple *
    1567         8656 : build_and_add_sum (tree type, tree op1, tree op2, enum tree_code opcode)
    1568              : {
    1569         8656 :   gimple *op1def = NULL, *op2def = NULL;
    1570         8656 :   gimple_stmt_iterator gsi;
    1571         8656 :   tree op;
    1572         8656 :   gassign *sum;
    1573              : 
    1574              :   /* Create the addition statement.  */
    1575         8656 :   op = make_ssa_name (type);
    1576         8656 :   sum = gimple_build_assign (op, opcode, op1, op2);
    1577              : 
    1578              :   /* Find an insertion place and insert.  */
    1579         8656 :   if (TREE_CODE (op1) == SSA_NAME)
    1580         8655 :     op1def = SSA_NAME_DEF_STMT (op1);
    1581         8656 :   if (TREE_CODE (op2) == SSA_NAME)
    1582         8365 :     op2def = SSA_NAME_DEF_STMT (op2);
    1583         8655 :   if ((!op1def || gimple_nop_p (op1def))
    1584         8742 :       && (!op2def || gimple_nop_p (op2def)))
    1585              :     {
    1586           86 :       gsi = gsi_start_nondebug_after_labels_bb
    1587           86 :                 (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    1588           86 :       if (!gsi_end_p (gsi)
    1589           86 :           && is_gimple_call (gsi_stmt (gsi))
    1590           98 :           && (gimple_call_flags (gsi_stmt (gsi)) & ECF_RETURNS_TWICE))
    1591              :         {
    1592              :           /* Don't add statements before a returns_twice call at the start
    1593              :              of a function.  */
    1594            2 :           split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    1595            2 :           gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    1596              :         }
    1597           86 :       if (gsi_end_p (gsi))
    1598              :         {
    1599            2 :           gimple_stmt_iterator gsi2
    1600            2 :             = gsi_last_bb (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    1601            2 :           gimple_set_uid (sum,
    1602            2 :                           gsi_end_p (gsi2) ? 1 : gimple_uid (gsi_stmt (gsi2)));
    1603              :         }
    1604              :       else
    1605           84 :         gimple_set_uid (sum, gimple_uid (gsi_stmt (gsi)));
    1606           86 :       gsi_insert_before (&gsi, sum, GSI_NEW_STMT);
    1607              :     }
    1608              :   else
    1609              :     {
    1610         8570 :       gimple *insert_point;
    1611         8569 :       if ((!op1def || gimple_nop_p (op1def))
    1612        17139 :            || (op2def && !gimple_nop_p (op2def)
    1613         8247 :                && reassoc_stmt_dominates_stmt_p (op1def, op2def)))
    1614              :         insert_point = op2def;
    1615              :       else
    1616              :         insert_point = op1def;
    1617         8570 :       insert_stmt_after (sum, insert_point);
    1618              :     }
    1619         8656 :   update_stmt (sum);
    1620              : 
    1621         8656 :   return sum;
    1622              : }
    1623              : 
    1624              : /* Perform un-distribution of divisions and multiplications.
    1625              :    A * X + B * X is transformed into (A + B) * X and A / X + B / X
    1626              :    to (A + B) / X for real X.
    1627              : 
    1628              :    The algorithm is organized as follows.
    1629              : 
    1630              :     - First we walk the addition chain *OPS looking for summands that
    1631              :       are defined by a multiplication or a real division.  This results
    1632              :       in the candidates bitmap with relevant indices into *OPS.
    1633              : 
    1634              :     - Second we build the chains of multiplications or divisions for
    1635              :       these candidates, counting the number of occurrences of (operand, code)
    1636              :       pairs in all of the candidates chains.
    1637              : 
    1638              :     - Third we sort the (operand, code) pairs by number of occurrence and
    1639              :       process them starting with the pair with the most uses.
    1640              : 
    1641              :       * For each such pair we walk the candidates again to build a
    1642              :         second candidate bitmap noting all multiplication/division chains
    1643              :         that have at least one occurrence of (operand, code).
    1644              : 
    1645              :       * We build an alternate addition chain only covering these
    1646              :         candidates with one (operand, code) operation removed from their
    1647              :         multiplication/division chain.
    1648              : 
    1649              :       * The first candidate gets replaced by the alternate addition chain
    1650              :         multiplied/divided by the operand.
    1651              : 
    1652              :       * All candidate chains get disabled for further processing and
    1653              :         processing of (operand, code) pairs continues.
    1654              : 
    1655              :   The alternate addition chains built are re-processed by the main
    1656              :   reassociation algorithm which allows optimizing a * x * y + b * y * x
    1657              :   to (a + b ) * x * y in one invocation of the reassociation pass.  */
    1658              : 
    1659              : static bool
    1660      4760050 : undistribute_ops_list (enum tree_code opcode,
    1661              :                        vec<operand_entry *> *ops, class loop *loop)
    1662              : {
    1663      4760050 :   unsigned int length = ops->length ();
    1664      4760050 :   operand_entry *oe1;
    1665      4760050 :   unsigned i, j;
    1666      4760050 :   unsigned nr_candidates, nr_candidates2;
    1667      4760050 :   sbitmap_iterator sbi0;
    1668      4760050 :   vec<operand_entry *> *subops;
    1669      4760050 :   bool changed = false;
    1670      4760050 :   unsigned int next_oecount_id = 0;
    1671              : 
    1672      4760050 :   if (length <= 1
    1673      4760050 :       || opcode != PLUS_EXPR)
    1674              :     return false;
    1675              : 
    1676              :   /* Build a list of candidates to process.  */
    1677      2340835 :   auto_sbitmap candidates (length);
    1678      2340835 :   bitmap_clear (candidates);
    1679      2340835 :   nr_candidates = 0;
    1680      7198241 :   FOR_EACH_VEC_ELT (*ops, i, oe1)
    1681              :     {
    1682      4857406 :       enum tree_code dcode;
    1683      4857406 :       gimple *oe1def;
    1684              : 
    1685      4857406 :       if (TREE_CODE (oe1->op) != SSA_NAME)
    1686      1785734 :         continue;
    1687      3071672 :       oe1def = SSA_NAME_DEF_STMT (oe1->op);
    1688      3071672 :       if (!is_gimple_assign (oe1def))
    1689      1116581 :         continue;
    1690      1955091 :       dcode = gimple_assign_rhs_code (oe1def);
    1691      3681535 :       if ((dcode != MULT_EXPR
    1692      1955091 :            && dcode != RDIV_EXPR)
    1693      1955091 :           || !is_reassociable_op (oe1def, dcode, loop))
    1694      1726444 :         continue;
    1695              : 
    1696       228647 :       bitmap_set_bit (candidates, i);
    1697       228647 :       nr_candidates++;
    1698              :     }
    1699              : 
    1700      2340835 :   if (nr_candidates < 2)
    1701              :     return false;
    1702              : 
    1703        16344 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1704              :     {
    1705            1 :       fprintf (dump_file, "searching for un-distribute opportunities ");
    1706            2 :       print_generic_expr (dump_file,
    1707            1 :         (*ops)[bitmap_first_set_bit (candidates)]->op, TDF_NONE);
    1708            1 :       fprintf (dump_file, " %d\n", nr_candidates);
    1709              :     }
    1710              : 
    1711              :   /* Build linearized sub-operand lists and the counting table.  */
    1712        16344 :   cvec.create (0);
    1713              : 
    1714        16344 :   hash_table<oecount_hasher> ctable (15);
    1715              : 
    1716              :   /* ??? Macro arguments cannot have multi-argument template types in
    1717              :      them.  This typedef is needed to workaround that limitation.  */
    1718        16344 :   typedef vec<operand_entry *> vec_operand_entry_t_heap;
    1719        32688 :   subops = XCNEWVEC (vec_operand_entry_t_heap, ops->length ());
    1720        72721 :   EXECUTE_IF_SET_IN_BITMAP (candidates, 0, i, sbi0)
    1721              :     {
    1722        40033 :       gimple *oedef;
    1723        40033 :       enum tree_code oecode;
    1724        40033 :       unsigned j;
    1725              : 
    1726        40033 :       oedef = SSA_NAME_DEF_STMT ((*ops)[i]->op);
    1727        40033 :       oecode = gimple_assign_rhs_code (oedef);
    1728        40033 :       linearize_expr_tree (&subops[i], oedef,
    1729              :                            associative_tree_code (oecode), false);
    1730              : 
    1731       200853 :       FOR_EACH_VEC_ELT (subops[i], j, oe1)
    1732              :         {
    1733        80754 :           oecount c;
    1734        80754 :           int *slot;
    1735        80754 :           int idx;
    1736        80754 :           c.oecode = oecode;
    1737        80754 :           c.cnt = 1;
    1738        80754 :           c.id = next_oecount_id++;
    1739        80754 :           c.op = oe1->op;
    1740        80754 :           cvec.safe_push (c);
    1741        80754 :           idx = cvec.length () + 41;
    1742        80754 :           slot = ctable.find_slot (idx, INSERT);
    1743        80754 :           if (!*slot)
    1744              :             {
    1745        79738 :               *slot = idx;
    1746              :             }
    1747              :           else
    1748              :             {
    1749         1016 :               cvec.pop ();
    1750         1016 :               cvec[*slot - 42].cnt++;
    1751              :             }
    1752              :         }
    1753              :     }
    1754              : 
    1755              :   /* Sort the counting table.  */
    1756        16344 :   cvec.qsort (oecount_cmp);
    1757              : 
    1758        16344 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1759              :     {
    1760            1 :       oecount *c;
    1761            1 :       fprintf (dump_file, "Candidates:\n");
    1762            5 :       FOR_EACH_VEC_ELT (cvec, j, c)
    1763              :         {
    1764            3 :           fprintf (dump_file, "  %u %s: ", c->cnt,
    1765            3 :                    c->oecode == MULT_EXPR
    1766              :                    ? "*" : c->oecode == RDIV_EXPR ? "/" : "?");
    1767            3 :           print_generic_expr (dump_file, c->op);
    1768            3 :           fprintf (dump_file, "\n");
    1769              :         }
    1770              :     }
    1771              : 
    1772              :   /* Process the (operand, code) pairs in order of most occurrence.  */
    1773        16344 :   auto_sbitmap candidates2 (length);
    1774        33398 :   while (!cvec.is_empty ())
    1775              :     {
    1776        16984 :       oecount *c = &cvec.last ();
    1777        16984 :       if (c->cnt < 2)
    1778              :         break;
    1779              : 
    1780              :       /* Now collect the operands in the outer chain that contain
    1781              :          the common operand in their inner chain.  */
    1782          710 :       bitmap_clear (candidates2);
    1783          710 :       nr_candidates2 = 0;
    1784         4718 :       EXECUTE_IF_SET_IN_BITMAP (candidates, 0, i, sbi0)
    1785              :         {
    1786         3298 :           gimple *oedef;
    1787         3298 :           enum tree_code oecode;
    1788         3298 :           unsigned j;
    1789         3298 :           tree op = (*ops)[i]->op;
    1790              : 
    1791              :           /* If we undistributed in this chain already this may be
    1792              :              a constant.  */
    1793         3298 :           if (TREE_CODE (op) != SSA_NAME)
    1794          762 :             continue;
    1795              : 
    1796         2536 :           oedef = SSA_NAME_DEF_STMT (op);
    1797         2536 :           oecode = gimple_assign_rhs_code (oedef);
    1798         2536 :           if (oecode != c->oecode)
    1799            2 :             continue;
    1800              : 
    1801         8935 :           FOR_EACH_VEC_ELT (subops[i], j, oe1)
    1802              :             {
    1803         4588 :               if (oe1->op == c->op)
    1804              :                 {
    1805         1218 :                   bitmap_set_bit (candidates2, i);
    1806         1218 :                   ++nr_candidates2;
    1807         1218 :                   break;
    1808              :                 }
    1809              :             }
    1810              :         }
    1811              : 
    1812          710 :       if (nr_candidates2 >= 2)
    1813              :         {
    1814          336 :           operand_entry *oe1, *oe2;
    1815          336 :           gimple *prod;
    1816          336 :           int first = bitmap_first_set_bit (candidates2);
    1817              : 
    1818              :           /* Build the new addition chain.  */
    1819          336 :           oe1 = (*ops)[first];
    1820          336 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1821              :             {
    1822            0 :               fprintf (dump_file, "Building (");
    1823            0 :               print_generic_expr (dump_file, oe1->op);
    1824              :             }
    1825          336 :           zero_one_operation (&oe1->op, c->oecode, c->op);
    1826         1226 :           EXECUTE_IF_SET_IN_BITMAP (candidates2, first+1, i, sbi0)
    1827              :             {
    1828          554 :               gimple *sum;
    1829          554 :               oe2 = (*ops)[i];
    1830          554 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1831              :                 {
    1832            0 :                   fprintf (dump_file, " + ");
    1833            0 :                   print_generic_expr (dump_file, oe2->op);
    1834              :                 }
    1835          554 :               zero_one_operation (&oe2->op, c->oecode, c->op);
    1836          554 :               sum = build_and_add_sum (TREE_TYPE (oe1->op),
    1837              :                                        oe1->op, oe2->op, opcode);
    1838          554 :               oe2->op = build_zero_cst (TREE_TYPE (oe2->op));
    1839          554 :               oe2->rank = 0;
    1840          554 :               oe1->op = gimple_get_lhs (sum);
    1841              :             }
    1842              : 
    1843              :           /* Apply the multiplication/division.  */
    1844          336 :           prod = build_and_add_sum (TREE_TYPE (oe1->op),
    1845              :                                     oe1->op, c->op, c->oecode);
    1846          336 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1847              :             {
    1848            0 :               fprintf (dump_file, ") %s ", c->oecode == MULT_EXPR ? "*" : "/");
    1849            0 :               print_generic_expr (dump_file, c->op);
    1850            0 :               fprintf (dump_file, "\n");
    1851              :             }
    1852              : 
    1853              :           /* Record it in the addition chain and disable further
    1854              :              undistribution with this op.  */
    1855          336 :           oe1->op = gimple_assign_lhs (prod);
    1856          336 :           oe1->rank = get_rank (oe1->op);
    1857          336 :           subops[first].release ();
    1858              : 
    1859          336 :           changed = true;
    1860              :         }
    1861              : 
    1862          710 :       cvec.pop ();
    1863              :     }
    1864              : 
    1865        71212 :   for (i = 0; i < ops->length (); ++i)
    1866        54868 :     subops[i].release ();
    1867        16344 :   free (subops);
    1868        16344 :   cvec.release ();
    1869              : 
    1870        16344 :   return changed;
    1871      2340835 : }
    1872              : 
    1873              : /* Pair to hold the information of one specific VECTOR_TYPE SSA_NAME:
    1874              :    first: element index for each relevant BIT_FIELD_REF.
    1875              :    second: the index of vec ops* for each relevant BIT_FIELD_REF.  */
    1876              : typedef std::pair<unsigned, unsigned> v_info_elem;
    1877         7995 : struct v_info {
    1878              :   tree vec_type;
    1879              :   auto_vec<v_info_elem, 32> vec;
    1880              : };
    1881              : typedef v_info *v_info_ptr;
    1882              : 
    1883              : /* Comparison function for qsort on VECTOR SSA_NAME trees by machine mode.  */
    1884              : static int
    1885        10558 : sort_by_mach_mode (const void *p_i, const void *p_j)
    1886              : {
    1887        10558 :   const tree tr1 = *((const tree *) p_i);
    1888        10558 :   const tree tr2 = *((const tree *) p_j);
    1889        10558 :   unsigned int mode1 = TYPE_MODE (TREE_TYPE (tr1));
    1890        10558 :   unsigned int mode2 = TYPE_MODE (TREE_TYPE (tr2));
    1891        10558 :   if (mode1 > mode2)
    1892              :     return 1;
    1893        10528 :   else if (mode1 < mode2)
    1894              :     return -1;
    1895        10491 :   if (SSA_NAME_VERSION (tr1) < SSA_NAME_VERSION (tr2))
    1896              :     return -1;
    1897         5054 :   else if (SSA_NAME_VERSION (tr1) > SSA_NAME_VERSION (tr2))
    1898         5054 :     return 1;
    1899              :   return 0;
    1900              : }
    1901              : 
    1902              : /* Cleanup hash map for VECTOR information.  */
    1903              : static void
    1904      4534112 : cleanup_vinfo_map (hash_map<tree, v_info_ptr> &info_map)
    1905              : {
    1906      4542107 :   for (hash_map<tree, v_info_ptr>::iterator it = info_map.begin ();
    1907      4542107 :        it != info_map.end (); ++it)
    1908              :     {
    1909         7995 :       v_info_ptr info = (*it).second;
    1910         7995 :       delete info;
    1911         7995 :       (*it).second = NULL;
    1912              :     }
    1913      4534112 : }
    1914              : 
    1915              : /* Perform un-distribution of BIT_FIELD_REF on VECTOR_TYPE.
    1916              :      V1[0] + V1[1] + ... + V1[k] + V2[0] + V2[1] + ... + V2[k] + ... Vn[k]
    1917              :    is transformed to
    1918              :      Vs = (V1 + V2 + ... + Vn)
    1919              :      Vs[0] + Vs[1] + ... + Vs[k]
    1920              : 
    1921              :    The basic steps are listed below:
    1922              : 
    1923              :     1) Check the addition chain *OPS by looking those summands coming from
    1924              :        VECTOR bit_field_ref on VECTOR type.  Put the information into
    1925              :        v_info_map for each satisfied summand, using VECTOR SSA_NAME as key.
    1926              : 
    1927              :     2) For each key (VECTOR SSA_NAME), validate all its BIT_FIELD_REFs are
    1928              :        continuous, they can cover the whole VECTOR perfectly without any holes.
    1929              :        Obtain one VECTOR list which contain candidates to be transformed.
    1930              : 
    1931              :     3) Sort the VECTOR list by machine mode of VECTOR type, for each group of
    1932              :        candidates with same mode, build the addition statements for them and
    1933              :        generate BIT_FIELD_REFs accordingly.
    1934              : 
    1935              :    TODO:
    1936              :        The current implementation requires the whole VECTORs should be fully
    1937              :        covered, but it can be extended to support partial, checking adjacent
    1938              :        but not fill the whole, it may need some cost model to define the
    1939              :        boundary to do or not.
    1940              : */
    1941              : static bool
    1942      4760050 : undistribute_bitref_for_vector (enum tree_code opcode,
    1943              :                                 vec<operand_entry *> *ops, struct loop *loop)
    1944              : {
    1945      4760050 :   if (ops->length () <= 1)
    1946              :     return false;
    1947              : 
    1948      4756712 :   if (opcode != PLUS_EXPR
    1949      4756712 :       && opcode != MULT_EXPR
    1950              :       && opcode != BIT_XOR_EXPR
    1951      1281494 :       && opcode != BIT_IOR_EXPR
    1952       883763 :       && opcode != BIT_AND_EXPR)
    1953              :     return false;
    1954              : 
    1955      4534112 :   hash_map<tree, v_info_ptr> v_info_map;
    1956      4534112 :   operand_entry *oe1;
    1957      4534112 :   unsigned i;
    1958              : 
    1959              :   /* Find those summands from VECTOR BIT_FIELD_REF in addition chain, put the
    1960              :      information into map.  */
    1961     18378541 :   FOR_EACH_VEC_ELT (*ops, i, oe1)
    1962              :     {
    1963      9310317 :       enum tree_code dcode;
    1964      9310317 :       gimple *oe1def;
    1965              : 
    1966      9310317 :       if (TREE_CODE (oe1->op) != SSA_NAME)
    1967      3265644 :         continue;
    1968      6044673 :       oe1def = SSA_NAME_DEF_STMT (oe1->op);
    1969      6044673 :       if (!is_gimple_assign (oe1def))
    1970      1527767 :         continue;
    1971      4516906 :       dcode = gimple_assign_rhs_code (oe1def);
    1972      4516906 :       if (dcode != BIT_FIELD_REF || !is_reassociable_op (oe1def, dcode, loop))
    1973      4462022 :         continue;
    1974              : 
    1975        54884 :       tree rhs = gimple_assign_rhs1 (oe1def);
    1976        54884 :       tree vec = TREE_OPERAND (rhs, 0);
    1977        54884 :       tree vec_type = TREE_TYPE (vec);
    1978              : 
    1979        54884 :       if (TREE_CODE (vec) != SSA_NAME || !VECTOR_TYPE_P (vec_type))
    1980        30419 :         continue;
    1981              : 
    1982              :       /* Ignore it if target machine can't support this VECTOR type.  */
    1983        24465 :       if (!VECTOR_MODE_P (TYPE_MODE (vec_type)))
    1984         5601 :         continue;
    1985              : 
    1986              :       /* Check const vector type, constrain BIT_FIELD_REF offset and size.  */
    1987        18864 :       if (!TYPE_VECTOR_SUBPARTS (vec_type).is_constant ())
    1988              :         continue;
    1989              : 
    1990        18864 :       if (VECTOR_TYPE_P (TREE_TYPE (rhs))
    1991        18864 :           || !is_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs))))
    1992         7810 :         continue;
    1993              : 
    1994              :       /* The type of BIT_FIELD_REF might not be equal to the element type of
    1995              :          the vector.  We want to use a vector type with element type the
    1996              :          same as the BIT_FIELD_REF and size the same as TREE_TYPE (vec).  */
    1997        11054 :       if (!useless_type_conversion_p (TREE_TYPE (rhs), TREE_TYPE (vec_type)))
    1998              :         {
    1999         1901 :           machine_mode simd_mode;
    2000         1901 :           unsigned HOST_WIDE_INT size, nunits;
    2001         1901 :           unsigned HOST_WIDE_INT elem_size
    2002         1901 :             = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
    2003         3802 :           if (!GET_MODE_BITSIZE (TYPE_MODE (vec_type)).is_constant (&size))
    2004      9299697 :             continue;
    2005         1901 :           if (size <= elem_size || (size % elem_size) != 0)
    2006            0 :             continue;
    2007         1901 :           nunits = size / elem_size;
    2008         1901 :           if (!mode_for_vector (SCALAR_TYPE_MODE (TREE_TYPE (rhs)),
    2009         1901 :                                 nunits).exists (&simd_mode))
    2010            0 :             continue;
    2011         1901 :           vec_type = build_vector_type_for_mode (TREE_TYPE (rhs), simd_mode);
    2012              : 
    2013              :           /* Ignore it if target machine can't support this VECTOR type.  */
    2014         1901 :           if (!VECTOR_MODE_P (TYPE_MODE (vec_type)))
    2015            0 :             continue;
    2016              : 
    2017              :           /* Check const vector type, constrain BIT_FIELD_REF offset and
    2018              :              size.  */
    2019         1901 :           if (!TYPE_VECTOR_SUBPARTS (vec_type).is_constant ())
    2020              :             continue;
    2021              : 
    2022         3802 :           if (maybe_ne (GET_MODE_SIZE (TYPE_MODE (vec_type)),
    2023         3802 :                         GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (vec)))))
    2024            0 :             continue;
    2025              :         }
    2026              : 
    2027        11054 :       tree elem_type = TREE_TYPE (vec_type);
    2028        11054 :       unsigned HOST_WIDE_INT elem_size = tree_to_uhwi (TYPE_SIZE (elem_type));
    2029        11054 :       if (maybe_ne (bit_field_size (rhs), elem_size))
    2030            0 :         continue;
    2031              : 
    2032        11054 :       unsigned idx;
    2033        11054 :       if (!constant_multiple_p (bit_field_offset (rhs), elem_size, &idx))
    2034            0 :         continue;
    2035              : 
    2036              :       /* Ignore it if target machine can't support this type of VECTOR
    2037              :          operation.  */
    2038        11054 :       optab op_tab = optab_for_tree_code (opcode, vec_type, optab_vector);
    2039        11054 :       if (optab_handler (op_tab, TYPE_MODE (vec_type)) == CODE_FOR_nothing)
    2040          434 :         continue;
    2041              : 
    2042        10620 :       bool existed;
    2043        10620 :       v_info_ptr &info = v_info_map.get_or_insert (vec, &existed);
    2044        10620 :       if (!existed)
    2045              :         {
    2046         7995 :           info = new v_info;
    2047         7995 :           info->vec_type = vec_type;
    2048              :         }
    2049         2625 :       else if (!types_compatible_p (vec_type, info->vec_type))
    2050            0 :         continue;
    2051        10620 :       info->vec.safe_push (std::make_pair (idx, i));
    2052              :     }
    2053              : 
    2054              :   /* At least two VECTOR to combine.  */
    2055      4534112 :   if (v_info_map.elements () <= 1)
    2056              :     {
    2057      4533863 :       cleanup_vinfo_map (v_info_map);
    2058      4533863 :       return false;
    2059              :     }
    2060              : 
    2061              :   /* Verify all VECTOR candidates by checking two conditions:
    2062              :        1) sorted offsets are adjacent, no holes.
    2063              :        2) can fill the whole VECTOR perfectly.
    2064              :      And add the valid candidates to a vector for further handling.  */
    2065          249 :   auto_vec<tree> valid_vecs (v_info_map.elements ());
    2066         1211 :   for (hash_map<tree, v_info_ptr>::iterator it = v_info_map.begin ();
    2067         1211 :        it != v_info_map.end (); ++it)
    2068              :     {
    2069          962 :       tree cand_vec = (*it).first;
    2070          962 :       v_info_ptr cand_info = (*it).second;
    2071          962 :       unsigned int num_elems
    2072          962 :         = TYPE_VECTOR_SUBPARTS (cand_info->vec_type).to_constant ();
    2073         1924 :       if (cand_info->vec.length () != num_elems)
    2074          597 :         continue;
    2075          365 :       sbitmap holes = sbitmap_alloc (num_elems);
    2076          365 :       bitmap_ones (holes);
    2077          365 :       bool valid = true;
    2078          365 :       v_info_elem *curr;
    2079         2124 :       FOR_EACH_VEC_ELT (cand_info->vec, i, curr)
    2080              :         {
    2081         1394 :           if (!bitmap_bit_p (holes, curr->first))
    2082              :             {
    2083              :               valid = false;
    2084              :               break;
    2085              :             }
    2086              :           else
    2087         1394 :             bitmap_clear_bit (holes, curr->first);
    2088              :         }
    2089          365 :       if (valid && bitmap_empty_p (holes))
    2090          365 :         valid_vecs.quick_push (cand_vec);
    2091          365 :       sbitmap_free (holes);
    2092              :     }
    2093              : 
    2094              :   /* At least two VECTOR to combine.  */
    2095          249 :   if (valid_vecs.length () <= 1)
    2096              :     {
    2097          213 :       cleanup_vinfo_map (v_info_map);
    2098          213 :       return false;
    2099              :     }
    2100              : 
    2101           36 :   valid_vecs.qsort (sort_by_mach_mode);
    2102              :   /* Go through all candidates by machine mode order, query the mode_to_total
    2103              :      to get the total number for each mode and skip the single one.  */
    2104          112 :   for (unsigned i = 0; i < valid_vecs.length () - 1; ++i)
    2105              :     {
    2106           40 :       tree tvec = valid_vecs[i];
    2107           40 :       enum machine_mode mode = TYPE_MODE (TREE_TYPE (tvec));
    2108              : 
    2109              :       /* Skip modes with only a single candidate.  */
    2110           40 :       if (TYPE_MODE (TREE_TYPE (valid_vecs[i + 1])) != mode)
    2111            0 :         continue;
    2112              : 
    2113           40 :       unsigned int idx, j;
    2114           40 :       gimple *sum = NULL;
    2115           40 :       tree sum_vec = tvec;
    2116           40 :       v_info_ptr info_ptr = *(v_info_map.get (tvec));
    2117           40 :       v_info_elem *elem;
    2118           40 :       tree vec_type = info_ptr->vec_type;
    2119              : 
    2120              :       /* Build the sum for all candidates with same mode.  */
    2121          323 :       do
    2122              :         {
    2123          969 :           sum = build_and_add_sum (vec_type, sum_vec,
    2124          323 :                                    valid_vecs[i + 1], opcode);
    2125              :           /* Update the operands only after build_and_add_sum,
    2126              :              so that we don't have to repeat the placement algorithm
    2127              :              of build_and_add_sum.  */
    2128          323 :           if (sum_vec == tvec
    2129          323 :               && !useless_type_conversion_p (vec_type, TREE_TYPE (sum_vec)))
    2130              :             {
    2131           18 :               gimple_stmt_iterator gsi = gsi_for_stmt (sum);
    2132           18 :               tree vce = build1 (VIEW_CONVERT_EXPR, vec_type, sum_vec);
    2133           18 :               tree lhs = make_ssa_name (vec_type);
    2134           18 :               gimple *g = gimple_build_assign (lhs, VIEW_CONVERT_EXPR, vce);
    2135           18 :               gimple_set_uid (g, gimple_uid (sum));
    2136           18 :               gsi_insert_before (&gsi, g, GSI_NEW_STMT);
    2137           18 :               gimple_assign_set_rhs1 (sum, lhs);
    2138           18 :               update_stmt (sum);
    2139              :             }
    2140          323 :           if (!useless_type_conversion_p (vec_type,
    2141          323 :                                           TREE_TYPE (valid_vecs[i + 1])))
    2142              :             {
    2143          270 :               gimple_stmt_iterator gsi = gsi_for_stmt (sum);
    2144          810 :               tree vce = build1 (VIEW_CONVERT_EXPR, vec_type,
    2145          270 :                                  valid_vecs[i + 1]);
    2146          270 :               tree lhs = make_ssa_name (vec_type);
    2147          270 :               gimple *g = gimple_build_assign (lhs, VIEW_CONVERT_EXPR, vce);
    2148          270 :               gimple_set_uid (g, gimple_uid (sum));
    2149          270 :               gsi_insert_before (&gsi, g, GSI_NEW_STMT);
    2150          270 :               gimple_assign_set_rhs2 (sum, lhs);
    2151          270 :               update_stmt (sum);
    2152              :             }
    2153          323 :           sum_vec = gimple_get_lhs (sum);
    2154          323 :           info_ptr = *(v_info_map.get (valid_vecs[i + 1]));
    2155          323 :           gcc_assert (types_compatible_p (vec_type, info_ptr->vec_type));
    2156              :           /* Update those related ops of current candidate VECTOR.  */
    2157         1569 :           FOR_EACH_VEC_ELT (info_ptr->vec, j, elem)
    2158              :             {
    2159         1246 :               idx = elem->second;
    2160         1246 :               gimple *def = SSA_NAME_DEF_STMT ((*ops)[idx]->op);
    2161              :               /* Set this then op definition will get DCEd later.  */
    2162         1246 :               gimple_set_visited (def, true);
    2163         1246 :               if (opcode == PLUS_EXPR
    2164         1246 :                   || opcode == BIT_XOR_EXPR
    2165           96 :                   || opcode == BIT_IOR_EXPR)
    2166         1186 :                 (*ops)[idx]->op = build_zero_cst (TREE_TYPE ((*ops)[idx]->op));
    2167           60 :               else if (opcode == MULT_EXPR)
    2168           24 :                 (*ops)[idx]->op = build_one_cst (TREE_TYPE ((*ops)[idx]->op));
    2169              :               else
    2170              :                 {
    2171           36 :                   gcc_assert (opcode == BIT_AND_EXPR);
    2172           36 :                   (*ops)[idx]->op
    2173           36 :                     = build_all_ones_cst (TREE_TYPE ((*ops)[idx]->op));
    2174              :                 }
    2175         1246 :               (*ops)[idx]->rank = 0;
    2176              :             }
    2177          323 :           if (dump_file && (dump_flags & TDF_DETAILS))
    2178              :             {
    2179            0 :               fprintf (dump_file, "Generating addition -> ");
    2180            0 :               print_gimple_stmt (dump_file, sum, 0);
    2181              :             }
    2182          323 :           i++;
    2183              :         }
    2184          323 :       while ((i < valid_vecs.length () - 1)
    2185          363 :              && TYPE_MODE (TREE_TYPE (valid_vecs[i + 1])) == mode);
    2186              : 
    2187              :       /* Referring to first valid VECTOR with this mode, generate the
    2188              :          BIT_FIELD_REF statements accordingly.  */
    2189           40 :       info_ptr = *(v_info_map.get (tvec));
    2190           40 :       gcc_assert (sum);
    2191           40 :       tree elem_type = TREE_TYPE (vec_type);
    2192          224 :       FOR_EACH_VEC_ELT (info_ptr->vec, j, elem)
    2193              :         {
    2194          144 :           idx = elem->second;
    2195          144 :           tree dst = make_ssa_name (elem_type);
    2196          144 :           tree pos = bitsize_int (elem->first
    2197              :                                   * tree_to_uhwi (TYPE_SIZE (elem_type)));
    2198          144 :           tree bfr = build3 (BIT_FIELD_REF, elem_type, sum_vec,
    2199          144 :                              TYPE_SIZE (elem_type), pos);
    2200          144 :           gimple *gs = gimple_build_assign (dst, BIT_FIELD_REF, bfr);
    2201          144 :           insert_stmt_after (gs, sum);
    2202          144 :           gimple *def = SSA_NAME_DEF_STMT ((*ops)[idx]->op);
    2203              :           /* Set this then op definition will get DCEd later.  */
    2204          144 :           gimple_set_visited (def, true);
    2205          144 :           (*ops)[idx]->op = gimple_assign_lhs (gs);
    2206          144 :           (*ops)[idx]->rank = get_rank ((*ops)[idx]->op);
    2207          144 :           if (dump_file && (dump_flags & TDF_DETAILS))
    2208              :             {
    2209            0 :               fprintf (dump_file, "Generating bit_field_ref -> ");
    2210            0 :               print_gimple_stmt (dump_file, gs, 0);
    2211              :             }
    2212              :         }
    2213              :     }
    2214              : 
    2215           36 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2216            0 :     fprintf (dump_file, "undistributiong bit_field_ref for vector done.\n");
    2217              : 
    2218           36 :   cleanup_vinfo_map (v_info_map);
    2219              : 
    2220           36 :   return true;
    2221      4534361 : }
    2222              : 
    2223              : /* If OPCODE is BIT_IOR_EXPR or BIT_AND_EXPR and CURR is a comparison
    2224              :    expression, examine the other OPS to see if any of them are comparisons
    2225              :    of the same values, which we may be able to combine or eliminate.
    2226              :    For example, we can rewrite (a < b) | (a == b) as (a <= b).  */
    2227              : 
    2228              : static bool
    2229      9766679 : eliminate_redundant_comparison (enum tree_code opcode,
    2230              :                                 vec<operand_entry *> *ops,
    2231              :                                 unsigned int currindex,
    2232              :                                 operand_entry *curr)
    2233              : {
    2234      9766679 :   tree op1, op2;
    2235      9766679 :   enum tree_code lcode, rcode;
    2236      9766679 :   gimple *def1, *def2;
    2237      9766679 :   int i;
    2238      9766679 :   operand_entry *oe;
    2239              : 
    2240      9766679 :   if (opcode != BIT_IOR_EXPR && opcode != BIT_AND_EXPR)
    2241              :     return false;
    2242              : 
    2243              :   /* Check that CURR is a comparison.  */
    2244      2047157 :   if (TREE_CODE (curr->op) != SSA_NAME)
    2245              :     return false;
    2246      1542981 :   def1 = SSA_NAME_DEF_STMT (curr->op);
    2247      1542981 :   if (!is_gimple_assign (def1))
    2248              :     return false;
    2249      1331033 :   lcode = gimple_assign_rhs_code (def1);
    2250      1331033 :   if (TREE_CODE_CLASS (lcode) != tcc_comparison)
    2251              :     return false;
    2252       537604 :   op1 = gimple_assign_rhs1 (def1);
    2253       537604 :   op2 = gimple_assign_rhs2 (def1);
    2254              : 
    2255              :   /* Now look for a similar comparison in the remaining OPS.  */
    2256       897200 :   for (i = currindex + 1; ops->iterate (i, &oe); i++)
    2257              :     {
    2258       359830 :       tree t;
    2259              : 
    2260       359830 :       if (TREE_CODE (oe->op) != SSA_NAME)
    2261           41 :         continue;
    2262       359789 :       def2 = SSA_NAME_DEF_STMT (oe->op);
    2263       359789 :       if (!is_gimple_assign (def2))
    2264         7450 :         continue;
    2265       352339 :       rcode = gimple_assign_rhs_code (def2);
    2266       352339 :       if (TREE_CODE_CLASS (rcode) != tcc_comparison)
    2267         6544 :         continue;
    2268              : 
    2269              :       /* If we got here, we have a match.  See if we can combine the
    2270              :          two comparisons.  */
    2271       345795 :       tree type = TREE_TYPE (gimple_assign_lhs (def1));
    2272       345795 :       if (opcode == BIT_IOR_EXPR)
    2273       196778 :         t = maybe_fold_or_comparisons (type, TRUTH_OR_EXPR,
    2274              :                                        lcode, op1, op2,
    2275              :                                        rcode, gimple_assign_rhs1 (def2),
    2276              :                                        gimple_assign_rhs2 (def2));
    2277              :       else
    2278       149017 :         t = maybe_fold_and_comparisons (type, TRUTH_AND_EXPR,
    2279              :                                         lcode, op1, op2,
    2280              :                                         rcode, gimple_assign_rhs1 (def2),
    2281              :                                         gimple_assign_rhs2 (def2));
    2282       345795 :       if (!t)
    2283       345513 :         continue;
    2284              : 
    2285              :       /* maybe_fold_and_comparisons and maybe_fold_or_comparisons
    2286              :          always give us a boolean_type_node value back.  If the original
    2287              :          BIT_AND_EXPR or BIT_IOR_EXPR was of a wider integer type,
    2288              :          we need to convert.  */
    2289          282 :       if (!useless_type_conversion_p (TREE_TYPE (curr->op), TREE_TYPE (t)))
    2290              :         {
    2291            2 :           if (!fold_convertible_p (TREE_TYPE (curr->op), t))
    2292            0 :             continue;
    2293            2 :           t = fold_convert (TREE_TYPE (curr->op), t);
    2294              :         }
    2295              : 
    2296          282 :       if (TREE_CODE (t) != INTEGER_CST
    2297          282 :           && !operand_equal_p (t, curr->op, 0))
    2298              :         {
    2299          278 :           enum tree_code subcode;
    2300          278 :           tree newop1, newop2;
    2301          278 :           if (!COMPARISON_CLASS_P (t))
    2302           48 :             continue;
    2303          250 :           extract_ops_from_tree (t, &subcode, &newop1, &newop2);
    2304          250 :           STRIP_USELESS_TYPE_CONVERSION (newop1);
    2305          250 :           STRIP_USELESS_TYPE_CONVERSION (newop2);
    2306          250 :           if (!is_gimple_val (newop1) || !is_gimple_val (newop2))
    2307            0 :             continue;
    2308          250 :           if (lcode == TREE_CODE (t)
    2309          182 :               && operand_equal_p (op1, newop1, 0)
    2310          432 :               && operand_equal_p (op2, newop2, 0))
    2311          137 :             t = curr->op;
    2312          133 :           else if ((TREE_CODE (newop1) == SSA_NAME
    2313          113 :                     && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (newop1))
    2314          206 :                    || (TREE_CODE (newop2) == SSA_NAME
    2315           53 :                        && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (newop2)))
    2316           20 :             continue;
    2317              :         }
    2318              : 
    2319          234 :       if (dump_file && (dump_flags & TDF_DETAILS))
    2320              :         {
    2321            6 :           fprintf (dump_file, "Equivalence: ");
    2322            6 :           print_generic_expr (dump_file, curr->op);
    2323            6 :           fprintf (dump_file, " %s ", op_symbol_code (opcode));
    2324            6 :           print_generic_expr (dump_file, oe->op);
    2325            6 :           fprintf (dump_file, " -> ");
    2326            6 :           print_generic_expr (dump_file, t);
    2327            6 :           fprintf (dump_file, "\n");
    2328              :         }
    2329              : 
    2330              :       /* Now we can delete oe, as it has been subsumed by the new combined
    2331              :          expression t.  */
    2332          234 :       ops->ordered_remove (i);
    2333          234 :       reassociate_stats.ops_eliminated ++;
    2334              : 
    2335              :       /* If t is the same as curr->op, we're done.  Otherwise we must
    2336              :          replace curr->op with t.  Special case is if we got a constant
    2337              :          back, in which case we add it to the end instead of in place of
    2338              :          the current entry.  */
    2339          234 :       if (TREE_CODE (t) == INTEGER_CST)
    2340              :         {
    2341            4 :           ops->ordered_remove (currindex);
    2342            4 :           add_to_ops_vec (ops, t);
    2343              :         }
    2344          230 :       else if (!operand_equal_p (t, curr->op, 0))
    2345              :         {
    2346           93 :           gimple *sum;
    2347           93 :           enum tree_code subcode;
    2348           93 :           tree newop1;
    2349           93 :           tree newop2;
    2350           93 :           gcc_assert (COMPARISON_CLASS_P (t));
    2351           93 :           extract_ops_from_tree (t, &subcode, &newop1, &newop2);
    2352           93 :           STRIP_USELESS_TYPE_CONVERSION (newop1);
    2353           93 :           STRIP_USELESS_TYPE_CONVERSION (newop2);
    2354           93 :           gcc_checking_assert (is_gimple_val (newop1)
    2355              :                                && is_gimple_val (newop2));
    2356           93 :           sum = build_and_add_sum (TREE_TYPE (t), newop1, newop2, subcode);
    2357           93 :           curr->op = gimple_get_lhs (sum);
    2358              :         }
    2359              :       return true;
    2360              :     }
    2361              : 
    2362              :   return false;
    2363              : }
    2364              : 
    2365              : 
    2366              : /* Transform repeated addition of same values into multiply with
    2367              :    constant.  */
    2368              : static bool
    2369      2343977 : transform_add_to_multiply (vec<operand_entry *> *ops)
    2370              : {
    2371      2343977 :   operand_entry *oe;
    2372      2343977 :   tree op = NULL_TREE;
    2373      2343977 :   int j;
    2374      2343977 :   int i, start = -1, end = 0, count = 0;
    2375      2343977 :   auto_vec<std::pair <int, int> > indxs;
    2376      2343977 :   bool changed = false;
    2377              : 
    2378      2343977 :   if (!INTEGRAL_TYPE_P (TREE_TYPE ((*ops)[0]->op))
    2379        69009 :       && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE ((*ops)[0]->op))
    2380        31859 :           || !flag_unsafe_math_optimizations))
    2381              :     return false;
    2382              : 
    2383              :   /* Look for repeated operands.  */
    2384      7081557 :   FOR_EACH_VEC_ELT (*ops, i, oe)
    2385              :     {
    2386      4774791 :       if (start == -1)
    2387              :         {
    2388      2306766 :           count = 1;
    2389      2306766 :           op = oe->op;
    2390      2306766 :           start = i;
    2391              :         }
    2392      2468025 :       else if (operand_equal_p (oe->op, op, 0))
    2393              :         {
    2394          155 :           count++;
    2395          155 :           end = i;
    2396              :         }
    2397              :       else
    2398              :         {
    2399      2467870 :           if (count > 1)
    2400           47 :             indxs.safe_push (std::make_pair (start, end));
    2401      2467870 :           count = 1;
    2402      2467870 :           op = oe->op;
    2403      2467870 :           start = i;
    2404              :         }
    2405              :     }
    2406              : 
    2407      2306766 :   if (count > 1)
    2408           38 :     indxs.safe_push (std::make_pair (start, end));
    2409              : 
    2410      2306924 :   for (j = indxs.length () - 1; j >= 0; --j)
    2411              :     {
    2412              :       /* Convert repeated operand addition to multiplication.  */
    2413           85 :       start = indxs[j].first;
    2414           85 :       end = indxs[j].second;
    2415           85 :       op = (*ops)[start]->op;
    2416           85 :       count = end - start + 1;
    2417          325 :       for (i = end; i >= start; --i)
    2418          240 :         ops->unordered_remove (i);
    2419           85 :       tree tmp = make_ssa_name (TREE_TYPE (op));
    2420           85 :       tree cst = build_int_cst (integer_type_node, count);
    2421           85 :       gassign *mul_stmt
    2422           85 :         = gimple_build_assign (tmp, MULT_EXPR,
    2423           85 :                                op, fold_convert (TREE_TYPE (op), cst));
    2424           85 :       gimple_set_visited (mul_stmt, true);
    2425           85 :       add_to_ops_vec (ops, tmp, mul_stmt);
    2426           85 :       changed = true;
    2427              :     }
    2428              : 
    2429              :   return changed;
    2430      2343977 : }
    2431              : 
    2432              : 
    2433              : /* Perform various identities and other optimizations on the list of
    2434              :    operand entries, stored in OPS.  The tree code for the binary
    2435              :    operation between all the operands is OPCODE.  */
    2436              : 
    2437              : static void
    2438      4760401 : optimize_ops_list (enum tree_code opcode,
    2439              :                    vec<operand_entry *> *ops)
    2440              : {
    2441      4778530 :   unsigned int length = ops->length ();
    2442      4778530 :   unsigned int i;
    2443      4778530 :   operand_entry *oe;
    2444      9555147 :   operand_entry *oelast = NULL;
    2445      9555147 :   bool iterate = false;
    2446              : 
    2447      4778530 :   if (length == 1)
    2448              :     return;
    2449              : 
    2450      4776617 :   oelast = ops->last ();
    2451              : 
    2452              :   /* If the last two are constants, pop the constants off, merge them
    2453              :      and try the next two.  */
    2454      4776617 :   if (oelast->rank == 0 && is_gimple_min_invariant (oelast->op))
    2455              :     {
    2456      3456002 :       operand_entry *oelm1 = (*ops)[length - 2];
    2457              : 
    2458      3456002 :       if (oelm1->rank == 0
    2459        14433 :           && is_gimple_min_invariant (oelm1->op)
    2460      3470435 :           && useless_type_conversion_p (TREE_TYPE (oelm1->op),
    2461        14433 :                                        TREE_TYPE (oelast->op)))
    2462              :         {
    2463        14433 :           tree folded = fold_binary (opcode, TREE_TYPE (oelm1->op),
    2464              :                                      oelm1->op, oelast->op);
    2465              : 
    2466        14433 :           if (folded && is_gimple_min_invariant (folded))
    2467              :             {
    2468        14383 :               if (dump_file && (dump_flags & TDF_DETAILS))
    2469            0 :                 fprintf (dump_file, "Merging constants\n");
    2470              : 
    2471        14383 :               ops->pop ();
    2472        14383 :               ops->pop ();
    2473              : 
    2474        14383 :               add_to_ops_vec (ops, folded);
    2475        14383 :               reassociate_stats.constants_eliminated++;
    2476              : 
    2477        14383 :               optimize_ops_list (opcode, ops);
    2478        14383 :               return;
    2479              :             }
    2480              :         }
    2481              :     }
    2482              : 
    2483      4762234 :   eliminate_using_constants (opcode, ops);
    2484      4762234 :   oelast = NULL;
    2485              : 
    2486     19294791 :   for (i = 0; ops->iterate (i, &oe);)
    2487              :     {
    2488      9770325 :       bool done = false;
    2489              : 
    2490      9770325 :       if (eliminate_not_pairs (opcode, ops, i, oe))
    2491            2 :         return;
    2492      9770324 :       if (eliminate_duplicate_pair (opcode, ops, &done, i, oe, oelast)
    2493      9770175 :           || (!done && eliminate_plus_minus_pair (opcode, ops, i, oe))
    2494     19537003 :           || (!done && eliminate_redundant_comparison (opcode, ops, i, oe)))
    2495              :         {
    2496         3879 :           if (done)
    2497              :             return;
    2498         3878 :           iterate = true;
    2499         3878 :           oelast = NULL;
    2500         3878 :           continue;
    2501              :         }
    2502      9766445 :       oelast = oe;
    2503      9766445 :       i++;
    2504              :     }
    2505              : 
    2506      4762232 :   if (iterate)
    2507              :     optimize_ops_list (opcode, ops);
    2508              : }
    2509              : 
    2510              : /* The following functions are subroutines to optimize_range_tests and allow
    2511              :    it to try to change a logical combination of comparisons into a range
    2512              :    test.
    2513              : 
    2514              :    For example, both
    2515              :         X == 2 || X == 5 || X == 3 || X == 4
    2516              :    and
    2517              :         X >= 2 && X <= 5
    2518              :    are converted to
    2519              :         (unsigned) (X - 2) <= 3
    2520              : 
    2521              :    For more information see comments above fold_test_range in fold-const.cc,
    2522              :    this implementation is for GIMPLE.  */
    2523              : 
    2524              : 
    2525              : 
    2526              : /* Dump the range entry R to FILE, skipping its expression if SKIP_EXP.  */
    2527              : 
    2528              : void
    2529          141 : dump_range_entry (FILE *file, struct range_entry *r, bool skip_exp)
    2530              : {
    2531          141 :   if (!skip_exp)
    2532           59 :     print_generic_expr (file, r->exp);
    2533          251 :   fprintf (file, " %c[", r->in_p ? '+' : '-');
    2534          141 :   print_generic_expr (file, r->low);
    2535          141 :   fputs (", ", file);
    2536          141 :   print_generic_expr (file, r->high);
    2537          141 :   fputc (']', file);
    2538          141 : }
    2539              : 
    2540              : /* Dump the range entry R to STDERR.  */
    2541              : 
    2542              : DEBUG_FUNCTION void
    2543            0 : debug_range_entry (struct range_entry *r)
    2544              : {
    2545            0 :   dump_range_entry (stderr, r, false);
    2546            0 :   fputc ('\n', stderr);
    2547            0 : }
    2548              : 
    2549              : /* This is similar to make_range in fold-const.cc, but on top of
    2550              :    GIMPLE instead of trees.  If EXP is non-NULL, it should be
    2551              :    an SSA_NAME and STMT argument is ignored, otherwise STMT
    2552              :    argument should be a GIMPLE_COND.  */
    2553              : 
    2554              : void
    2555      5722732 : init_range_entry (struct range_entry *r, tree exp, gimple *stmt)
    2556              : {
    2557      5722732 :   int in_p;
    2558      5722732 :   tree low, high;
    2559      5722732 :   bool is_bool;
    2560              : 
    2561      5722732 :   r->exp = NULL_TREE;
    2562      5722732 :   r->in_p = false;
    2563      5722732 :   r->low = NULL_TREE;
    2564      5722732 :   r->high = NULL_TREE;
    2565      5722732 :   if (exp != NULL_TREE
    2566      5722732 :       && (TREE_CODE (exp) != SSA_NAME || !INTEGRAL_TYPE_P (TREE_TYPE (exp))))
    2567       802153 :     return;
    2568              : 
    2569              :   /* Start with simply saying "EXP != 0" and then look at the code of EXP
    2570              :      and see if we can refine the range.  Some of the cases below may not
    2571              :      happen, but it doesn't seem worth worrying about this.  We "continue"
    2572              :      the outer loop when we've changed something; otherwise we "break"
    2573              :      the switch, which will "break" the while.  */
    2574      5225087 :   low = exp ? build_int_cst (TREE_TYPE (exp), 0) : boolean_false_node;
    2575      5225087 :   high = low;
    2576      5225087 :   in_p = 0;
    2577      5225087 :   is_bool = false;
    2578      5225087 :   if (exp == NULL_TREE)
    2579      5224863 :     is_bool = true;
    2580      1592158 :   else if (TYPE_PRECISION (TREE_TYPE (exp)) == 1)
    2581              :     {
    2582       670181 :       if (TYPE_UNSIGNED (TREE_TYPE (exp)))
    2583              :         is_bool = true;
    2584              :       else
    2585              :         return;
    2586              :     }
    2587       921977 :   else if (TREE_CODE (TREE_TYPE (exp)) == BOOLEAN_TYPE)
    2588      4302886 :     is_bool = true;
    2589              : 
    2590      8317797 :   while (1)
    2591              :     {
    2592      8317797 :       enum tree_code code;
    2593      8317797 :       tree arg0, arg1, exp_type;
    2594      8317797 :       tree nexp;
    2595      8317797 :       location_t loc;
    2596              : 
    2597      8317797 :       if (exp != NULL_TREE)
    2598              :         {
    2599      4684868 :           if (TREE_CODE (exp) != SSA_NAME
    2600      4684868 :               || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (exp))
    2601              :             break;
    2602              : 
    2603      4684868 :           stmt = SSA_NAME_DEF_STMT (exp);
    2604      4684868 :           if (!is_gimple_assign (stmt))
    2605              :             break;
    2606              : 
    2607      2863673 :           code = gimple_assign_rhs_code (stmt);
    2608      2863673 :           arg0 = gimple_assign_rhs1 (stmt);
    2609      2863673 :           arg1 = gimple_assign_rhs2 (stmt);
    2610      2863673 :           exp_type = TREE_TYPE (exp);
    2611              :         }
    2612              :       else
    2613              :         {
    2614      3632929 :           code = gimple_cond_code (stmt);
    2615      3632929 :           arg0 = gimple_cond_lhs (stmt);
    2616      3632929 :           arg1 = gimple_cond_rhs (stmt);
    2617      3632929 :           exp_type = boolean_type_node;
    2618              :         }
    2619              : 
    2620      6496602 :       if (TREE_CODE (arg0) != SSA_NAME
    2621      5159563 :           || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg0)
    2622     11655738 :           || ssa_name_maybe_undef_p (arg0))
    2623              :         break;
    2624      5152342 :       loc = gimple_location (stmt);
    2625      5152342 :       switch (code)
    2626              :         {
    2627        34162 :         case BIT_NOT_EXPR:
    2628        34162 :           if (TREE_CODE (TREE_TYPE (exp)) == BOOLEAN_TYPE
    2629              :               /* Ensure the range is either +[-,0], +[0,0],
    2630              :                  -[-,0], -[0,0] or +[1,-], +[1,1], -[1,-] or
    2631              :                  -[1,1].  If it is e.g. +[-,-] or -[-,-]
    2632              :                  or similar expression of unconditional true or
    2633              :                  false, it should not be negated.  */
    2634        34162 :               && ((high && integer_zerop (high))
    2635            0 :                   || (low && integer_onep (low))))
    2636              :             {
    2637         6364 :               in_p = !in_p;
    2638         6364 :               exp = arg0;
    2639         6364 :               continue;
    2640              :             }
    2641              :           break;
    2642         1873 :         case SSA_NAME:
    2643         1873 :           exp = arg0;
    2644         1873 :           continue;
    2645       244651 :         CASE_CONVERT:
    2646       244651 :           if (is_bool)
    2647              :             {
    2648       126439 :               if ((TYPE_PRECISION (exp_type) == 1
    2649       119981 :                    || TREE_CODE (exp_type) == BOOLEAN_TYPE)
    2650       126439 :                   && TYPE_PRECISION (TREE_TYPE (arg0)) > 1)
    2651              :                 return;
    2652              :             }
    2653       118212 :           else if (TYPE_PRECISION (TREE_TYPE (arg0)) == 1)
    2654              :             {
    2655         4501 :               if (TYPE_UNSIGNED (TREE_TYPE (arg0)))
    2656              :                 is_bool = true;
    2657              :               else
    2658              :                 return;
    2659              :             }
    2660       113711 :           else if (TREE_CODE (TREE_TYPE (arg0)) == BOOLEAN_TYPE)
    2661       128191 :             is_bool = true;
    2662       241902 :           goto do_default;
    2663              :         case EQ_EXPR:
    2664              :         case NE_EXPR:
    2665              :         case LT_EXPR:
    2666              :         case LE_EXPR:
    2667              :         case GE_EXPR:
    2668              :         case GT_EXPR:
    2669              :           is_bool = true;
    2670              :           /* FALLTHRU */
    2671       664299 :         default:
    2672       664299 :           if (!is_bool)
    2673              :             return;
    2674       362764 :         do_default:
    2675      4812023 :           nexp = make_range_step (loc, code, arg0, arg1, exp_type,
    2676              :                                   &low, &high, &in_p);
    2677      4812023 :           if (nexp != NULL_TREE)
    2678              :             {
    2679      3084697 :               exp = nexp;
    2680      3084697 :               gcc_assert (TREE_CODE (exp) == SSA_NAME);
    2681      3084697 :               continue;
    2682              :             }
    2683              :           break;
    2684              :         }
    2685              :       break;
    2686              :     }
    2687      4920579 :   if (is_bool)
    2688              :     {
    2689      4304638 :       r->exp = exp;
    2690      4304638 :       r->in_p = in_p;
    2691      4304638 :       r->low = low;
    2692      4304638 :       r->high = high;
    2693              :     }
    2694              : }
    2695              : 
    2696              : /* Comparison function for qsort.  Sort entries
    2697              :    without SSA_NAME exp first, then with SSA_NAMEs sorted
    2698              :    by increasing SSA_NAME_VERSION, and for the same SSA_NAMEs
    2699              :    by increasing ->low and if ->low is the same, by increasing
    2700              :    ->high.  ->low == NULL_TREE means minimum, ->high == NULL_TREE
    2701              :    maximum.  */
    2702              : 
    2703              : static int
    2704      6173417 : range_entry_cmp (const void *a, const void *b)
    2705              : {
    2706      6173417 :   const struct range_entry *p = (const struct range_entry *) a;
    2707      6173417 :   const struct range_entry *q = (const struct range_entry *) b;
    2708              : 
    2709      6173417 :   if (p->exp != NULL_TREE && TREE_CODE (p->exp) == SSA_NAME)
    2710              :     {
    2711      2492466 :       if (q->exp != NULL_TREE && TREE_CODE (q->exp) == SSA_NAME)
    2712              :         {
    2713              :           /* Group range_entries for the same SSA_NAME together.  */
    2714      2440440 :           if (SSA_NAME_VERSION (p->exp) < SSA_NAME_VERSION (q->exp))
    2715              :             return -1;
    2716      1055092 :           else if (SSA_NAME_VERSION (p->exp) > SSA_NAME_VERSION (q->exp))
    2717              :             return 1;
    2718              :           /* If ->low is different, NULL low goes first, then by
    2719              :              ascending low.  */
    2720       168317 :           if (p->low != NULL_TREE)
    2721              :             {
    2722       151871 :               if (q->low != NULL_TREE)
    2723              :                 {
    2724       144664 :                   tree tem = fold_binary (LT_EXPR, boolean_type_node,
    2725              :                                           p->low, q->low);
    2726       144664 :                   if (tem && integer_onep (tem))
    2727              :                     return -1;
    2728        70227 :                   tem = fold_binary (GT_EXPR, boolean_type_node,
    2729              :                                      p->low, q->low);
    2730        70227 :                   if (tem && integer_onep (tem))
    2731              :                     return 1;
    2732              :                 }
    2733              :               else
    2734              :                 return 1;
    2735              :             }
    2736        16446 :           else if (q->low != NULL_TREE)
    2737              :             return -1;
    2738              :           /* If ->high is different, NULL high goes last, before that by
    2739              :              ascending high.  */
    2740        30552 :           if (p->high != NULL_TREE)
    2741              :             {
    2742        30472 :               if (q->high != NULL_TREE)
    2743              :                 {
    2744        30307 :                   tree tem = fold_binary (LT_EXPR, boolean_type_node,
    2745              :                                           p->high, q->high);
    2746        30307 :                   if (tem && integer_onep (tem))
    2747              :                     return -1;
    2748         8976 :                   tem = fold_binary (GT_EXPR, boolean_type_node,
    2749              :                                      p->high, q->high);
    2750         8976 :                   if (tem && integer_onep (tem))
    2751              :                     return 1;
    2752              :                 }
    2753              :               else
    2754              :                 return -1;
    2755              :             }
    2756           80 :           else if (q->high != NULL_TREE)
    2757              :             return 1;
    2758              :           /* If both ranges are the same, sort below by ascending idx.  */
    2759              :         }
    2760              :       else
    2761              :         return 1;
    2762              :     }
    2763      3680951 :   else if (q->exp != NULL_TREE && TREE_CODE (q->exp) == SSA_NAME)
    2764              :     return -1;
    2765              : 
    2766      3609132 :   if (p->idx < q->idx)
    2767              :     return -1;
    2768              :   else
    2769              :     {
    2770      1821253 :       gcc_checking_assert (p->idx > q->idx);
    2771              :       return 1;
    2772              :     }
    2773              : }
    2774              : 
    2775              : /* Helper function for update_range_test.  Force EXPR into an SSA_NAME,
    2776              :    insert needed statements BEFORE or after GSI.  */
    2777              : 
    2778              : static tree
    2779        25319 : force_into_ssa_name (gimple_stmt_iterator *gsi, tree expr, bool before)
    2780              : {
    2781        25319 :   enum gsi_iterator_update m = before ? GSI_SAME_STMT : GSI_CONTINUE_LINKING;
    2782        25319 :   tree ret = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE, before, m);
    2783        25319 :   if (TREE_CODE (ret) != SSA_NAME)
    2784              :     {
    2785           34 :       gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)), ret);
    2786           34 :       if (before)
    2787           34 :         gsi_insert_before (gsi, g, GSI_SAME_STMT);
    2788              :       else
    2789            0 :         gsi_insert_after (gsi, g, GSI_CONTINUE_LINKING);
    2790           34 :       ret = gimple_assign_lhs (g);
    2791              :     }
    2792        25319 :   return ret;
    2793              : }
    2794              : 
    2795              : /* Helper routine of optimize_range_test.
    2796              :    [EXP, IN_P, LOW, HIGH] is a merged range for
    2797              :    RANGE and OTHERRANGE through OTHERRANGE + COUNT - 1 ranges,
    2798              :    OPCODE and OPS are arguments of optimize_range_tests.  If OTHERRANGE
    2799              :    is NULL, OTHERRANGEP should not be and then OTHERRANGEP points to
    2800              :    an array of COUNT pointers to other ranges.  Return
    2801              :    true if the range merge has been successful.
    2802              :    If OPCODE is ERROR_MARK, this is called from within
    2803              :    maybe_optimize_range_tests and is performing inter-bb range optimization.
    2804              :    In that case, whether an op is BIT_AND_EXPR or BIT_IOR_EXPR is found in
    2805              :    oe->rank.  */
    2806              : 
    2807              : static bool
    2808        25319 : update_range_test (struct range_entry *range, struct range_entry *otherrange,
    2809              :                    struct range_entry **otherrangep,
    2810              :                    unsigned int count, enum tree_code opcode,
    2811              :                    vec<operand_entry *> *ops, tree exp, gimple_seq seq,
    2812              :                    bool in_p, tree low, tree high)
    2813              : {
    2814        25319 :   unsigned int idx = range->idx;
    2815        25319 :   struct range_entry *swap_with = NULL;
    2816        25319 :   basic_block rewrite_bb_first = NULL, rewrite_bb_last = NULL;
    2817        25319 :   if (opcode == ERROR_MARK)
    2818              :     {
    2819              :       /* For inter-bb range test optimization, pick from the range tests
    2820              :          the one which is tested in the earliest condition (one dominating
    2821              :          the others), because otherwise there could be some UB (e.g. signed
    2822              :          overflow) in following bbs that we'd expose which wasn't there in
    2823              :          the original program.  See PR104196.  */
    2824        14515 :       basic_block orig_range_bb = BASIC_BLOCK_FOR_FN (cfun, (*ops)[idx]->id);
    2825        14515 :       basic_block range_bb = orig_range_bb;
    2826        32159 :       for (unsigned int i = 0; i < count; i++)
    2827              :         {
    2828        17644 :           struct range_entry *this_range;
    2829        17644 :           if (otherrange)
    2830        10485 :             this_range = otherrange + i;
    2831              :           else
    2832         7159 :             this_range = otherrangep[i];
    2833        17644 :           operand_entry *oe = (*ops)[this_range->idx];
    2834        17644 :           basic_block this_bb = BASIC_BLOCK_FOR_FN (cfun, oe->id);
    2835        17644 :           if (range_bb != this_bb
    2836        17644 :               && dominated_by_p (CDI_DOMINATORS, range_bb, this_bb))
    2837              :             {
    2838         6931 :               swap_with = this_range;
    2839         6931 :               range_bb = this_bb;
    2840         6931 :               idx = this_range->idx;
    2841              :             }
    2842              :         }
    2843              :       /* If seq is non-NULL, it can contain statements that use SSA_NAMEs
    2844              :          only defined in later blocks.  In this case we can't move the
    2845              :          merged comparison earlier, so instead check if there are any stmts
    2846              :          that might trigger signed integer overflow in between and rewrite
    2847              :          them.  But only after we check if the optimization is possible.  */
    2848        14515 :       if (seq && swap_with)
    2849              :         {
    2850         3730 :           rewrite_bb_first = range_bb;
    2851         3730 :           rewrite_bb_last = orig_range_bb;
    2852         3730 :           idx = range->idx;
    2853         3730 :           swap_with = NULL;
    2854              :         }
    2855              :     }
    2856        25319 :   operand_entry *oe = (*ops)[idx];
    2857        25319 :   tree op = oe->op;
    2858        25319 :   gimple *stmt = op ? SSA_NAME_DEF_STMT (op)
    2859        12362 :                     : last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id));
    2860        25319 :   location_t loc = gimple_location (stmt);
    2861        25319 :   tree optype = op ? TREE_TYPE (op) : boolean_type_node;
    2862        25319 :   tree tem = build_range_check (loc, optype, unshare_expr (exp),
    2863              :                                 in_p, low, high);
    2864        25319 :   gimple_stmt_iterator gsi;
    2865        25319 :   unsigned int i, uid;
    2866              : 
    2867        25319 :   if (tem == NULL_TREE)
    2868              :     return false;
    2869              : 
    2870              :   /* If op is default def SSA_NAME, there is no place to insert the
    2871              :      new comparison.  Give up, unless we can use OP itself as the
    2872              :      range test.  */
    2873        38276 :   if (op && SSA_NAME_IS_DEFAULT_DEF (op))
    2874              :     {
    2875            0 :       if (op == range->exp
    2876            0 :           && ((TYPE_PRECISION (optype) == 1 && TYPE_UNSIGNED (optype))
    2877            0 :               || TREE_CODE (optype) == BOOLEAN_TYPE)
    2878            0 :           && (op == tem
    2879            0 :               || (TREE_CODE (tem) == EQ_EXPR
    2880            0 :                   && TREE_OPERAND (tem, 0) == op
    2881            0 :                   && integer_onep (TREE_OPERAND (tem, 1))))
    2882            0 :           && opcode != BIT_IOR_EXPR
    2883            0 :           && (opcode != ERROR_MARK || oe->rank != BIT_IOR_EXPR))
    2884              :         {
    2885              :           stmt = NULL;
    2886              :           tem = op;
    2887              :         }
    2888              :       else
    2889              :         return false;
    2890              :     }
    2891              : 
    2892        25319 :   if (swap_with)
    2893         1584 :     std::swap (range->idx, swap_with->idx);
    2894              : 
    2895        25319 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2896              :     {
    2897           39 :       struct range_entry *r;
    2898           39 :       fprintf (dump_file, "Optimizing range tests ");
    2899           39 :       dump_range_entry (dump_file, range, false);
    2900          180 :       for (i = 0; i < count; i++)
    2901              :         {
    2902          102 :           if (otherrange)
    2903           82 :             r = otherrange + i;
    2904              :           else
    2905           20 :             r = otherrangep[i];
    2906          102 :           if (r->exp
    2907          102 :               && r->exp != range->exp
    2908           20 :               && TREE_CODE (r->exp) == SSA_NAME)
    2909              :             {
    2910           20 :               fprintf (dump_file, " and ");
    2911           20 :               dump_range_entry (dump_file, r, false);
    2912              :             }
    2913              :           else
    2914              :             {
    2915           82 :               fprintf (dump_file, " and");
    2916           82 :               dump_range_entry (dump_file, r, true);
    2917              :             }
    2918              :         }
    2919           39 :       fprintf (dump_file, "\n into ");
    2920           39 :       print_generic_expr (dump_file, tem);
    2921           39 :       fprintf (dump_file, "\n");
    2922              :     }
    2923              : 
    2924              :   /* In inter-bb range optimization mode, if we have a seq, we can't
    2925              :      move the merged comparison to the earliest bb from the comparisons
    2926              :      being replaced, so instead rewrite stmts that could trigger signed
    2927              :      integer overflow.  */
    2928        32348 :   for (basic_block bb = rewrite_bb_last;
    2929        32348 :        bb != rewrite_bb_first; bb = single_pred (bb))
    2930        14058 :     for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    2931        27742 :          !gsi_end_p (gsi); gsi_next (&gsi))
    2932              :       {
    2933        20713 :         gimple *stmt = gsi_stmt (gsi);
    2934        20713 :         if (gimple_needing_rewrite_undefined (stmt))
    2935              :           {
    2936           63 :             gimple_stmt_iterator gsip = gsi;
    2937           63 :             gimple_stmt_iterator gsin = gsi;
    2938           63 :             gsi_prev (&gsip);
    2939           63 :             gsi_next (&gsin);
    2940           63 :             rewrite_to_defined_unconditional (&gsi);
    2941           63 :             unsigned uid = gimple_uid (stmt);
    2942           63 :             if (gsi_end_p (gsip))
    2943           27 :               gsip = gsi_after_labels (bb);
    2944              :             else
    2945           36 :               gsi_next (&gsip);
    2946          253 :             for (; gsi_stmt (gsip) != gsi_stmt (gsin);
    2947          190 :                  gsi_next (&gsip))
    2948          190 :               gimple_set_uid (gsi_stmt (gsip), uid);
    2949              :           }
    2950              :       }
    2951              : 
    2952        25319 :   if (opcode == BIT_IOR_EXPR
    2953        19055 :       || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
    2954        17896 :     tem = invert_truthvalue_loc (loc, tem);
    2955              : 
    2956        25319 :   tem = fold_convert_loc (loc, optype, tem);
    2957        25319 :   if (stmt)
    2958              :     {
    2959        25319 :       gsi = gsi_for_stmt (stmt);
    2960        25319 :       uid = gimple_uid (stmt);
    2961              :     }
    2962              :   else
    2963              :     {
    2964            0 :       gsi = gsi_none ();
    2965            0 :       uid = 0;
    2966              :     }
    2967        25319 :   if (stmt == NULL)
    2968            0 :     gcc_checking_assert (tem == op);
    2969              :   /* In rare cases range->exp can be equal to lhs of stmt.
    2970              :      In that case we have to insert after the stmt rather then before
    2971              :      it.  If stmt is a PHI, insert it at the start of the basic block.  */
    2972        25319 :   else if (op != range->exp)
    2973              :     {
    2974        25319 :       gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
    2975        25319 :       tem = force_into_ssa_name (&gsi, tem, true);
    2976        25319 :       gsi_prev (&gsi);
    2977              :     }
    2978            0 :   else if (gimple_code (stmt) != GIMPLE_PHI)
    2979              :     {
    2980            0 :       gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
    2981            0 :       tem = force_into_ssa_name (&gsi, tem, false);
    2982              :     }
    2983              :   else
    2984              :     {
    2985            0 :       gsi = gsi_after_labels (gimple_bb (stmt));
    2986            0 :       if (!gsi_end_p (gsi))
    2987            0 :         uid = gimple_uid (gsi_stmt (gsi));
    2988              :       else
    2989              :         {
    2990            0 :           gsi = gsi_start_bb (gimple_bb (stmt));
    2991            0 :           uid = 1;
    2992            0 :           while (!gsi_end_p (gsi))
    2993              :             {
    2994            0 :               uid = gimple_uid (gsi_stmt (gsi));
    2995            0 :               gsi_next (&gsi);
    2996              :             }
    2997              :         }
    2998            0 :       gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
    2999            0 :       tem = force_into_ssa_name (&gsi, tem, true);
    3000            0 :       if (gsi_end_p (gsi))
    3001            0 :         gsi = gsi_last_bb (gimple_bb (stmt));
    3002              :       else
    3003            0 :         gsi_prev (&gsi);
    3004              :     }
    3005        95324 :   for (; !gsi_end_p (gsi); gsi_prev (&gsi))
    3006        88020 :     if (gimple_uid (gsi_stmt (gsi)))
    3007              :       break;
    3008              :     else
    3009        66353 :       gimple_set_uid (gsi_stmt (gsi), uid);
    3010              : 
    3011        25319 :   oe->op = tem;
    3012        25319 :   range->exp = exp;
    3013        25319 :   range->low = low;
    3014        25319 :   range->high = high;
    3015        25319 :   range->in_p = in_p;
    3016              : 
    3017        54939 :   for (i = 0; i < count; i++)
    3018              :     {
    3019        29620 :       if (otherrange)
    3020        18446 :         range = otherrange + i;
    3021              :       else
    3022        11174 :         range = otherrangep[i];
    3023        29620 :       oe = (*ops)[range->idx];
    3024              :       /* Now change all the other range test immediate uses, so that
    3025              :          those tests will be optimized away.  */
    3026        29620 :       if (opcode == ERROR_MARK)
    3027              :         {
    3028        17644 :           if (oe->op)
    3029         2341 :             oe->op = build_int_cst (TREE_TYPE (oe->op),
    3030         2341 :                                     oe->rank == BIT_IOR_EXPR ? 0 : 1);
    3031              :           else
    3032        15303 :             oe->op = (oe->rank == BIT_IOR_EXPR
    3033        15303 :                       ? boolean_false_node : boolean_true_node);
    3034              :         }
    3035              :       else
    3036        11976 :         oe->op = error_mark_node;
    3037        29620 :       range->exp = NULL_TREE;
    3038        29620 :       range->low = NULL_TREE;
    3039        29620 :       range->high = NULL_TREE;
    3040              :     }
    3041              :   return true;
    3042              : }
    3043              : 
    3044              : /* Optimize X == CST1 || X == CST2
    3045              :    if popcount (CST1 ^ CST2) == 1 into
    3046              :    (X & ~(CST1 ^ CST2)) == (CST1 & ~(CST1 ^ CST2)).
    3047              :    Similarly for ranges.  E.g.
    3048              :    X != 2 && X != 3 && X != 10 && X != 11
    3049              :    will be transformed by the previous optimization into
    3050              :    !((X - 2U) <= 1U || (X - 10U) <= 1U)
    3051              :    and this loop can transform that into
    3052              :    !(((X & ~8) - 2U) <= 1U).  */
    3053              : 
    3054              : static bool
    3055        23275 : optimize_range_tests_xor (enum tree_code opcode, tree type,
    3056              :                           tree lowi, tree lowj, tree highi, tree highj,
    3057              :                           vec<operand_entry *> *ops,
    3058              :                           struct range_entry *rangei,
    3059              :                           struct range_entry *rangej)
    3060              : {
    3061        23275 :   tree lowxor, highxor, tem, exp;
    3062              :   /* Check lowi ^ lowj == highi ^ highj and
    3063              :      popcount (lowi ^ lowj) == 1.  */
    3064        23275 :   lowxor = fold_binary (BIT_XOR_EXPR, type, lowi, lowj);
    3065        23275 :   if (lowxor == NULL_TREE || TREE_CODE (lowxor) != INTEGER_CST)
    3066              :     return false;
    3067        23275 :   if (!integer_pow2p (lowxor))
    3068              :     return false;
    3069         3335 :   highxor = fold_binary (BIT_XOR_EXPR, type, highi, highj);
    3070         3335 :   if (!tree_int_cst_equal (lowxor, highxor))
    3071              :     return false;
    3072              : 
    3073         2924 :   exp = rangei->exp;
    3074         2924 :   scalar_int_mode mode = as_a <scalar_int_mode> (TYPE_MODE (type));
    3075         2924 :   int prec = GET_MODE_PRECISION (mode);
    3076         2924 :   if (TYPE_PRECISION (type) < prec
    3077         2923 :       || (wi::to_wide (TYPE_MIN_VALUE (type))
    3078         8770 :           != wi::min_value (prec, TYPE_SIGN (type)))
    3079         5847 :       || (wi::to_wide (TYPE_MAX_VALUE (type))
    3080         8770 :           != wi::max_value (prec, TYPE_SIGN (type))))
    3081              :     {
    3082            1 :       type = build_nonstandard_integer_type (prec, TYPE_UNSIGNED (type));
    3083            1 :       exp = fold_convert (type, exp);
    3084            1 :       lowxor = fold_convert (type, lowxor);
    3085            1 :       lowi = fold_convert (type, lowi);
    3086            1 :       highi = fold_convert (type, highi);
    3087              :     }
    3088         2924 :   tem = fold_build1 (BIT_NOT_EXPR, type, lowxor);
    3089         2924 :   exp = fold_build2 (BIT_AND_EXPR, type, exp, tem);
    3090         2924 :   lowj = fold_build2 (BIT_AND_EXPR, type, lowi, tem);
    3091         2924 :   highj = fold_build2 (BIT_AND_EXPR, type, highi, tem);
    3092         2924 :   if (update_range_test (rangei, rangej, NULL, 1, opcode, ops, exp,
    3093              :                          NULL, rangei->in_p, lowj, highj))
    3094              :     return true;
    3095              :   return false;
    3096              : }
    3097              : 
    3098              : /* Optimize X == CST1 || X == CST2
    3099              :    if popcount (CST2 - CST1) == 1 into
    3100              :    ((X - CST1) & ~(CST2 - CST1)) == 0.
    3101              :    Similarly for ranges.  E.g.
    3102              :    X == 43 || X == 76 || X == 44 || X == 78 || X == 77 || X == 46
    3103              :    || X == 75 || X == 45
    3104              :    will be transformed by the previous optimization into
    3105              :    (X - 43U) <= 3U || (X - 75U) <= 3U
    3106              :    and this loop can transform that into
    3107              :    ((X - 43U) & ~(75U - 43U)) <= 3U.  */
    3108              : static bool
    3109        18004 : optimize_range_tests_diff (enum tree_code opcode, tree type,
    3110              :                            tree lowi, tree lowj, tree highi, tree highj,
    3111              :                            vec<operand_entry *> *ops,
    3112              :                            struct range_entry *rangei,
    3113              :                            struct range_entry *rangej)
    3114              : {
    3115        18004 :   tree tem1, tem2, mask;
    3116              :   /* Check highi - lowi == highj - lowj.  */
    3117        18004 :   tem1 = fold_binary (MINUS_EXPR, type, highi, lowi);
    3118        18004 :   if (tem1 == NULL_TREE || TREE_CODE (tem1) != INTEGER_CST)
    3119              :     return false;
    3120        18004 :   tem2 = fold_binary (MINUS_EXPR, type, highj, lowj);
    3121        18004 :   if (!tree_int_cst_equal (tem1, tem2))
    3122              :     return false;
    3123              :   /* Check popcount (lowj - lowi) == 1.  */
    3124        11927 :   tem1 = fold_binary (MINUS_EXPR, type, lowj, lowi);
    3125        11927 :   if (tem1 == NULL_TREE || TREE_CODE (tem1) != INTEGER_CST)
    3126              :     return false;
    3127        11927 :   if (!integer_pow2p (tem1))
    3128              :     return false;
    3129              : 
    3130         1982 :   scalar_int_mode mode = as_a <scalar_int_mode> (TYPE_MODE (type));
    3131         1982 :   int prec = GET_MODE_PRECISION (mode);
    3132         1982 :   if (TYPE_PRECISION (type) < prec
    3133         1978 :       || (wi::to_wide (TYPE_MIN_VALUE (type))
    3134         5938 :           != wi::min_value (prec, TYPE_SIGN (type)))
    3135         3960 :       || (wi::to_wide (TYPE_MAX_VALUE (type))
    3136         5938 :           != wi::max_value (prec, TYPE_SIGN (type))))
    3137            4 :     type = build_nonstandard_integer_type (prec, 1);
    3138              :   else
    3139         1978 :     type = unsigned_type_for (type);
    3140         1982 :   tem1 = fold_convert (type, tem1);
    3141         1982 :   tem2 = fold_convert (type, tem2);
    3142         1982 :   lowi = fold_convert (type, lowi);
    3143         1982 :   mask = fold_build1 (BIT_NOT_EXPR, type, tem1);
    3144         1982 :   tem1 = fold_build2 (MINUS_EXPR, type,
    3145              :                       fold_convert (type, rangei->exp), lowi);
    3146         1982 :   tem1 = fold_build2 (BIT_AND_EXPR, type, tem1, mask);
    3147         1982 :   lowj = build_int_cst (type, 0);
    3148         1982 :   if (update_range_test (rangei, rangej, NULL, 1, opcode, ops, tem1,
    3149              :                          NULL, rangei->in_p, lowj, tem2))
    3150              :     return true;
    3151              :   return false;
    3152              : }
    3153              : 
    3154              : /* It does some common checks for function optimize_range_tests_xor and
    3155              :    optimize_range_tests_diff.
    3156              :    If OPTIMIZE_XOR is TRUE, it calls optimize_range_tests_xor.
    3157              :    Else it calls optimize_range_tests_diff.  */
    3158              : 
    3159              : static bool
    3160      2184512 : optimize_range_tests_1 (enum tree_code opcode, int first, int length,
    3161              :                         bool optimize_xor, vec<operand_entry *> *ops,
    3162              :                         struct range_entry *ranges)
    3163              : {
    3164      2184512 :   int i, j;
    3165      2184512 :   bool any_changes = false;
    3166      3815938 :   for (i = first; i < length; i++)
    3167              :     {
    3168      1631426 :       tree lowi, highi, lowj, highj, type, tem;
    3169              : 
    3170      1631426 :       if (ranges[i].exp == NULL_TREE || ranges[i].in_p)
    3171      1016184 :         continue;
    3172       615242 :       type = TREE_TYPE (ranges[i].exp);
    3173       615242 :       if (!INTEGRAL_TYPE_P (type))
    3174        49742 :         continue;
    3175       565500 :       lowi = ranges[i].low;
    3176       565500 :       if (lowi == NULL_TREE)
    3177        35768 :         lowi = TYPE_MIN_VALUE (type);
    3178       565500 :       highi = ranges[i].high;
    3179       565500 :       if (highi == NULL_TREE)
    3180         6800 :         continue;
    3181       947213 :       for (j = i + 1; j < length && j < i + 64; j++)
    3182              :         {
    3183       393419 :           bool changes;
    3184       393419 :           if (ranges[i].exp != ranges[j].exp || ranges[j].in_p)
    3185       352140 :             continue;
    3186        41279 :           lowj = ranges[j].low;
    3187        41279 :           if (lowj == NULL_TREE)
    3188            0 :             continue;
    3189        41279 :           highj = ranges[j].high;
    3190        41279 :           if (highj == NULL_TREE)
    3191          154 :             highj = TYPE_MAX_VALUE (type);
    3192              :           /* Check lowj > highi.  */
    3193        41279 :           tem = fold_binary (GT_EXPR, boolean_type_node,
    3194              :                              lowj, highi);
    3195        41279 :           if (tem == NULL_TREE || !integer_onep (tem))
    3196            0 :             continue;
    3197        41279 :           if (optimize_xor)
    3198        23275 :             changes = optimize_range_tests_xor (opcode, type, lowi, lowj,
    3199              :                                                 highi, highj, ops,
    3200              :                                                 ranges + i, ranges + j);
    3201              :           else
    3202        18004 :             changes = optimize_range_tests_diff (opcode, type, lowi, lowj,
    3203              :                                                  highi, highj, ops,
    3204              :                                                  ranges + i, ranges + j);
    3205        41279 :           if (changes)
    3206              :             {
    3207              :               any_changes = true;
    3208              :               break;
    3209              :             }
    3210              :         }
    3211              :     }
    3212      2184512 :   return any_changes;
    3213              : }
    3214              : 
    3215              : /* Helper function of optimize_range_tests_to_bit_test.  Handle a single
    3216              :    range, EXP, LOW, HIGH, compute bit mask of bits to test and return
    3217              :    EXP on success, NULL otherwise.  */
    3218              : 
    3219              : static tree
    3220       170248 : extract_bit_test_mask (tree exp, int prec, tree totallow, tree low, tree high,
    3221              :                        wide_int *mask, tree *totallowp)
    3222              : {
    3223       170248 :   tree tem = int_const_binop (MINUS_EXPR, high, low);
    3224       170248 :   if (tem == NULL_TREE
    3225       170248 :       || TREE_CODE (tem) != INTEGER_CST
    3226       170248 :       || TREE_OVERFLOW (tem)
    3227       159170 :       || tree_int_cst_sgn (tem) == -1
    3228       329418 :       || compare_tree_int (tem, prec) != -1)
    3229              :     return NULL_TREE;
    3230              : 
    3231       155193 :   unsigned HOST_WIDE_INT max = tree_to_uhwi (tem) + 1;
    3232       155193 :   *mask = wi::shifted_mask (0, max, false, prec);
    3233       155193 :   if (TREE_CODE (exp) == BIT_AND_EXPR
    3234       155193 :       && TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
    3235              :     {
    3236         5202 :       widest_int msk = wi::to_widest (TREE_OPERAND (exp, 1));
    3237         5202 :       msk = wi::zext (~msk, TYPE_PRECISION (TREE_TYPE (exp)));
    3238         5202 :       if (wi::popcount (msk) == 1
    3239         5202 :           && wi::ltu_p (msk, prec - max))
    3240              :         {
    3241         4435 :           *mask |= wi::shifted_mask (msk.to_uhwi (), max, false, prec);
    3242         4435 :           max += msk.to_uhwi ();
    3243         4435 :           exp = TREE_OPERAND (exp, 0);
    3244         4435 :           if (integer_zerop (low)
    3245         2223 :               && TREE_CODE (exp) == PLUS_EXPR
    3246         6268 :               && TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
    3247              :             {
    3248         1833 :               tree ret = TREE_OPERAND (exp, 0);
    3249         1833 :               STRIP_NOPS (ret);
    3250         1833 :               widest_int bias
    3251         1833 :                 = wi::neg (wi::sext (wi::to_widest (TREE_OPERAND (exp, 1)),
    3252         3666 :                                      TYPE_PRECISION (TREE_TYPE (low))));
    3253         1833 :               tree tbias = wide_int_to_tree (TREE_TYPE (ret), bias);
    3254         1833 :               if (totallowp)
    3255              :                 {
    3256         1797 :                   *totallowp = tbias;
    3257         1797 :                   return ret;
    3258              :                 }
    3259           36 :               else if (!tree_int_cst_lt (totallow, tbias))
    3260              :                 return NULL_TREE;
    3261           36 :               bias = wi::to_widest (tbias);
    3262           36 :               bias -= wi::to_widest (totallow);
    3263           36 :               if (bias >= 0 && bias < prec - max)
    3264              :                 {
    3265           26 :                   *mask = wi::lshift (*mask, bias);
    3266           26 :                   return ret;
    3267              :                 }
    3268         1833 :             }
    3269              :         }
    3270         5202 :     }
    3271       153370 :   if (totallowp)
    3272              :     return exp;
    3273        13838 :   if (!tree_int_cst_lt (totallow, low))
    3274              :     return exp;
    3275        13816 :   tem = int_const_binop (MINUS_EXPR, low, totallow);
    3276        13816 :   if (tem == NULL_TREE
    3277        13816 :       || TREE_CODE (tem) != INTEGER_CST
    3278        13816 :       || TREE_OVERFLOW (tem)
    3279        27475 :       || compare_tree_int (tem, prec - max) == 1)
    3280              :     return NULL_TREE;
    3281              : 
    3282         9800 :   *mask = wi::lshift (*mask, wi::to_widest (tem));
    3283         9800 :   return exp;
    3284              : }
    3285              : 
    3286              : /* Attempt to optimize small range tests using bit test.
    3287              :    E.g.
    3288              :    X != 43 && X != 76 && X != 44 && X != 78 && X != 49
    3289              :    && X != 77 && X != 46 && X != 75 && X != 45 && X != 82
    3290              :    has been by earlier optimizations optimized into:
    3291              :    ((X - 43U) & ~32U) > 3U && X != 49 && X != 82
    3292              :    As all the 43 through 82 range is less than 64 numbers,
    3293              :    for 64-bit word targets optimize that into:
    3294              :    (X - 43U) > 40U && ((1 << (X - 43U)) & 0x8F0000004FULL) == 0  */
    3295              : 
    3296              : static bool
    3297      1092264 : optimize_range_tests_to_bit_test (enum tree_code opcode, int first, int length,
    3298              :                                   vec<operand_entry *> *ops,
    3299              :                                   struct range_entry *ranges)
    3300              : {
    3301      1092264 :   int i, j;
    3302      1092264 :   bool any_changes = false;
    3303      1092264 :   int prec = GET_MODE_BITSIZE (word_mode);
    3304      1092264 :   auto_vec<struct range_entry *, 64> candidates;
    3305              : 
    3306      1526298 :   for (i = first; i < length - 1; i++)
    3307              :     {
    3308       434034 :       tree lowi, highi, lowj, highj, type;
    3309              : 
    3310       434034 :       if (ranges[i].exp == NULL_TREE || ranges[i].in_p)
    3311       292705 :         continue;
    3312       175781 :       type = TREE_TYPE (ranges[i].exp);
    3313       175781 :       if (!INTEGRAL_TYPE_P (type))
    3314        17101 :         continue;
    3315       158680 :       lowi = ranges[i].low;
    3316       158680 :       if (lowi == NULL_TREE)
    3317        11177 :         lowi = TYPE_MIN_VALUE (type);
    3318       158680 :       highi = ranges[i].high;
    3319       158680 :       if (highi == NULL_TREE)
    3320         2413 :         continue;
    3321       156267 :       wide_int mask;
    3322       156267 :       tree exp = extract_bit_test_mask (ranges[i].exp, prec, lowi, lowi,
    3323              :                                         highi, &mask, &lowi);
    3324       156267 :       if (exp == NULL_TREE)
    3325        14938 :         continue;
    3326       141329 :       candidates.truncate (0);
    3327       141329 :       int end = MIN (i + 64, length);
    3328       309885 :       for (j = i + 1; j < end; j++)
    3329              :         {
    3330       168556 :           tree exp2;
    3331       168556 :           if (ranges[j].exp == NULL_TREE || ranges[j].in_p)
    3332       158738 :             continue;
    3333       100073 :           if (ranges[j].exp == exp)
    3334              :             ;
    3335        86580 :           else if (TREE_CODE (ranges[j].exp) == BIT_AND_EXPR)
    3336              :             {
    3337         1455 :               exp2 = TREE_OPERAND (ranges[j].exp, 0);
    3338         1455 :               if (exp2 == exp)
    3339              :                 ;
    3340         1015 :               else if (TREE_CODE (exp2) == PLUS_EXPR)
    3341              :                 {
    3342          814 :                   exp2 = TREE_OPERAND (exp2, 0);
    3343          814 :                   STRIP_NOPS (exp2);
    3344          814 :                   if (exp2 != exp)
    3345          766 :                     continue;
    3346              :                 }
    3347              :               else
    3348          201 :                 continue;
    3349              :             }
    3350              :           else
    3351        85125 :             continue;
    3352        13981 :           lowj = ranges[j].low;
    3353        13981 :           if (lowj == NULL_TREE)
    3354            0 :             continue;
    3355        13981 :           highj = ranges[j].high;
    3356        13981 :           if (highj == NULL_TREE)
    3357           77 :             highj = TYPE_MAX_VALUE (TREE_TYPE (lowj));
    3358        13981 :           wide_int mask2;
    3359        13981 :           exp2 = extract_bit_test_mask (ranges[j].exp, prec, lowi, lowj,
    3360              :                                         highj, &mask2, NULL);
    3361        13981 :           if (exp2 != exp)
    3362         4163 :             continue;
    3363         9818 :           mask |= mask2;
    3364         9818 :           candidates.safe_push (&ranges[j]);
    3365        13981 :         }
    3366              : 
    3367              :       /* If every possible relative value of the expression is a valid shift
    3368              :          amount, then we can merge the entry test in the bit test.  In this
    3369              :          case, if we would need otherwise 2 or more comparisons, then use
    3370              :          the bit test; in the other cases, the threshold is 3 comparisons.  */
    3371       141329 :       bool entry_test_needed;
    3372       141329 :       int_range_max r;
    3373       282658 :       if (TREE_CODE (exp) == SSA_NAME
    3374       281224 :           && get_range_query (cfun)->range_of_expr (r, exp)
    3375       140612 :           && !r.undefined_p ()
    3376       140612 :           && !r.varying_p ()
    3377       322510 :           && wi::leu_p (r.upper_bound () - r.lower_bound (), prec - 1))
    3378              :         {
    3379         4906 :           wide_int min = r.lower_bound ();
    3380         4906 :           wide_int ilowi = wi::to_wide (lowi);
    3381         4906 :           if (wi::lt_p (min, ilowi, TYPE_SIGN (TREE_TYPE (lowi))))
    3382              :             {
    3383          562 :               lowi = wide_int_to_tree (TREE_TYPE (lowi), min);
    3384          562 :               mask = wi::lshift (mask, ilowi - min);
    3385              :             }
    3386         4344 :           else if (wi::gt_p (min, ilowi, TYPE_SIGN (TREE_TYPE (lowi))))
    3387              :             {
    3388            1 :               lowi = wide_int_to_tree (TREE_TYPE (lowi), min);
    3389            1 :               mask = wi::lrshift (mask, min - ilowi);
    3390              :             }
    3391         4906 :           entry_test_needed = false;
    3392         4906 :         }
    3393              :       else
    3394              :         entry_test_needed = true;
    3395       287564 :       if (candidates.length () >= (entry_test_needed ? 2 : 1))
    3396              :         {
    3397          896 :           tree high = wide_int_to_tree (TREE_TYPE (lowi),
    3398          448 :                                         wi::to_widest (lowi)
    3399         1344 :                                         + prec - 1 - wi::clz (mask));
    3400          448 :           operand_entry *oe = (*ops)[ranges[i].idx];
    3401          448 :           tree op = oe->op;
    3402          448 :           gimple *stmt = op ? SSA_NAME_DEF_STMT (op)
    3403           50 :                             : last_nondebug_stmt (BASIC_BLOCK_FOR_FN
    3404          398 :                                                           (cfun, oe->id));
    3405          448 :           location_t loc = gimple_location (stmt);
    3406          448 :           tree optype = op ? TREE_TYPE (op) : boolean_type_node;
    3407              : 
    3408              :           /* See if it isn't cheaper to pretend the minimum value of the
    3409              :              range is 0, if maximum value is small enough.
    3410              :              We can avoid then subtraction of the minimum value, but the
    3411              :              mask constant could be perhaps more expensive.  */
    3412          448 :           if (compare_tree_int (lowi, 0) > 0
    3413          371 :               && compare_tree_int (high, prec) < 0
    3414          833 :               && (entry_test_needed || wi::ltu_p (r.upper_bound (), prec)))
    3415              :             {
    3416          153 :               int cost_diff;
    3417          153 :               HOST_WIDE_INT m = tree_to_uhwi (lowi);
    3418          153 :               rtx reg = gen_raw_REG (word_mode, 10000);
    3419          153 :               bool speed_p = optimize_bb_for_speed_p (gimple_bb (stmt));
    3420          153 :               cost_diff = set_src_cost (gen_rtx_PLUS (word_mode, reg,
    3421              :                                                       GEN_INT (-m)),
    3422              :                                         word_mode, speed_p);
    3423          153 :               rtx r = immed_wide_int_const (mask, word_mode);
    3424          153 :               cost_diff += set_src_cost (gen_rtx_AND (word_mode, reg, r),
    3425              :                                          word_mode, speed_p);
    3426          153 :               r = immed_wide_int_const (wi::lshift (mask, m), word_mode);
    3427          153 :               cost_diff -= set_src_cost (gen_rtx_AND (word_mode, reg, r),
    3428              :                                          word_mode, speed_p);
    3429          153 :               if (cost_diff > 0)
    3430              :                 {
    3431           62 :                   mask = wi::lshift (mask, m);
    3432           62 :                   lowi = build_zero_cst (TREE_TYPE (lowi));
    3433              :                 }
    3434              :             }
    3435              : 
    3436          448 :           tree tem;
    3437          448 :           if (entry_test_needed)
    3438              :             {
    3439          387 :               tem = build_range_check (loc, optype, unshare_expr (exp),
    3440              :                                        false, lowi, high);
    3441          387 :               if (tem == NULL_TREE || is_gimple_val (tem))
    3442            0 :                 continue;
    3443              :             }
    3444              :           else
    3445           61 :             tem = NULL_TREE;
    3446          448 :           tree etype = unsigned_type_for (TREE_TYPE (exp));
    3447          448 :           exp = fold_build2_loc (loc, MINUS_EXPR, etype,
    3448              :                                  fold_convert_loc (loc, etype, exp),
    3449              :                                  fold_convert_loc (loc, etype, lowi));
    3450          448 :           exp = fold_convert_loc (loc, integer_type_node, exp);
    3451          448 :           tree word_type = lang_hooks.types.type_for_mode (word_mode, 1);
    3452          448 :           exp = fold_build2_loc (loc, LSHIFT_EXPR, word_type,
    3453              :                                  build_int_cst (word_type, 1), exp);
    3454          896 :           exp = fold_build2_loc (loc, BIT_AND_EXPR, word_type, exp,
    3455          448 :                                  wide_int_to_tree (word_type, mask));
    3456          448 :           exp = fold_build2_loc (loc, EQ_EXPR, optype, exp,
    3457              :                                  build_zero_cst (word_type));
    3458          448 :           if (is_gimple_val (exp))
    3459            0 :             continue;
    3460              : 
    3461              :           /* The shift might have undefined behavior if TEM is true,
    3462              :              but reassociate_bb isn't prepared to have basic blocks
    3463              :              split when it is running.  So, temporarily emit a code
    3464              :              with BIT_IOR_EXPR instead of &&, and fix it up in
    3465              :              branch_fixup.  */
    3466          448 :           gimple_seq seq = NULL;
    3467          448 :           if (tem)
    3468              :             {
    3469          387 :               tem = force_gimple_operand (tem, &seq, true, NULL_TREE);
    3470          387 :               gcc_assert (TREE_CODE (tem) == SSA_NAME);
    3471          387 :               gimple_set_visited (SSA_NAME_DEF_STMT (tem), true);
    3472              :             }
    3473          448 :           gimple_seq seq2;
    3474          448 :           exp = force_gimple_operand (exp, &seq2, true, NULL_TREE);
    3475          448 :           gimple_seq_add_seq_without_update (&seq, seq2);
    3476          448 :           gcc_assert (TREE_CODE (exp) == SSA_NAME);
    3477          448 :           gimple_set_visited (SSA_NAME_DEF_STMT (exp), true);
    3478          448 :           if (tem)
    3479              :             {
    3480          387 :               gimple *g = gimple_build_assign (make_ssa_name (optype),
    3481              :                                                BIT_IOR_EXPR, tem, exp);
    3482          387 :               gimple_set_location (g, loc);
    3483          387 :               gimple_seq_add_stmt_without_update (&seq, g);
    3484          387 :               exp = gimple_assign_lhs (g);
    3485              :             }
    3486          448 :           tree val = build_zero_cst (optype);
    3487         1344 :           if (update_range_test (&ranges[i], NULL, candidates.address (),
    3488              :                                  candidates.length (), opcode, ops, exp,
    3489              :                                  seq, false, val, val))
    3490              :             {
    3491          448 :               any_changes = true;
    3492          448 :               if (tem)
    3493          387 :                 reassoc_branch_fixups.safe_push (tem);
    3494              :             }
    3495              :           else
    3496            0 :             gimple_seq_discard (seq);
    3497              :         }
    3498       156267 :     }
    3499      1092264 :   return any_changes;
    3500      1092264 : }
    3501              : 
    3502              : /* Optimize x != 0 && y != 0 && z != 0 into (x | y | z) != 0
    3503              :    and similarly x != -1 && y != -1 && y != -1 into (x & y & z) != -1.
    3504              :    Also, handle x < C && y < C && z < C where C is power of two as
    3505              :    (x | y | z) < C.  And also handle signed x < 0 && y < 0 && z < 0
    3506              :    as (x | y | z) < 0.  */
    3507              : 
    3508              : static bool
    3509      1092264 : optimize_range_tests_cmp_bitwise (enum tree_code opcode, int first, int length,
    3510              :                                   vec<operand_entry *> *ops,
    3511              :                                   struct range_entry *ranges)
    3512              : {
    3513      1092264 :   int i;
    3514      1092264 :   unsigned int b;
    3515      1092264 :   bool any_changes = false;
    3516      1092264 :   auto_vec<int, 128> buckets;
    3517      1092264 :   auto_vec<int, 32> chains;
    3518      1092264 :   auto_vec<struct range_entry *, 32> candidates;
    3519              : 
    3520      1907984 :   for (i = first; i < length; i++)
    3521              :     {
    3522       815720 :       int idx;
    3523              : 
    3524      1164979 :       if (ranges[i].exp == NULL_TREE
    3525       795629 :           || TREE_CODE (ranges[i].exp) != SSA_NAME
    3526       790956 :           || TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) <= 1
    3527      1282181 :           || TREE_CODE (TREE_TYPE (ranges[i].exp)) == BOOLEAN_TYPE)
    3528       349259 :         continue;
    3529              : 
    3530       466461 :       if (ranges[i].low != NULL_TREE
    3531       439310 :           && ranges[i].high != NULL_TREE
    3532       384799 :           && ranges[i].in_p
    3533       697384 :           && tree_int_cst_equal (ranges[i].low, ranges[i].high))
    3534              :         {
    3535       201087 :           idx = !integer_zerop (ranges[i].low);
    3536       201087 :           if (idx && !integer_all_onesp (ranges[i].low))
    3537       111943 :             continue;
    3538              :         }
    3539       265374 :       else if (ranges[i].high != NULL_TREE
    3540       210829 :                && TREE_CODE (ranges[i].high) == INTEGER_CST
    3541       210829 :                && ranges[i].in_p)
    3542              :         {
    3543        39103 :           wide_int w = wi::to_wide (ranges[i].high);
    3544        39103 :           int prec = TYPE_PRECISION (TREE_TYPE (ranges[i].exp));
    3545        39103 :           int l = wi::clz (w);
    3546        39103 :           idx = 2;
    3547       107007 :           if (l <= 0
    3548        39103 :               || l >= prec
    3549        72124 :               || w != wi::mask (prec - l, false, prec))
    3550        28801 :             continue;
    3551        10302 :           if (!((TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
    3552         6492 :                  && ranges[i].low == NULL_TREE)
    3553        10302 :                 || (ranges[i].low
    3554         8306 :                     && integer_zerop (ranges[i].low))))
    3555         3385 :             continue;
    3556        39103 :         }
    3557       428703 :       else if (ranges[i].high == NULL_TREE
    3558        54545 :                && ranges[i].low != NULL_TREE
    3559              :                /* Perform this optimization only in the last
    3560              :                   reassoc pass, as it interferes with the reassociation
    3561              :                   itself or could also with VRP etc. which might not
    3562              :                   be able to virtually undo the optimization.  */
    3563        54511 :                && !reassoc_insert_powi_p
    3564        27570 :                && !TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
    3565       253500 :                && integer_zerop (ranges[i].low))
    3566              :         idx = 3;
    3567              :       else
    3568       202432 :         continue;
    3569              : 
    3570       119900 :       b = TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) * 4 + idx;
    3571       119900 :       if (buckets.length () <= b)
    3572       105714 :         buckets.safe_grow_cleared (b + 1, true);
    3573       119900 :       if (chains.length () <= (unsigned) i)
    3574       119900 :         chains.safe_grow (i + 1, true);
    3575       119900 :       chains[i] = buckets[b];
    3576       119900 :       buckets[b] = i + 1;
    3577              :     }
    3578              : 
    3579     20280717 :   FOR_EACH_VEC_ELT (buckets, b, i)
    3580     19188453 :     if (i && chains[i - 1])
    3581              :       {
    3582         7027 :         int j, k = i;
    3583         7027 :         if ((b % 4) == 2)
    3584              :           {
    3585              :             /* When ranges[X - 1].high + 1 is a power of two,
    3586              :                we need to process the same bucket up to
    3587              :                precision - 1 times, each time split the entries
    3588              :                with the same high bound into one chain and the
    3589              :                rest into another one to be processed later.  */
    3590              :             int this_prev = i;
    3591              :             int other_prev = 0;
    3592          146 :             for (j = chains[i - 1]; j; j = chains[j - 1])
    3593              :               {
    3594           79 :                 if (tree_int_cst_equal (ranges[i - 1].high,
    3595           79 :                                         ranges[j - 1].high))
    3596              :                   {
    3597           70 :                     chains[this_prev - 1] = j;
    3598           70 :                     this_prev = j;
    3599              :                   }
    3600            9 :                 else if (other_prev == 0)
    3601              :                   {
    3602            7 :                     buckets[b] = j;
    3603            7 :                     other_prev = j;
    3604              :                   }
    3605              :                 else
    3606              :                   {
    3607            2 :                     chains[other_prev - 1] = j;
    3608            2 :                     other_prev = j;
    3609              :                   }
    3610              :               }
    3611           67 :             chains[this_prev - 1] = 0;
    3612           67 :             if (other_prev)
    3613            7 :               chains[other_prev - 1] = 0;
    3614           67 :             if (chains[i - 1] == 0)
    3615              :               {
    3616            5 :                 if (other_prev)
    3617            5 :                   b--;
    3618            5 :                 continue;
    3619              :               }
    3620              :           }
    3621        17108 :         for (j = chains[i - 1]; j; j = chains[j - 1])
    3622              :           {
    3623        10086 :             gimple *gk = SSA_NAME_DEF_STMT (ranges[k - 1].exp);
    3624        10086 :             gimple *gj = SSA_NAME_DEF_STMT (ranges[j - 1].exp);
    3625        10086 :             if (reassoc_stmt_dominates_stmt_p (gk, gj))
    3626         2907 :               k = j;
    3627              :           }
    3628         7022 :         tree type1 = TREE_TYPE (ranges[k - 1].exp);
    3629         7022 :         tree type2 = NULL_TREE;
    3630         7022 :         candidates.truncate (0);
    3631         7022 :         if (POINTER_TYPE_P (type1) || TREE_CODE (type1) == OFFSET_TYPE)
    3632          704 :           type1 = pointer_sized_int_node;
    3633        24130 :         for (j = i; j; j = chains[j - 1])
    3634              :           {
    3635        17108 :             tree type = TREE_TYPE (ranges[j - 1].exp);
    3636        17108 :             if (POINTER_TYPE_P (type) || TREE_CODE (type) == OFFSET_TYPE)
    3637         1454 :               type = pointer_sized_int_node;
    3638        17108 :             if ((b % 4) == 3)
    3639              :               {
    3640              :                 /* For the signed < 0 cases, the types should be
    3641              :                    really compatible (all signed with the same precision,
    3642              :                    instead put ranges that have different in_p from
    3643              :                    k first.  */
    3644         3674 :                 if (!useless_type_conversion_p (type1, type))
    3645            0 :                   continue;
    3646         3674 :                 if (ranges[j - 1].in_p != ranges[k - 1].in_p)
    3647         1009 :                   candidates.safe_push (&ranges[j - 1]);
    3648         3674 :                 type2 = type1;
    3649         3674 :                 continue;
    3650              :               }
    3651        13434 :             if (j == k
    3652        13434 :                 || useless_type_conversion_p (type1, type))
    3653              :               ;
    3654          413 :             else if (type2 == NULL_TREE
    3655          413 :                      || useless_type_conversion_p (type2, type))
    3656              :               {
    3657          413 :                 if (type2 == NULL_TREE)
    3658          399 :                   type2 = type;
    3659          413 :                 candidates.safe_push (&ranges[j - 1]);
    3660              :               }
    3661              :           }
    3662         7022 :         unsigned l = candidates.length ();
    3663        24130 :         for (j = i; j; j = chains[j - 1])
    3664              :           {
    3665        17108 :             tree type = TREE_TYPE (ranges[j - 1].exp);
    3666        17108 :             if (j == k)
    3667         7022 :               continue;
    3668        10086 :             if (POINTER_TYPE_P (type) || TREE_CODE (type) == OFFSET_TYPE)
    3669          750 :               type = pointer_sized_int_node;
    3670        10086 :             if ((b % 4) == 3)
    3671              :               {
    3672         2034 :                 if (!useless_type_conversion_p (type1, type))
    3673            0 :                   continue;
    3674         2034 :                 if (ranges[j - 1].in_p == ranges[k - 1].in_p)
    3675         1025 :                   candidates.safe_push (&ranges[j - 1]);
    3676         2034 :                 continue;
    3677              :               }
    3678         8052 :             if (useless_type_conversion_p (type1, type))
    3679              :               ;
    3680          826 :             else if (type2 == NULL_TREE
    3681          413 :                      || useless_type_conversion_p (type2, type))
    3682          413 :               continue;
    3683         7639 :             candidates.safe_push (&ranges[j - 1]);
    3684              :           }
    3685         7022 :         gimple_seq seq = NULL;
    3686         7022 :         tree op = NULL_TREE;
    3687         7022 :         unsigned int id;
    3688         7022 :         struct range_entry *r;
    3689         7022 :         candidates.safe_push (&ranges[k - 1]);
    3690        24130 :         FOR_EACH_VEC_ELT (candidates, id, r)
    3691              :           {
    3692        17108 :             gimple *g;
    3693        17108 :             enum tree_code code;
    3694        17108 :             if (id == 0)
    3695              :               {
    3696         7022 :                 op = r->exp;
    3697         7022 :                 continue;
    3698              :               }
    3699        10086 :             if (id == l
    3700         8678 :                 || POINTER_TYPE_P (TREE_TYPE (op))
    3701        18147 :                 || TREE_CODE (TREE_TYPE (op)) == OFFSET_TYPE)
    3702              :               {
    3703         2030 :                 code = (b % 4) == 3 ? BIT_NOT_EXPR : NOP_EXPR;
    3704         2030 :                 tree type3 = id >= l ? type1 : pointer_sized_int_node;
    3705         2030 :                 if (code == BIT_NOT_EXPR
    3706         2030 :                     && TREE_CODE (TREE_TYPE (op)) == OFFSET_TYPE)
    3707              :                   {
    3708            0 :                     g = gimple_build_assign (make_ssa_name (type3),
    3709              :                                              NOP_EXPR, op);
    3710            0 :                     gimple_seq_add_stmt_without_update (&seq, g);
    3711            0 :                     op = gimple_assign_lhs (g);
    3712              :                   }
    3713         2030 :                 g = gimple_build_assign (make_ssa_name (type3), code, op);
    3714         2030 :                 gimple_seq_add_stmt_without_update (&seq, g);
    3715         2030 :                 op = gimple_assign_lhs (g);
    3716              :               }
    3717        10086 :             tree type = TREE_TYPE (r->exp);
    3718        10086 :             tree exp = r->exp;
    3719        10086 :             if (POINTER_TYPE_P (type)
    3720         9331 :                 || TREE_CODE (type) == OFFSET_TYPE
    3721        19412 :                 || (id >= l && !useless_type_conversion_p (type1, type)))
    3722              :               {
    3723          760 :                 tree type3 = id >= l ? type1 : pointer_sized_int_node;
    3724          760 :                 g = gimple_build_assign (make_ssa_name (type3), NOP_EXPR, exp);
    3725          760 :                 gimple_seq_add_stmt_without_update (&seq, g);
    3726          760 :                 exp = gimple_assign_lhs (g);
    3727              :               }
    3728        10086 :             if ((b % 4) == 3)
    3729         3358 :               code = r->in_p ? BIT_IOR_EXPR : BIT_AND_EXPR;
    3730              :             else
    3731         8052 :               code = (b % 4) == 1 ? BIT_AND_EXPR : BIT_IOR_EXPR;
    3732        20172 :             g = gimple_build_assign (make_ssa_name (id >= l ? type1 : type2),
    3733              :                                      code, op, exp);
    3734        10086 :             gimple_seq_add_stmt_without_update (&seq, g);
    3735        10086 :             op = gimple_assign_lhs (g);
    3736              :           }
    3737         7022 :         type1 = TREE_TYPE (ranges[k - 1].exp);
    3738         7022 :         if (POINTER_TYPE_P (type1) || TREE_CODE (type1) == OFFSET_TYPE)
    3739              :           {
    3740          704 :             gimple *g
    3741          704 :               = gimple_build_assign (make_ssa_name (type1), NOP_EXPR, op);
    3742          704 :             gimple_seq_add_stmt_without_update (&seq, g);
    3743          704 :             op = gimple_assign_lhs (g);
    3744              :           }
    3745         7022 :         candidates.pop ();
    3746         7022 :         if (update_range_test (&ranges[k - 1], NULL, candidates.address (),
    3747              :                                candidates.length (), opcode, ops, op,
    3748              :                                seq, ranges[k - 1].in_p, ranges[k - 1].low,
    3749              :                                ranges[k - 1].high))
    3750              :           any_changes = true;
    3751              :         else
    3752            0 :           gimple_seq_discard (seq);
    3753         7084 :         if ((b % 4) == 2 && buckets[b] != i)
    3754              :           /* There is more work to do for this bucket.  */
    3755            2 :           b--;
    3756              :       }
    3757              : 
    3758      1092264 :   return any_changes;
    3759      1092264 : }
    3760              : 
    3761              : /* Attempt to optimize for signed a and b where b is known to be >= 0:
    3762              :    a >= 0 && a < b into (unsigned) a < (unsigned) b
    3763              :    a >= 0 && a <= b into (unsigned) a <= (unsigned) b  */
    3764              : 
    3765              : static bool
    3766      1092264 : optimize_range_tests_var_bound (enum tree_code opcode, int first, int length,
    3767              :                                 vec<operand_entry *> *ops,
    3768              :                                 struct range_entry *ranges,
    3769              :                                 basic_block first_bb)
    3770              : {
    3771      1092264 :   int i;
    3772      1092264 :   bool any_changes = false;
    3773      1092264 :   hash_map<tree, int> *map = NULL;
    3774              : 
    3775      1907984 :   for (i = first; i < length; i++)
    3776              :     {
    3777       815720 :       if (ranges[i].exp == NULL_TREE
    3778       796186 :           || TREE_CODE (ranges[i].exp) != SSA_NAME
    3779       791513 :           || !ranges[i].in_p)
    3780       325080 :         continue;
    3781              : 
    3782       490640 :       tree type = TREE_TYPE (ranges[i].exp);
    3783       935943 :       if (!INTEGRAL_TYPE_P (type)
    3784       480816 :           || TYPE_UNSIGNED (type)
    3785       201601 :           || ranges[i].low == NULL_TREE
    3786       192334 :           || !integer_zerop (ranges[i].low)
    3787       564538 :           || ranges[i].high != NULL_TREE)
    3788       445303 :         continue;
    3789              :       /* EXP >= 0 here.  */
    3790        45337 :       if (map == NULL)
    3791        43672 :         map = new hash_map <tree, int>;
    3792        45337 :       map->put (ranges[i].exp, i);
    3793              :     }
    3794              : 
    3795      1092264 :   if (map == NULL)
    3796              :     return false;
    3797              : 
    3798       134710 :   for (i = 0; i < length; i++)
    3799              :     {
    3800        91038 :       bool in_p = ranges[i].in_p;
    3801        91038 :       if (ranges[i].low == NULL_TREE
    3802        90339 :           || ranges[i].high == NULL_TREE)
    3803        90481 :         continue;
    3804        43224 :       if (!integer_zerop (ranges[i].low)
    3805        43224 :           || !integer_zerop (ranges[i].high))
    3806              :         {
    3807         8886 :           if (ranges[i].exp
    3808         4443 :               && TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) == 1
    3809            0 :               && TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
    3810            0 :               && integer_onep (ranges[i].low)
    3811         4443 :               && integer_onep (ranges[i].high))
    3812            0 :             in_p = !in_p;
    3813              :           else
    3814         4443 :             continue;
    3815              :         }
    3816              : 
    3817        38781 :       gimple *stmt;
    3818        38781 :       tree_code ccode;
    3819        38781 :       tree rhs1, rhs2;
    3820        38781 :       if (ranges[i].exp)
    3821              :         {
    3822        37691 :           if (TREE_CODE (ranges[i].exp) != SSA_NAME)
    3823            5 :             continue;
    3824        37686 :           stmt = SSA_NAME_DEF_STMT (ranges[i].exp);
    3825        37686 :           if (!is_gimple_assign (stmt))
    3826          941 :             continue;
    3827        36745 :           ccode = gimple_assign_rhs_code (stmt);
    3828        36745 :           rhs1 = gimple_assign_rhs1 (stmt);
    3829        36745 :           rhs2 = gimple_assign_rhs2 (stmt);
    3830              :         }
    3831              :       else
    3832              :         {
    3833         1090 :           operand_entry *oe = (*ops)[ranges[i].idx];
    3834         1090 :           stmt = last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id));
    3835         1090 :           if (gimple_code (stmt) != GIMPLE_COND)
    3836            0 :             continue;
    3837         1090 :           ccode = gimple_cond_code (stmt);
    3838         1090 :           rhs1 = gimple_cond_lhs (stmt);
    3839         1090 :           rhs2 = gimple_cond_rhs (stmt);
    3840              :         }
    3841              : 
    3842        37835 :       if (TREE_CODE (rhs1) != SSA_NAME
    3843        37326 :           || rhs2 == NULL_TREE
    3844        37223 :           || TREE_CODE (rhs2) != SSA_NAME)
    3845          675 :         continue;
    3846              : 
    3847        37160 :       switch (ccode)
    3848              :         {
    3849        35958 :         case GT_EXPR:
    3850        35958 :         case GE_EXPR:
    3851        35958 :         case LT_EXPR:
    3852        35958 :         case LE_EXPR:
    3853        35958 :           break;
    3854         1202 :         default:
    3855         1202 :           continue;
    3856              :         }
    3857        35958 :       if (in_p)
    3858          805 :         ccode = invert_tree_comparison (ccode, false);
    3859        35958 :       switch (ccode)
    3860              :         {
    3861        14850 :         case GT_EXPR:
    3862        14850 :         case GE_EXPR:
    3863        14850 :           std::swap (rhs1, rhs2);
    3864        14850 :           ccode = swap_tree_comparison (ccode);
    3865        14850 :           break;
    3866              :         case LT_EXPR:
    3867              :         case LE_EXPR:
    3868              :           break;
    3869            0 :         default:
    3870            0 :           gcc_unreachable ();
    3871              :         }
    3872              : 
    3873        35958 :       int *idx = map->get (rhs1);
    3874        35958 :       if (idx == NULL)
    3875         1071 :         continue;
    3876              : 
    3877              :       /* maybe_optimize_range_tests allows statements without side-effects
    3878              :          in the basic blocks as long as they are consumed in the same bb.
    3879              :          Make sure rhs2's def stmt is not among them, otherwise we can't
    3880              :          use safely get_nonzero_bits on it.  E.g. in:
    3881              :           # RANGE [-83, 1] NONZERO 173
    3882              :           # k_32 = PHI <k_47(13), k_12(9)>
    3883              :          ...
    3884              :           if (k_32 >= 0)
    3885              :             goto <bb 5>; [26.46%]
    3886              :           else
    3887              :             goto <bb 9>; [73.54%]
    3888              : 
    3889              :           <bb 5> [local count: 140323371]:
    3890              :           # RANGE [0, 1] NONZERO 1
    3891              :           _5 = (int) k_32;
    3892              :           # RANGE [0, 4] NONZERO 4
    3893              :           _21 = _5 << 2;
    3894              :           # RANGE [0, 4] NONZERO 4
    3895              :           iftmp.0_44 = (char) _21;
    3896              :           if (k_32 < iftmp.0_44)
    3897              :             goto <bb 6>; [84.48%]
    3898              :           else
    3899              :             goto <bb 9>; [15.52%]
    3900              :          the ranges on _5/_21/iftmp.0_44 are flow sensitive, assume that
    3901              :          k_32 >= 0.  If we'd optimize k_32 >= 0 to true and k_32 < iftmp.0_44
    3902              :          to (unsigned) k_32 < (unsigned) iftmp.0_44, then we would execute
    3903              :          those stmts even for negative k_32 and the value ranges would be no
    3904              :          longer guaranteed and so the optimization would be invalid.  */
    3905        34889 :       while (opcode == ERROR_MARK)
    3906              :         {
    3907          640 :           gimple *g = SSA_NAME_DEF_STMT (rhs2);
    3908          640 :           basic_block bb2 = gimple_bb (g);
    3909          640 :           if (bb2
    3910          640 :               && bb2 != first_bb
    3911          640 :               && dominated_by_p (CDI_DOMINATORS, bb2, first_bb))
    3912              :             {
    3913              :               /* As an exception, handle a few common cases.  */
    3914          558 :               if (gimple_assign_cast_p (g)
    3915          558 :                   && INTEGRAL_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (g))))
    3916              :                 {
    3917           42 :                   tree op0 = gimple_assign_rhs1 (g);
    3918           42 :                   if (TYPE_UNSIGNED (TREE_TYPE (op0))
    3919           42 :                       && (TYPE_PRECISION (TREE_TYPE (rhs2))
    3920           12 :                           > TYPE_PRECISION (TREE_TYPE (op0))))
    3921              :                     /* Zero-extension is always ok.  */
    3922              :                     break;
    3923           32 :                   else if (TYPE_PRECISION (TREE_TYPE (rhs2))
    3924           32 :                            == TYPE_PRECISION (TREE_TYPE (op0))
    3925           32 :                            && TREE_CODE (op0) == SSA_NAME)
    3926              :                     {
    3927              :                       /* Cast from signed to unsigned or vice versa.  Retry
    3928              :                          with the op0 as new rhs2.  */
    3929            2 :                       rhs2 = op0;
    3930            2 :                       continue;
    3931              :                     }
    3932              :                 }
    3933          516 :               else if (is_gimple_assign (g)
    3934          516 :                        && gimple_assign_rhs_code (g) == BIT_AND_EXPR
    3935            0 :                        && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST
    3936         1032 :                        && !wi::neg_p (wi::to_wide (gimple_assign_rhs2 (g))))
    3937              :                 /* Masking with INTEGER_CST with MSB clear is always ok
    3938              :                    too.  */
    3939              :                 break;
    3940              :               rhs2 = NULL_TREE;
    3941              :             }
    3942              :           break;
    3943              :         }
    3944        34341 :       if (rhs2 == NULL_TREE)
    3945          546 :         continue;
    3946              : 
    3947        34898 :       wide_int nz = get_nonzero_bits (rhs2);
    3948        34341 :       if (wi::neg_p (nz))
    3949        33784 :         continue;
    3950              : 
    3951              :       /* We have EXP < RHS2 or EXP <= RHS2 where EXP >= 0
    3952              :          and RHS2 is known to be RHS2 >= 0.  */
    3953          557 :       tree utype = unsigned_type_for (TREE_TYPE (rhs1));
    3954              : 
    3955          557 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3956              :         {
    3957            7 :           struct range_entry *r = &ranges[*idx];
    3958            7 :           fprintf (dump_file, "Optimizing range test ");
    3959            7 :           print_generic_expr (dump_file, r->exp);
    3960            7 :           fprintf (dump_file, " +[");
    3961            7 :           print_generic_expr (dump_file, r->low);
    3962            7 :           fprintf (dump_file, ", ");
    3963            7 :           print_generic_expr (dump_file, r->high);
    3964            7 :           fprintf (dump_file, "] and comparison ");
    3965            7 :           print_generic_expr (dump_file, rhs1);
    3966            7 :           fprintf (dump_file, " %s ", op_symbol_code (ccode));
    3967            7 :           print_generic_expr (dump_file, rhs2);
    3968            7 :           fprintf (dump_file, "\n into (");
    3969            7 :           print_generic_expr (dump_file, utype);
    3970            7 :           fprintf (dump_file, ") ");
    3971            7 :           print_generic_expr (dump_file, rhs1);
    3972            7 :           fprintf (dump_file, " %s (", op_symbol_code (ccode));
    3973            7 :           print_generic_expr (dump_file, utype);
    3974            7 :           fprintf (dump_file, ") ");
    3975            7 :           print_generic_expr (dump_file, rhs2);
    3976            7 :           fprintf (dump_file, "\n");
    3977              :         }
    3978              : 
    3979          557 :       operand_entry *oe = (*ops)[ranges[i].idx];
    3980          557 :       ranges[i].in_p = 0;
    3981          557 :       if (opcode == BIT_IOR_EXPR
    3982          518 :           || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
    3983              :         {
    3984           40 :           ranges[i].in_p = 1;
    3985           40 :           ccode = invert_tree_comparison (ccode, false);
    3986              :         }
    3987              : 
    3988          557 :       unsigned int uid = gimple_uid (stmt);
    3989          557 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    3990          557 :       gimple *g = gimple_build_assign (make_ssa_name (utype), NOP_EXPR, rhs1);
    3991          557 :       gimple_set_uid (g, uid);
    3992          557 :       rhs1 = gimple_assign_lhs (g);
    3993          557 :       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    3994          557 :       if (!useless_type_conversion_p (utype, TREE_TYPE (rhs2)))
    3995              :         {
    3996          557 :           g = gimple_build_assign (make_ssa_name (utype), NOP_EXPR, rhs2);
    3997          557 :           gimple_set_uid (g, uid);
    3998          557 :           rhs2 = gimple_assign_lhs (g);
    3999          557 :           gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    4000              :         }
    4001          557 :       if (tree_swap_operands_p (rhs1, rhs2))
    4002              :         {
    4003          510 :           std::swap (rhs1, rhs2);
    4004          510 :           ccode = swap_tree_comparison (ccode);
    4005              :         }
    4006          557 :       if (gimple_code (stmt) == GIMPLE_COND)
    4007              :         {
    4008            8 :           gcond *c = as_a <gcond *> (stmt);
    4009            8 :           gimple_cond_set_code (c, ccode);
    4010            8 :           gimple_cond_set_lhs (c, rhs1);
    4011            8 :           gimple_cond_set_rhs (c, rhs2);
    4012            8 :           update_stmt (stmt);
    4013              :         }
    4014              :       else
    4015              :         {
    4016          549 :           tree ctype = oe->op ? TREE_TYPE (oe->op) : boolean_type_node;
    4017          549 :           if (!INTEGRAL_TYPE_P (ctype)
    4018          549 :               || (TREE_CODE (ctype) != BOOLEAN_TYPE
    4019            3 :                   && TYPE_PRECISION (ctype) != 1))
    4020            3 :             ctype = boolean_type_node;
    4021          549 :           g = gimple_build_assign (make_ssa_name (ctype), ccode, rhs1, rhs2);
    4022          549 :           gimple_set_uid (g, uid);
    4023          549 :           gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    4024          549 :           if (oe->op && ctype != TREE_TYPE (oe->op))
    4025              :             {
    4026            3 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (oe->op)),
    4027              :                                        NOP_EXPR, gimple_assign_lhs (g));
    4028            3 :               gimple_set_uid (g, uid);
    4029            3 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    4030              :             }
    4031          549 :           ranges[i].exp = gimple_assign_lhs (g);
    4032          549 :           oe->op = ranges[i].exp;
    4033          549 :           ranges[i].low = build_zero_cst (TREE_TYPE (ranges[i].exp));
    4034          549 :           ranges[i].high = ranges[i].low;
    4035              :         }
    4036          557 :       oe = (*ops)[ranges[*idx].idx];
    4037              :       /* Now change all the other range test immediate uses, so that
    4038              :          those tests will be optimized away.  */
    4039          557 :       if (opcode == ERROR_MARK)
    4040              :         {
    4041           11 :           if (oe->op)
    4042            3 :             oe->op = build_int_cst (TREE_TYPE (oe->op),
    4043            3 :                                     oe->rank == BIT_IOR_EXPR ? 0 : 1);
    4044              :           else
    4045            8 :             oe->op = (oe->rank == BIT_IOR_EXPR
    4046            8 :                       ? boolean_false_node : boolean_true_node);
    4047              :         }
    4048              :       else
    4049          546 :         oe->op = error_mark_node;
    4050          557 :       ranges[*idx].exp = NULL_TREE;
    4051          557 :       ranges[*idx].low = NULL_TREE;
    4052          557 :       ranges[*idx].high = NULL_TREE;
    4053          557 :       any_changes = true;
    4054              :     }
    4055              : 
    4056        43672 :   delete map;
    4057              :   return any_changes;
    4058              : }
    4059              : 
    4060              : /* Optimize range tests, similarly how fold_range_test optimizes
    4061              :    it on trees.  The tree code for the binary
    4062              :    operation between all the operands is OPCODE.
    4063              :    If OPCODE is ERROR_MARK, optimize_range_tests is called from within
    4064              :    maybe_optimize_range_tests for inter-bb range optimization.
    4065              :    In that case if oe->op is NULL, oe->id is bb->index whose
    4066              :    GIMPLE_COND is && or ||ed into the test, and oe->rank says
    4067              :    the actual opcode.
    4068              :    FIRST_BB is the first basic block if OPCODE is ERROR_MARK.  */
    4069              : 
    4070              : static bool
    4071      1092381 : optimize_range_tests (enum tree_code opcode,
    4072              :                       vec<operand_entry *> *ops, basic_block first_bb)
    4073              : {
    4074      1092381 :   unsigned int length = ops->length (), i, j, first;
    4075      1092381 :   operand_entry *oe;
    4076      1092381 :   struct range_entry *ranges;
    4077      2184645 :   bool any_changes = false;
    4078              : 
    4079      1092381 :   if (length == 1)
    4080              :     return false;
    4081              : 
    4082      1092264 :   ranges = XNEWVEC (struct range_entry, length);
    4083      4486576 :   for (i = 0; i < length; i++)
    4084              :     {
    4085      2302048 :       oe = (*ops)[i];
    4086      2302048 :       ranges[i].idx = i;
    4087      2302048 :       init_range_entry (ranges + i, oe->op,
    4088      2302048 :                         oe->op
    4089              :                         ? NULL
    4090       270277 :                         : last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id)));
    4091              :       /* For | invert it now, we will invert it again before emitting
    4092              :          the optimized expression.  */
    4093      2302048 :       if (opcode == BIT_IOR_EXPR
    4094      1615708 :           || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
    4095       887270 :         ranges[i].in_p = !ranges[i].in_p;
    4096              :     }
    4097              : 
    4098      1092264 :   qsort (ranges, length, sizeof (*ranges), range_entry_cmp);
    4099      3670856 :   for (i = 0; i < length; i++)
    4100      1868014 :     if (ranges[i].exp != NULL_TREE && TREE_CODE (ranges[i].exp) == SSA_NAME)
    4101              :       break;
    4102              : 
    4103              :   /* Try to merge ranges.  */
    4104      1894444 :   for (first = i; i < length; i++)
    4105              :     {
    4106       802180 :       tree low = ranges[i].low;
    4107       802180 :       tree high = ranges[i].high;
    4108       802180 :       int in_p = ranges[i].in_p;
    4109       802180 :       int update_fail_count = 0;
    4110              : 
    4111       815720 :       for (j = i + 1; j < length; j++)
    4112              :         {
    4113       434034 :           if (ranges[i].exp != ranges[j].exp)
    4114              :             break;
    4115        35069 :           if (!merge_ranges (&in_p, &low, &high, in_p, low, high,
    4116        35069 :                              ranges[j].in_p, ranges[j].low, ranges[j].high))
    4117              :             break;
    4118              :         }
    4119              : 
    4120       802180 :       if (j == i + 1)
    4121       789237 :         continue;
    4122              : 
    4123        12943 :       if (update_range_test (ranges + i, ranges + i + 1, NULL, j - i - 1,
    4124              :                              opcode, ops, ranges[i].exp, NULL, in_p,
    4125              :                              low, high))
    4126              :         {
    4127        12943 :           i = j - 1;
    4128        12943 :           any_changes = true;
    4129              :         }
    4130              :       /* Avoid quadratic complexity if all merge_ranges calls would succeed,
    4131              :          while update_range_test would fail.  */
    4132              :       else if (update_fail_count == 64)
    4133              :         i = j - 1;
    4134              :       else
    4135        12943 :         ++update_fail_count;
    4136              :     }
    4137              : 
    4138      1092264 :   any_changes |= optimize_range_tests_1 (opcode, first, length, true,
    4139              :                                          ops, ranges);
    4140              : 
    4141      1092264 :   if (BRANCH_COST (optimize_function_for_speed_p (cfun), false) >= 2)
    4142      1092248 :     any_changes |= optimize_range_tests_1 (opcode, first, length, false,
    4143              :                                            ops, ranges);
    4144      1092264 :   if (lshift_cheap_p (optimize_function_for_speed_p (cfun)))
    4145      1092264 :     any_changes |= optimize_range_tests_to_bit_test (opcode, first, length,
    4146              :                                                      ops, ranges);
    4147      1092264 :   any_changes |= optimize_range_tests_var_bound (opcode, first, length, ops,
    4148              :                                                  ranges, first_bb);
    4149      1092264 :   any_changes |= optimize_range_tests_cmp_bitwise (opcode, first, length,
    4150              :                                                    ops, ranges);
    4151              : 
    4152      1092264 :   if (any_changes && opcode != ERROR_MARK)
    4153              :     {
    4154              :       j = 0;
    4155        36334 :       FOR_EACH_VEC_ELT (*ops, i, oe)
    4156              :         {
    4157        25405 :           if (oe->op == error_mark_node)
    4158        12522 :             continue;
    4159        12883 :           else if (i != j)
    4160         5321 :             (*ops)[j] = oe;
    4161        12883 :           j++;
    4162              :         }
    4163        10929 :       ops->truncate (j);
    4164              :     }
    4165              : 
    4166      1092264 :   XDELETEVEC (ranges);
    4167      1092264 :   return any_changes;
    4168              : }
    4169              : 
    4170              : /* A subroutine of optimize_vec_cond_expr to extract and canonicalize
    4171              :    the operands of the VEC_COND_EXPR.  Returns ERROR_MARK on failure,
    4172              :    otherwise the comparison code.  TYPE is a return value that is set
    4173              :    to type of comparison.  */
    4174              : 
    4175              : static tree_code
    4176        56776 : ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type,
    4177              :                   tree *lhs, tree *rhs, gassign **vcond)
    4178              : {
    4179        56776 :   if (TREE_CODE (var) != SSA_NAME)
    4180              :     return ERROR_MARK;
    4181              : 
    4182        49715 :   gassign *stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (var));
    4183        36296 :   if (stmt == NULL)
    4184              :     return ERROR_MARK;
    4185        36296 :   if (vcond)
    4186        36296 :     *vcond = stmt;
    4187              : 
    4188              :   /* ??? If we start creating more COND_EXPR, we could perform
    4189              :      this same optimization with them.  For now, simplify.  */
    4190        48635 :   if (gimple_assign_rhs_code (stmt) != VEC_COND_EXPR)
    4191              :     return ERROR_MARK;
    4192              : 
    4193         1375 :   tree cond = gimple_assign_rhs1 (stmt);
    4194         1375 :   tree_code cmp = TREE_CODE (cond);
    4195         1375 :   if (cmp != SSA_NAME)
    4196              :     return ERROR_MARK;
    4197              : 
    4198         1353 :   gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (cond));
    4199         1349 :   if (assign == NULL
    4200         1349 :       || TREE_CODE_CLASS (gimple_assign_rhs_code (assign)) != tcc_comparison)
    4201              :     return ERROR_MARK;
    4202              : 
    4203         1236 :   cmp = gimple_assign_rhs_code (assign);
    4204         1236 :   if (lhs)
    4205         1236 :     *lhs = gimple_assign_rhs1 (assign);
    4206         1236 :   if (rhs)
    4207         2472 :     *rhs = gimple_assign_rhs2 (assign);
    4208              : 
    4209              :   /* ??? For now, allow only canonical true and false result vectors.
    4210              :      We could expand this to other constants should the need arise,
    4211              :      but at the moment we don't create them.  */
    4212         1236 :   tree t = gimple_assign_rhs2 (stmt);
    4213         1236 :   tree f = gimple_assign_rhs3 (stmt);
    4214         1236 :   bool inv;
    4215         1236 :   if (integer_all_onesp (t))
    4216              :     inv = false;
    4217         1116 :   else if (integer_all_onesp (f))
    4218              :     {
    4219            1 :       cmp = invert_tree_comparison (cmp, false);
    4220            1 :       inv = true;
    4221              :     }
    4222              :   else
    4223              :     return ERROR_MARK;
    4224          121 :   if (!integer_zerop (inv ? t : f))
    4225              :     return ERROR_MARK;
    4226              : 
    4227              :   /* Success!  */
    4228          102 :   if (rets)
    4229          102 :     *rets = assign;
    4230          102 :   if (reti)
    4231           72 :     *reti = inv;
    4232          102 :   if (type)
    4233           72 :     *type = TREE_TYPE (cond);
    4234              :   return cmp;
    4235              : }
    4236              : 
    4237              : /* Optimize the condition of VEC_COND_EXPRs which have been combined
    4238              :    with OPCODE (either BIT_AND_EXPR or BIT_IOR_EXPR).  */
    4239              : 
    4240              : static bool
    4241        26638 : optimize_vec_cond_expr (tree_code opcode, vec<operand_entry *> *ops)
    4242              : {
    4243        26638 :   unsigned int length = ops->length (), i, j;
    4244        26638 :   bool any_changes = false;
    4245              : 
    4246        26638 :   if (length == 1)
    4247              :     return false;
    4248              : 
    4249        83366 :   for (i = 0; i < length; ++i)
    4250              :     {
    4251        56728 :       tree &elt0 = (*ops)[i]->op;
    4252              : 
    4253        56728 :       gassign *stmt0, *vcond0;
    4254        56728 :       bool invert;
    4255        56728 :       tree type, lhs0, rhs0;
    4256        56728 :       tree_code cmp0 = ovce_extract_ops (elt0, &stmt0, &invert, &type, &lhs0,
    4257              :                                          &rhs0, &vcond0);
    4258        56728 :       if (cmp0 == ERROR_MARK)
    4259        56656 :         continue;
    4260              : 
    4261          120 :       for (j = i + 1; j < length; ++j)
    4262              :         {
    4263           48 :           tree &elt1 = (*ops)[j]->op;
    4264              : 
    4265           48 :           gassign *stmt1, *vcond1;
    4266           48 :           tree lhs1, rhs1;
    4267           48 :           tree_code cmp1 = ovce_extract_ops (elt1, &stmt1, NULL, NULL, &lhs1,
    4268              :                                              &rhs1, &vcond1);
    4269           48 :           if (cmp1 == ERROR_MARK)
    4270           42 :             continue;
    4271              : 
    4272           30 :           tree comb;
    4273           30 :           if (opcode == BIT_AND_EXPR)
    4274            0 :             comb = maybe_fold_and_comparisons (type, TRUTH_AND_EXPR,
    4275              :                                                cmp0, lhs0, rhs0,
    4276              :                                                cmp1, lhs1, rhs1);
    4277           30 :           else if (opcode == BIT_IOR_EXPR)
    4278           30 :             comb = maybe_fold_or_comparisons (type, TRUTH_OR_EXPR,
    4279              :                                               cmp0, lhs0, rhs0,
    4280              :                                               cmp1, lhs1, rhs1);
    4281              :           else
    4282            0 :             gcc_unreachable ();
    4283           30 :           if (comb == NULL)
    4284           24 :             continue;
    4285              : 
    4286              :           /* Success! */
    4287            6 :           if (dump_file && (dump_flags & TDF_DETAILS))
    4288              :             {
    4289            0 :               fprintf (dump_file, "Transforming ");
    4290            0 :               print_generic_expr (dump_file, gimple_assign_lhs (stmt0));
    4291            0 :               fprintf (dump_file, " %c ", opcode == BIT_AND_EXPR ? '&' : '|');
    4292            0 :               print_generic_expr (dump_file, gimple_assign_lhs (stmt1));
    4293            0 :               fprintf (dump_file, " into ");
    4294            0 :               print_generic_expr (dump_file, comb);
    4295            0 :               fputc ('\n', dump_file);
    4296              :             }
    4297              : 
    4298            6 :           gimple_stmt_iterator gsi = gsi_for_stmt (vcond0);
    4299            6 :           tree exp = force_gimple_operand_gsi (&gsi, comb, true, NULL_TREE,
    4300              :                                                true, GSI_SAME_STMT);
    4301           18 :           tree res = gimple_build (&gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
    4302            6 :                                    VEC_COND_EXPR, TREE_TYPE (elt0), exp,
    4303              :                                    constant_boolean_node (true,
    4304            6 :                                                           TREE_TYPE (elt0)),
    4305              :                                    constant_boolean_node (false,
    4306            6 :                                                           TREE_TYPE (elt0)));
    4307            6 :           elt0 = res;
    4308              : 
    4309            6 :           elt1 = error_mark_node;
    4310            6 :           any_changes = true;
    4311              :         }
    4312              :     }
    4313              : 
    4314        26638 :   if (any_changes)
    4315              :     {
    4316              :       operand_entry *oe;
    4317              :       j = 0;
    4318           18 :       FOR_EACH_VEC_ELT (*ops, i, oe)
    4319              :         {
    4320           12 :           if (oe->op == error_mark_node)
    4321            6 :             continue;
    4322            6 :           else if (i != j)
    4323            0 :             (*ops)[j] = oe;
    4324            6 :           j++;
    4325              :         }
    4326            6 :       ops->truncate (j);
    4327              :     }
    4328              : 
    4329              :   return any_changes;
    4330              : }
    4331              : 
    4332              : /* Return true if STMT is a cast like:
    4333              :    <bb N>:
    4334              :    ...
    4335              :    _123 = (int) _234;
    4336              : 
    4337              :    <bb M>:
    4338              :    # _345 = PHI <_123(N), 1(...), 1(...)>
    4339              :    where _234 has bool type, _123 has single use and
    4340              :    bb N has a single successor M.  This is commonly used in
    4341              :    the last block of a range test.
    4342              : 
    4343              :    Also Return true if STMT is tcc_compare like:
    4344              :    <bb N>:
    4345              :    ...
    4346              :    _234 = a_2(D) == 2;
    4347              : 
    4348              :    <bb M>:
    4349              :    # _345 = PHI <_234(N), 1(...), 1(...)>
    4350              :    _346 = (int) _345;
    4351              :    where _234 has booltype, single use and
    4352              :    bb N has a single successor M.  This is commonly used in
    4353              :    the last block of a range test.  */
    4354              : 
    4355              : static bool
    4356     15917324 : final_range_test_p (gimple *stmt)
    4357              : {
    4358     15917324 :   basic_block bb, rhs_bb, lhs_bb;
    4359     15917324 :   edge e;
    4360     15917324 :   tree lhs, rhs;
    4361     15917324 :   use_operand_p use_p;
    4362     15917324 :   gimple *use_stmt;
    4363              : 
    4364     15917324 :   if (!gimple_assign_cast_p (stmt)
    4365     15917324 :       && (!is_gimple_assign (stmt)
    4366      5036390 :           || (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
    4367              :               != tcc_comparison)))
    4368              :     return false;
    4369       566518 :   bb = gimple_bb (stmt);
    4370       566518 :   if (!single_succ_p (bb))
    4371              :     return false;
    4372       566235 :   e = single_succ_edge (bb);
    4373       566235 :   if (e->flags & EDGE_COMPLEX)
    4374              :     return false;
    4375              : 
    4376       566235 :   lhs = gimple_assign_lhs (stmt);
    4377       566235 :   rhs = gimple_assign_rhs1 (stmt);
    4378       566235 :   if (gimple_assign_cast_p (stmt)
    4379       566235 :       && (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    4380       400150 :           || TREE_CODE (rhs) != SSA_NAME
    4381       379148 :           || TREE_CODE (TREE_TYPE (rhs)) != BOOLEAN_TYPE))
    4382              :     return false;
    4383              : 
    4384       188519 :   if (!gimple_assign_cast_p (stmt)
    4385       188519 :       && (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE))
    4386              :       return false;
    4387              : 
    4388              :   /* Test whether lhs is consumed only by a PHI in the only successor bb.  */
    4389       188041 :   if (!single_imm_use (lhs, &use_p, &use_stmt))
    4390              :     return false;
    4391              : 
    4392       184813 :   if (gimple_code (use_stmt) != GIMPLE_PHI
    4393       184813 :       || gimple_bb (use_stmt) != e->dest)
    4394              :     return false;
    4395              : 
    4396              :   /* And that the rhs is defined in the same loop.  */
    4397       183038 :   if (gimple_assign_cast_p (stmt))
    4398              :     {
    4399        65276 :       if (TREE_CODE (rhs) != SSA_NAME
    4400        65276 :           || !(rhs_bb = gimple_bb (SSA_NAME_DEF_STMT (rhs)))
    4401       130546 :           || !flow_bb_inside_loop_p (loop_containing_stmt (stmt), rhs_bb))
    4402           53 :         return false;
    4403              :     }
    4404              :   else
    4405              :     {
    4406       117762 :       if (TREE_CODE (lhs) != SSA_NAME
    4407       117762 :           || !(lhs_bb = gimple_bb (SSA_NAME_DEF_STMT (lhs)))
    4408       235524 :           || !flow_bb_inside_loop_p (loop_containing_stmt (stmt), lhs_bb))
    4409            0 :         return false;
    4410              :     }
    4411              : 
    4412              :   return true;
    4413              : }
    4414              : 
    4415              : /* Return true if BB is suitable basic block for inter-bb range test
    4416              :    optimization.  If BACKWARD is true, BB should be the only predecessor
    4417              :    of TEST_BB, and *OTHER_BB is either NULL and filled by the routine,
    4418              :    or compared with to find a common basic block to which all conditions
    4419              :    branch to if true resp. false.  If BACKWARD is false, TEST_BB should
    4420              :    be the only predecessor of BB.  *TEST_SWAPPED_P is set to true if
    4421              :    TEST_BB is a bb ending in condition where the edge to non-*OTHER_BB
    4422              :    block points to an empty block that falls through into *OTHER_BB and
    4423              :    the phi args match that path.  */
    4424              : 
    4425              : static bool
    4426     11551098 : suitable_cond_bb (basic_block bb, basic_block test_bb, basic_block *other_bb,
    4427              :                   bool *test_swapped_p, bool backward)
    4428              : {
    4429     11551098 :   edge_iterator ei, ei2;
    4430     11551098 :   edge e, e2;
    4431     11551098 :   gimple *stmt;
    4432     11551098 :   gphi_iterator gsi;
    4433     11551098 :   bool other_edge_seen = false;
    4434     11551098 :   bool is_cond;
    4435              : 
    4436     11551098 :   if (test_bb == bb)
    4437              :     return false;
    4438              :   /* Check last stmt first.  */
    4439     11551098 :   stmt = last_nondebug_stmt (bb);
    4440     11551098 :   if (stmt == NULL
    4441     10672586 :       || (gimple_code (stmt) != GIMPLE_COND
    4442       532453 :           && (backward || !final_range_test_p (stmt)))
    4443     10180232 :       || gimple_visited_p (stmt)
    4444     10120697 :       || stmt_could_throw_p (cfun, stmt)
    4445     21671681 :       || *other_bb == bb)
    4446              :     return false;
    4447     10120580 :   is_cond = gimple_code (stmt) == GIMPLE_COND;
    4448     10120580 :   if (is_cond)
    4449              :     {
    4450              :       /* If last stmt is GIMPLE_COND, verify that one of the succ edges
    4451              :          goes to the next bb (if BACKWARD, it is TEST_BB), and the other
    4452              :          to *OTHER_BB (if not set yet, try to find it out).  */
    4453     10080595 :       if (EDGE_COUNT (bb->succs) != 2)
    4454              :         return false;
    4455     19860809 :       FOR_EACH_EDGE (e, ei, bb->succs)
    4456              :         {
    4457     16401793 :           if (!(e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
    4458              :             return false;
    4459     16401793 :           if (e->dest == test_bb)
    4460              :             {
    4461      5457504 :               if (backward)
    4462      5454468 :                 continue;
    4463              :               else
    4464              :                 return false;
    4465              :             }
    4466     10944289 :           if (e->dest == bb)
    4467              :             return false;
    4468     10785968 :           if (*other_bb == NULL)
    4469              :             {
    4470     26515614 :               FOR_EACH_EDGE (e2, ei2, test_bb->succs)
    4471     17677076 :                 if (!(e2->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
    4472              :                   return false;
    4473     17677076 :                 else if (e->dest == e2->dest)
    4474      2408999 :                   *other_bb = e->dest;
    4475      8838538 :               if (*other_bb == NULL)
    4476              :                 return false;
    4477              :             }
    4478      4356429 :           if (e->dest == *other_bb)
    4479              :             other_edge_seen = true;
    4480       937995 :           else if (backward)
    4481              :             return false;
    4482              :         }
    4483      3459016 :       if (*other_bb == NULL || !other_edge_seen)
    4484              :         return false;
    4485              :     }
    4486        39985 :   else if (single_succ (bb) != *other_bb)
    4487              :     return false;
    4488              : 
    4489              :   /* Now check all PHIs of *OTHER_BB.  */
    4490      3457643 :   e = find_edge (bb, *other_bb);
    4491      3457643 :   e2 = find_edge (test_bb, *other_bb);
    4492      3464243 :  retry:;
    4493      5062133 :   for (gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi); gsi_next (&gsi))
    4494              :     {
    4495      2682003 :       gphi *phi = gsi.phi ();
    4496              :       /* If both BB and TEST_BB end with GIMPLE_COND, all PHI arguments
    4497              :          corresponding to BB and TEST_BB predecessor must be the same.  */
    4498      2682003 :       if (!operand_equal_p (gimple_phi_arg_def (phi, e->dest_idx),
    4499      2682003 :                             gimple_phi_arg_def (phi, e2->dest_idx), 0))
    4500              :         {
    4501              :           /* Otherwise, if one of the blocks doesn't end with GIMPLE_COND,
    4502              :              one of the PHIs should have the lhs of the last stmt in
    4503              :              that block as PHI arg and that PHI should have 0 or 1
    4504              :              corresponding to it in all other range test basic blocks
    4505              :              considered.  */
    4506      1153637 :           if (!is_cond)
    4507              :             {
    4508        41997 :               if (gimple_phi_arg_def (phi, e->dest_idx)
    4509        41997 :                   == gimple_assign_lhs (stmt)
    4510        41997 :                   && (integer_zerop (gimple_phi_arg_def (phi, e2->dest_idx))
    4511        18042 :                       || integer_onep (gimple_phi_arg_def (phi,
    4512        18042 :                                                            e2->dest_idx))))
    4513        37609 :                 continue;
    4514              :             }
    4515              :           else
    4516              :             {
    4517      1111640 :               gimple *test_last = last_nondebug_stmt (test_bb);
    4518      1111640 :               if (gimple_code (test_last) == GIMPLE_COND)
    4519              :                 {
    4520      1075399 :                   if (backward ? e2->src != test_bb : e->src != bb)
    4521              :                     return false;
    4522              : 
    4523              :                   /* For last_bb, handle also:
    4524              :                      if (x_3(D) == 3)
    4525              :                        goto <bb 6>; [34.00%]
    4526              :                      else
    4527              :                        goto <bb 7>; [66.00%]
    4528              : 
    4529              :                      <bb 6> [local count: 79512730]:
    4530              : 
    4531              :                      <bb 7> [local count: 1073741824]:
    4532              :                      # prephitmp_7 = PHI <1(3), 1(4), 0(5), 1(2), 1(6)>
    4533              :                      where bb 7 is *OTHER_BB, but the PHI values from the
    4534              :                      earlier bbs match the path through the empty bb
    4535              :                      in between.  */
    4536      1070771 :                   edge e3;
    4537      1070771 :                   if (backward)
    4538      1477418 :                     e3 = EDGE_SUCC (test_bb,
    4539              :                                     e2 == EDGE_SUCC (test_bb, 0) ? 1 : 0);
    4540              :                   else
    4541        21893 :                     e3 = EDGE_SUCC (bb,
    4542              :                                     e == EDGE_SUCC (bb, 0) ? 1 : 0);
    4543      1070771 :                   if (empty_block_p (e3->dest)
    4544        35337 :                       && single_succ_p (e3->dest)
    4545        35337 :                       && single_succ (e3->dest) == *other_bb
    4546        35067 :                       && single_pred_p (e3->dest)
    4547      1105074 :                       && single_succ_edge (e3->dest)->flags == EDGE_FALLTHRU)
    4548              :                     {
    4549         6600 :                       if (backward)
    4550         5927 :                         e2 = single_succ_edge (e3->dest);
    4551              :                       else
    4552          673 :                         e = single_succ_edge (e3->dest);
    4553         6600 :                       if (test_swapped_p)
    4554          304 :                         *test_swapped_p = true;
    4555         6600 :                       goto retry;
    4556              :                     }
    4557              :                 }
    4558        36241 :               else if (gimple_phi_arg_def (phi, e2->dest_idx)
    4559        36241 :                        == gimple_assign_lhs (test_last)
    4560        69036 :                        && (integer_zerop (gimple_phi_arg_def (phi,
    4561        32795 :                                                               e->dest_idx))
    4562        14978 :                            || integer_onep (gimple_phi_arg_def (phi,
    4563        14978 :                                                                 e->dest_idx))))
    4564        31915 :                 continue;
    4565              :             }
    4566              : 
    4567              :           return false;
    4568              :         }
    4569              :     }
    4570              :   return true;
    4571              : }
    4572              : 
    4573              : /* Return true if BB doesn't have side-effects that would disallow
    4574              :    range test optimization, all SSA_NAMEs set in the bb are consumed
    4575              :    in the bb and there are no PHIs.  */
    4576              : 
    4577              : bool
    4578      5487798 : no_side_effect_bb (basic_block bb)
    4579              : {
    4580      5487798 :   gimple_stmt_iterator gsi;
    4581      5487798 :   gimple *last;
    4582              : 
    4583      5487798 :   if (!gimple_seq_empty_p (phi_nodes (bb)))
    4584              :     return false;
    4585      4355885 :   last = last_nondebug_stmt (bb);
    4586     15459748 :   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    4587              :     {
    4588     11103863 :       gimple *stmt = gsi_stmt (gsi);
    4589     11103863 :       tree lhs;
    4590     11103863 :       imm_use_iterator imm_iter;
    4591     11103863 :       use_operand_p use_p;
    4592              : 
    4593     11103863 :       if (is_gimple_debug (stmt))
    4594      4848105 :         continue;
    4595      6255758 :       if (gimple_has_side_effects (stmt))
    4596      4355885 :         return false;
    4597      5332561 :       if (stmt == last)
    4598              :         return true;
    4599      4238878 :       if (!is_gimple_assign (stmt))
    4600              :         return false;
    4601      3550366 :       lhs = gimple_assign_lhs (stmt);
    4602      3550366 :       if (TREE_CODE (lhs) != SSA_NAME)
    4603              :         return false;
    4604      3304612 :       if (gimple_assign_rhs_could_trap_p (stmt))
    4605              :         return false;
    4606      4685217 :       FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
    4607              :         {
    4608      2785344 :           gimple *use_stmt = USE_STMT (use_p);
    4609      2785344 :           if (is_gimple_debug (use_stmt))
    4610       217906 :             continue;
    4611      2567438 :           if (gimple_bb (use_stmt) != bb)
    4612       541138 :             return false;
    4613      2441011 :         }
    4614              :     }
    4615              :   return false;
    4616              : }
    4617              : 
    4618              : /* If VAR is set by CODE (BIT_{AND,IOR}_EXPR) which is reassociable,
    4619              :    return true and fill in *OPS recursively.  */
    4620              : 
    4621              : static bool
    4622        94822 : get_ops (tree var, enum tree_code code, vec<operand_entry *> *ops,
    4623              :          class loop *loop)
    4624              : {
    4625        94822 :   gimple *stmt = SSA_NAME_DEF_STMT (var);
    4626        94822 :   tree rhs[2];
    4627        94822 :   int i;
    4628              : 
    4629        94822 :   if (!is_reassociable_op (stmt, code, loop))
    4630              :     return false;
    4631              : 
    4632        22462 :   rhs[0] = gimple_assign_rhs1 (stmt);
    4633        22462 :   rhs[1] = gimple_assign_rhs2 (stmt);
    4634        22462 :   gimple_set_visited (stmt, true);
    4635        67386 :   for (i = 0; i < 2; i++)
    4636        44924 :     if (TREE_CODE (rhs[i]) == SSA_NAME
    4637        44924 :         && !get_ops (rhs[i], code, ops, loop)
    4638        83280 :         && has_single_use (rhs[i]))
    4639              :       {
    4640        37587 :         operand_entry *oe = operand_entry_pool.allocate ();
    4641              : 
    4642        37587 :         oe->op = rhs[i];
    4643        37587 :         oe->rank = code;
    4644        37587 :         oe->id = 0;
    4645        37587 :         oe->count = 1;
    4646        37587 :         oe->stmt_to_insert = NULL;
    4647        37587 :         ops->safe_push (oe);
    4648              :       }
    4649              :   return true;
    4650              : }
    4651              : 
    4652              : /* Find the ops that were added by get_ops starting from VAR, see if
    4653              :    they were changed during update_range_test and if yes, create new
    4654              :    stmts.  */
    4655              : 
    4656              : static tree
    4657        10797 : update_ops (tree var, enum tree_code code, const vec<operand_entry *> &ops,
    4658              :             unsigned int *pidx, class loop *loop)
    4659              : {
    4660        10797 :   gimple *stmt = SSA_NAME_DEF_STMT (var);
    4661        10797 :   tree rhs[4];
    4662        10797 :   int i;
    4663              : 
    4664        10797 :   if (!is_reassociable_op (stmt, code, loop))
    4665              :     return NULL;
    4666              : 
    4667         3609 :   rhs[0] = gimple_assign_rhs1 (stmt);
    4668         3609 :   rhs[1] = gimple_assign_rhs2 (stmt);
    4669         3609 :   rhs[2] = rhs[0];
    4670         3609 :   rhs[3] = rhs[1];
    4671        10827 :   for (i = 0; i < 2; i++)
    4672         7218 :     if (TREE_CODE (rhs[i]) == SSA_NAME)
    4673              :       {
    4674         7218 :         rhs[2 + i] = update_ops (rhs[i], code, ops, pidx, loop);
    4675         7218 :         if (rhs[2 + i] == NULL_TREE)
    4676              :           {
    4677         6803 :             if (has_single_use (rhs[i]))
    4678         6785 :               rhs[2 + i] = ops[(*pidx)++]->op;
    4679              :             else
    4680           18 :               rhs[2 + i] = rhs[i];
    4681              :           }
    4682              :       }
    4683         3609 :   if ((rhs[2] != rhs[0] || rhs[3] != rhs[1])
    4684         3360 :       && (rhs[2] != rhs[1] || rhs[3] != rhs[0]))
    4685              :     {
    4686         3360 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    4687         3360 :       var = make_ssa_name (TREE_TYPE (var));
    4688         3360 :       gassign *g = gimple_build_assign (var, gimple_assign_rhs_code (stmt),
    4689              :                                         rhs[2], rhs[3]);
    4690         3360 :       gimple_set_uid (g, gimple_uid (stmt));
    4691         3360 :       gimple_set_visited (g, true);
    4692         3360 :       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    4693         3360 :       gimple_stmt_iterator gsi2 = gsi_for_stmt (g);
    4694         3360 :       if (fold_stmt_inplace (&gsi2))
    4695         2190 :         update_stmt (g);
    4696              :     }
    4697              :   return var;
    4698              : }
    4699              : 
    4700              : /* Structure to track the initial value passed to get_ops and
    4701              :    the range in the ops vector for each basic block.  */
    4702              : 
    4703              : struct inter_bb_range_test_entry
    4704              : {
    4705              :   tree op;
    4706              :   unsigned int first_idx, last_idx;
    4707              : };
    4708              : 
    4709              : /* Inter-bb range test optimization.
    4710              : 
    4711              :    Returns TRUE if a gimple conditional is optimized to a true/false,
    4712              :    otherwise return FALSE.
    4713              : 
    4714              :    This indicates to the caller that it should run a CFG cleanup pass
    4715              :    once reassociation is completed.  */
    4716              : 
    4717              : static bool
    4718     19325763 : maybe_optimize_range_tests (gimple *stmt)
    4719              : {
    4720     19325763 :   basic_block first_bb = gimple_bb (stmt);
    4721     19325763 :   basic_block last_bb = first_bb;
    4722     19325763 :   basic_block other_bb = NULL;
    4723     19325763 :   basic_block bb;
    4724     19325763 :   edge_iterator ei;
    4725     19325763 :   edge e;
    4726     19325763 :   auto_vec<operand_entry *> ops;
    4727     19325763 :   auto_vec<inter_bb_range_test_entry> bbinfo;
    4728     19325763 :   bool any_changes = false;
    4729     19325763 :   bool cfg_cleanup_needed = false;
    4730              : 
    4731              :   /* Consider only basic blocks that end with GIMPLE_COND or
    4732              :      a cast statement satisfying final_range_test_p.  All
    4733              :      but the last bb in the first_bb .. last_bb range
    4734              :      should end with GIMPLE_COND.  */
    4735     19325763 :   if (gimple_code (stmt) == GIMPLE_COND)
    4736              :     {
    4737      8892248 :       if (EDGE_COUNT (first_bb->succs) != 2)
    4738              :         return cfg_cleanup_needed;
    4739              :     }
    4740     10433515 :   else if (final_range_test_p (stmt))
    4741        73826 :     other_bb = single_succ (first_bb);
    4742              :   else
    4743              :     return cfg_cleanup_needed;
    4744              : 
    4745      8966074 :   if (stmt_could_throw_p (cfun, stmt))
    4746              :     return cfg_cleanup_needed;
    4747              : 
    4748              :   /* As relative ordering of post-dominator sons isn't fixed,
    4749              :      maybe_optimize_range_tests can be called first on any
    4750              :      bb in the range we want to optimize.  So, start searching
    4751              :      backwards, if first_bb can be set to a predecessor.  */
    4752      8968458 :   while (single_pred_p (first_bb))
    4753              :     {
    4754      5912623 :       basic_block pred_bb = single_pred (first_bb);
    4755      5912623 :       if (!suitable_cond_bb (pred_bb, first_bb, &other_bb, NULL, true))
    4756              :         break;
    4757       633842 :       if (!no_side_effect_bb (first_bb))
    4758              :         break;
    4759              :       first_bb = pred_bb;
    4760              :     }
    4761              :   /* If first_bb is last_bb, other_bb hasn't been computed yet.
    4762              :      Before starting forward search in last_bb successors, find
    4763              :      out the other_bb.  */
    4764      8965906 :   if (first_bb == last_bb)
    4765              :     {
    4766      8963559 :       other_bb = NULL;
    4767              :       /* As non-GIMPLE_COND last stmt always terminates the range,
    4768              :          if forward search didn't discover anything, just give up.  */
    4769      8963559 :       if (gimple_code (stmt) != GIMPLE_COND)
    4770              :         return cfg_cleanup_needed;
    4771              :       /* Look at both successors.  Either it ends with a GIMPLE_COND
    4772              :          and satisfies suitable_cond_bb, or ends with a cast and
    4773              :          other_bb is that cast's successor.  */
    4774     24666312 :       FOR_EACH_EDGE (e, ei, first_bb->succs)
    4775     17061582 :         if (!(e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE))
    4776     17061582 :             || e->dest == first_bb)
    4777              :           return cfg_cleanup_needed;
    4778     25886690 :         else if (single_pred_p (e->dest))
    4779              :           {
    4780     10110367 :             stmt = last_nondebug_stmt (e->dest);
    4781     10110367 :             if (stmt
    4782      9927666 :                 && gimple_code (stmt) == GIMPLE_COND
    4783     14639984 :                 && EDGE_COUNT (e->dest->succs) == 2)
    4784              :               {
    4785      4529617 :                 if (suitable_cond_bb (first_bb, e->dest, &other_bb,
    4786              :                                       NULL, true))
    4787              :                   break;
    4788              :                 else
    4789      3779823 :                   other_bb = NULL;
    4790              :               }
    4791      5580750 :             else if (stmt
    4792      5398049 :                      && final_range_test_p (stmt)
    4793      5649810 :                      && find_edge (first_bb, single_succ (e->dest)))
    4794              :               {
    4795        38822 :                 other_bb = single_succ (e->dest);
    4796        38822 :                 if (other_bb == first_bb)
    4797            0 :                   other_bb = NULL;
    4798              :               }
    4799              :           }
    4800      8354524 :       if (other_bb == NULL)
    4801              :         return cfg_cleanup_needed;
    4802              :     }
    4803              :   /* Now do the forward search, moving last_bb to successor bbs
    4804              :      that aren't other_bb.  */
    4805      1003762 :   while (EDGE_COUNT (last_bb->succs) == 2)
    4806              :     {
    4807      1617297 :       FOR_EACH_EDGE (e, ei, last_bb->succs)
    4808      1617297 :         if (e->dest != other_bb)
    4809              :           break;
    4810       998296 :       if (e == NULL)
    4811              :         break;
    4812       998296 :       if (!single_pred_p (e->dest))
    4813              :         break;
    4814       959344 :       if (!suitable_cond_bb (e->dest, last_bb, &other_bb, NULL, false))
    4815              :         break;
    4816       846980 :       if (!no_side_effect_bb (e->dest))
    4817              :         break;
    4818       212799 :       last_bb = e->dest;
    4819              :     }
    4820       790963 :   if (first_bb == last_bb)
    4821              :     return cfg_cleanup_needed;
    4822              :   /* Here basic blocks first_bb through last_bb's predecessor
    4823              :      end with GIMPLE_COND, all of them have one of the edges to
    4824              :      other_bb and another to another block in the range,
    4825              :      all blocks except first_bb don't have side-effects and
    4826              :      last_bb ends with either GIMPLE_COND, or cast satisfying
    4827              :      final_range_test_p.  */
    4828       215351 :   for (bb = last_bb; ; bb = single_pred (bb))
    4829              :     {
    4830       370331 :       enum tree_code code;
    4831       370331 :       tree lhs, rhs;
    4832       370331 :       inter_bb_range_test_entry bb_ent;
    4833              : 
    4834       370331 :       bb_ent.op = NULL_TREE;
    4835       370331 :       bb_ent.first_idx = ops.length ();
    4836       370331 :       bb_ent.last_idx = bb_ent.first_idx;
    4837       370331 :       e = find_edge (bb, other_bb);
    4838       370331 :       stmt = last_nondebug_stmt (bb);
    4839       370331 :       gimple_set_visited (stmt, true);
    4840       370331 :       if (gimple_code (stmt) != GIMPLE_COND)
    4841              :         {
    4842         5466 :           use_operand_p use_p;
    4843         5466 :           gimple *phi;
    4844         5466 :           edge e2;
    4845         5466 :           unsigned int d;
    4846              : 
    4847         5466 :           lhs = gimple_assign_lhs (stmt);
    4848         5466 :           rhs = gimple_assign_rhs1 (stmt);
    4849         5466 :           gcc_assert (bb == last_bb);
    4850              : 
    4851              :           /* stmt is
    4852              :              _123 = (int) _234;
    4853              :              OR
    4854              :              _234 = a_2(D) == 2;
    4855              : 
    4856              :              followed by:
    4857              :              <bb M>:
    4858              :              # _345 = PHI <_123(N), 1(...), 1(...)>
    4859              : 
    4860              :              or 0 instead of 1.  If it is 0, the _234
    4861              :              range test is anded together with all the
    4862              :              other range tests, if it is 1, it is ored with
    4863              :              them.  */
    4864         5466 :           single_imm_use (lhs, &use_p, &phi);
    4865         5466 :           gcc_assert (gimple_code (phi) == GIMPLE_PHI);
    4866         5466 :           e2 = find_edge (first_bb, other_bb);
    4867         5466 :           d = e2->dest_idx;
    4868         5466 :           gcc_assert (gimple_phi_arg_def (phi, e->dest_idx) == lhs);
    4869         5466 :           if (integer_zerop (gimple_phi_arg_def (phi, d)))
    4870              :             code = BIT_AND_EXPR;
    4871              :           else
    4872              :             {
    4873         2891 :               gcc_checking_assert (integer_onep (gimple_phi_arg_def (phi, d)));
    4874              :               code = BIT_IOR_EXPR;
    4875              :             }
    4876              : 
    4877              :           /* If _234 SSA_NAME_DEF_STMT is
    4878              :              _234 = _567 | _789;
    4879              :              (or &, corresponding to 1/0 in the phi arguments,
    4880              :              push into ops the individual range test arguments
    4881              :              of the bitwise or resp. and, recursively.  */
    4882         5466 :           if (TREE_CODE (rhs) == SSA_NAME
    4883         5466 :               && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
    4884              :                   != tcc_comparison)
    4885         2966 :               && !get_ops (rhs, code, &ops,
    4886              :                            loop_containing_stmt (stmt))
    4887         8257 :               && has_single_use (rhs))
    4888              :             {
    4889              :               /* Otherwise, push the _234 range test itself.  */
    4890         2771 :               operand_entry *oe = operand_entry_pool.allocate ();
    4891              : 
    4892         2771 :               oe->op = rhs;
    4893         2771 :               oe->rank = code;
    4894         2771 :               oe->id = 0;
    4895         2771 :               oe->count = 1;
    4896         2771 :               oe->stmt_to_insert = NULL;
    4897         2771 :               ops.safe_push (oe);
    4898         2771 :               bb_ent.last_idx++;
    4899         2771 :               bb_ent.op = rhs;
    4900              :             }
    4901         2695 :           else if (is_gimple_assign (stmt)
    4902         2695 :                    && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
    4903              :                        == tcc_comparison)
    4904         2500 :                    && !get_ops (lhs, code, &ops,
    4905              :                                 loop_containing_stmt (stmt))
    4906         5195 :                    && has_single_use (lhs))
    4907              :             {
    4908         2500 :               operand_entry *oe = operand_entry_pool.allocate ();
    4909         2500 :               oe->op = lhs;
    4910         2500 :               oe->rank = code;
    4911         2500 :               oe->id = 0;
    4912         2500 :               oe->count = 1;
    4913         2500 :               ops.safe_push (oe);
    4914         2500 :               bb_ent.last_idx++;
    4915         2500 :               bb_ent.op = lhs;
    4916              :             }
    4917              :           else
    4918              :             {
    4919          195 :               bb_ent.last_idx = ops.length ();
    4920          195 :               bb_ent.op = rhs;
    4921              :             }
    4922         5466 :           bbinfo.safe_push (bb_ent);
    4923        16593 :           for (unsigned int i = bb_ent.first_idx; i < bb_ent.last_idx; ++i)
    4924         5661 :             ops[i]->id = bb->index;
    4925         5466 :           continue;
    4926         5466 :         }
    4927       364865 :       else if (bb == last_bb)
    4928              :         {
    4929              :           /* For last_bb, handle also:
    4930              :              if (x_3(D) == 3)
    4931              :                goto <bb 6>; [34.00%]
    4932              :              else
    4933              :                goto <bb 7>; [66.00%]
    4934              : 
    4935              :              <bb 6> [local count: 79512730]:
    4936              : 
    4937              :              <bb 7> [local count: 1073741824]:
    4938              :              # prephitmp_7 = PHI <1(3), 1(4), 0(5), 1(2), 1(6)>
    4939              :              where bb 7 is OTHER_BB, but the PHI values from the
    4940              :              earlier bbs match the path through the empty bb
    4941              :              in between.  */
    4942       149514 :           bool test_swapped_p = false;
    4943       149514 :           bool ok = suitable_cond_bb (single_pred (last_bb), last_bb,
    4944              :                                       &other_bb, &test_swapped_p, true);
    4945       149514 :           gcc_assert (ok);
    4946       149514 :           if (test_swapped_p)
    4947          528 :             e = EDGE_SUCC (bb, e == EDGE_SUCC (bb, 0) ? 1 : 0);
    4948              :         }
    4949              :       /* Otherwise stmt is GIMPLE_COND.  */
    4950       364865 :       code = gimple_cond_code (stmt);
    4951       364865 :       lhs = gimple_cond_lhs (stmt);
    4952       364865 :       rhs = gimple_cond_rhs (stmt);
    4953       364865 :       if (TREE_CODE (lhs) == SSA_NAME
    4954       364801 :           && INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    4955       669553 :           && ((code != EQ_EXPR && code != NE_EXPR)
    4956       229217 :               || rhs != boolean_false_node
    4957              :                  /* Either push into ops the individual bitwise
    4958              :                     or resp. and operands, depending on which
    4959              :                     edge is other_bb.  */
    4960        44432 :               || !get_ops (lhs, (((e->flags & EDGE_TRUE_VALUE) == 0)
    4961        44432 :                                  ^ (code == EQ_EXPR))
    4962              :                                 ? BIT_AND_EXPR : BIT_IOR_EXPR, &ops,
    4963              :                            loop_containing_stmt (stmt))))
    4964              :         {
    4965              :           /* Or push the GIMPLE_COND stmt itself.  */
    4966       288969 :           operand_entry *oe = operand_entry_pool.allocate ();
    4967              : 
    4968       288969 :           oe->op = NULL;
    4969       577938 :           oe->rank = (e->flags & EDGE_TRUE_VALUE)
    4970       288969 :                      ? BIT_IOR_EXPR : BIT_AND_EXPR;
    4971              :           /* oe->op = NULL signs that there is no SSA_NAME
    4972              :              for the range test, and oe->id instead is the
    4973              :              basic block number, at which's end the GIMPLE_COND
    4974              :              is.  */
    4975       288969 :           oe->id = bb->index;
    4976       288969 :           oe->count = 1;
    4977       288969 :           oe->stmt_to_insert = NULL;
    4978       288969 :           ops.safe_push (oe);
    4979       288969 :           bb_ent.op = NULL;
    4980       288969 :           bb_ent.last_idx++;
    4981              :         }
    4982        75896 :       else if (ops.length () > bb_ent.first_idx)
    4983              :         {
    4984        15647 :           bb_ent.op = lhs;
    4985        15647 :           bb_ent.last_idx = ops.length ();
    4986              :         }
    4987       364865 :       bbinfo.safe_push (bb_ent);
    4988      1055896 :       for (unsigned int i = bb_ent.first_idx; i < bb_ent.last_idx; ++i)
    4989       326166 :         ops[i]->id = bb->index;
    4990       364865 :       if (bb == first_bb)
    4991              :         break;
    4992       215351 :     }
    4993     19446749 :   if (ops.length () > 1)
    4994       120986 :     any_changes = optimize_range_tests (ERROR_MARK, &ops, first_bb);
    4995       120986 :   if (any_changes)
    4996              :     {
    4997              :       unsigned int idx, max_idx = 0;
    4998              :       /* update_ops relies on has_single_use predicates returning the
    4999              :          same values as it did during get_ops earlier.  Additionally it
    5000              :          never removes statements, only adds new ones and it should walk
    5001              :          from the single imm use and check the predicate already before
    5002              :          making those changes.
    5003              :          On the other side, the handling of GIMPLE_COND directly can turn
    5004              :          previously multiply used SSA_NAMEs into single use SSA_NAMEs, so
    5005              :          it needs to be done in a separate loop afterwards.  */
    5006        21714 :       for (bb = last_bb, idx = 0; ; bb = single_pred (bb), idx++)
    5007              :         {
    5008        35847 :           if (bbinfo[idx].first_idx < bbinfo[idx].last_idx
    5009        35847 :               && bbinfo[idx].op != NULL_TREE)
    5010              :             {
    5011         3579 :               tree new_op;
    5012              : 
    5013         3579 :               max_idx = idx;
    5014         3579 :               stmt = last_nondebug_stmt (bb);
    5015         7158 :               new_op = update_ops (bbinfo[idx].op,
    5016              :                                    (enum tree_code)
    5017         3579 :                                    ops[bbinfo[idx].first_idx]->rank,
    5018         3579 :                                    ops, &bbinfo[idx].first_idx,
    5019              :                                    loop_containing_stmt (stmt));
    5020         3579 :               if (new_op == NULL_TREE)
    5021              :                 {
    5022          385 :                   gcc_assert (bb == last_bb);
    5023          385 :                   new_op = ops[bbinfo[idx].first_idx++]->op;
    5024              :                 }
    5025         3579 :               if (bbinfo[idx].op != new_op)
    5026              :                 {
    5027         3324 :                   imm_use_iterator iter;
    5028         3324 :                   use_operand_p use_p;
    5029         3324 :                   gimple *use_stmt, *cast_or_tcc_cmp_stmt = NULL;
    5030              : 
    5031         6658 :                   FOR_EACH_IMM_USE_STMT (use_stmt, iter, bbinfo[idx].op)
    5032         3334 :                     if (is_gimple_debug (use_stmt))
    5033           10 :                       continue;
    5034         3324 :                     else if (gimple_code (use_stmt) == GIMPLE_COND
    5035         3324 :                              || gimple_code (use_stmt) == GIMPLE_PHI)
    5036         6080 :                       FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
    5037         3040 :                         SET_USE (use_p, new_op);
    5038          284 :                     else if ((is_gimple_assign (use_stmt)
    5039          284 :                               && (TREE_CODE_CLASS
    5040              :                                   (gimple_assign_rhs_code (use_stmt))
    5041              :                                   == tcc_comparison)))
    5042              :                       cast_or_tcc_cmp_stmt = use_stmt;
    5043          284 :                     else if (gimple_assign_cast_p (use_stmt))
    5044              :                       cast_or_tcc_cmp_stmt = use_stmt;
    5045              :                     else
    5046            0 :                       gcc_unreachable ();
    5047              : 
    5048         3324 :                   if (cast_or_tcc_cmp_stmt)
    5049              :                     {
    5050          284 :                       gcc_assert (bb == last_bb);
    5051          284 :                       tree lhs = gimple_assign_lhs (cast_or_tcc_cmp_stmt);
    5052          284 :                       tree new_lhs = make_ssa_name (TREE_TYPE (lhs));
    5053          284 :                       enum tree_code rhs_code
    5054          284 :                         = gimple_assign_cast_p (cast_or_tcc_cmp_stmt)
    5055          284 :                         ? gimple_assign_rhs_code (cast_or_tcc_cmp_stmt)
    5056              :                         : CONVERT_EXPR;
    5057          284 :                       gassign *g;
    5058          284 :                       if (is_gimple_min_invariant (new_op))
    5059              :                         {
    5060           73 :                           new_op = fold_convert (TREE_TYPE (lhs), new_op);
    5061           73 :                           g = gimple_build_assign (new_lhs, new_op);
    5062              :                         }
    5063              :                       else
    5064          211 :                         g = gimple_build_assign (new_lhs, rhs_code, new_op);
    5065          284 :                       gimple_stmt_iterator gsi
    5066          284 :                         = gsi_for_stmt (cast_or_tcc_cmp_stmt);
    5067          284 :                       gimple_set_uid (g, gimple_uid (cast_or_tcc_cmp_stmt));
    5068          284 :                       gimple_set_visited (g, true);
    5069          284 :                       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    5070          568 :                       FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    5071          284 :                         if (is_gimple_debug (use_stmt))
    5072            0 :                           continue;
    5073          284 :                         else if (gimple_code (use_stmt) == GIMPLE_COND
    5074          284 :                                  || gimple_code (use_stmt) == GIMPLE_PHI)
    5075          568 :                           FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
    5076          284 :                             SET_USE (use_p, new_lhs);
    5077              :                         else
    5078          284 :                           gcc_unreachable ();
    5079              :                     }
    5080              :                 }
    5081              :             }
    5082        35847 :           if (bb == first_bb)
    5083              :             break;
    5084        21714 :         }
    5085        21714 :       for (bb = last_bb, idx = 0; ; bb = single_pred (bb), idx++)
    5086              :         {
    5087        35847 :           if (bbinfo[idx].first_idx < bbinfo[idx].last_idx
    5088        31810 :               && bbinfo[idx].op == NULL_TREE
    5089        67657 :               && ops[bbinfo[idx].first_idx]->op != NULL_TREE)
    5090              :             {
    5091        55346 :               gcond *cond_stmt = as_a <gcond *> (*gsi_last_bb (bb));
    5092              : 
    5093        27673 :               if (idx > max_idx)
    5094              :                 max_idx = idx;
    5095              : 
    5096              :               /* If we collapse the conditional to a true/false
    5097              :                  condition, then bubble that knowledge up to our caller.  */
    5098        27673 :               if (integer_zerop (ops[bbinfo[idx].first_idx]->op))
    5099              :                 {
    5100        11602 :                   gimple_cond_make_false (cond_stmt);
    5101        11602 :                   cfg_cleanup_needed = true;
    5102              :                 }
    5103        16071 :               else if (integer_onep (ops[bbinfo[idx].first_idx]->op))
    5104              :                 {
    5105         3713 :                   gimple_cond_make_true (cond_stmt);
    5106         3713 :                   cfg_cleanup_needed = true;
    5107              :                 }
    5108              :               else
    5109              :                 {
    5110        12358 :                   gimple_cond_set_code (cond_stmt, NE_EXPR);
    5111        12358 :                   gimple_cond_set_lhs (cond_stmt,
    5112        12358 :                                        ops[bbinfo[idx].first_idx]->op);
    5113        12358 :                   gimple_cond_set_rhs (cond_stmt, boolean_false_node);
    5114              :                 }
    5115        27673 :               update_stmt (cond_stmt);
    5116              :             }
    5117        35847 :           if (bb == first_bb)
    5118              :             break;
    5119        21714 :         }
    5120              : 
    5121              :       /* The above changes could result in basic blocks after the first
    5122              :          modified one, up to and including last_bb, to be executed even if
    5123              :          they would not be in the original program.  If the value ranges of
    5124              :          assignment lhs' in those bbs were dependent on the conditions
    5125              :          guarding those basic blocks which now can change, the VRs might
    5126              :          be incorrect.  As no_side_effect_bb should ensure those SSA_NAMEs
    5127              :          are only used within the same bb, it should be not a big deal if
    5128              :          we just reset all the VRs in those bbs.  See PR68671.  */
    5129        34591 :       for (bb = last_bb, idx = 0; idx < max_idx; bb = single_pred (bb), idx++)
    5130        20458 :         reset_flow_sensitive_info_in_bb (bb);
    5131              :     }
    5132              :   return cfg_cleanup_needed;
    5133     19325763 : }
    5134              : 
    5135              : /* Remove def stmt of VAR if VAR has zero uses and recurse
    5136              :    on rhs1 operand if so.  */
    5137              : 
    5138              : static void
    5139        71924 : remove_visited_stmt_chain (tree var)
    5140              : {
    5141        97006 :   gimple *stmt;
    5142        97006 :   gimple_stmt_iterator gsi;
    5143              : 
    5144       122088 :   while (1)
    5145              :     {
    5146        97006 :       if (TREE_CODE (var) != SSA_NAME || !has_zero_uses (var))
    5147              :         return;
    5148        36640 :       stmt = SSA_NAME_DEF_STMT (var);
    5149        36640 :       if (is_gimple_assign (stmt) && gimple_visited_p (stmt))
    5150              :         {
    5151        25082 :           var = gimple_assign_rhs1 (stmt);
    5152        25082 :           gsi = gsi_for_stmt (stmt);
    5153        25082 :           reassoc_remove_stmt (&gsi);
    5154        25082 :           release_defs (stmt);
    5155              :         }
    5156              :       else
    5157              :         return;
    5158              :     }
    5159              : }
    5160              : 
    5161              : /* This function checks three consecutive operands in
    5162              :    passed operands vector OPS starting from OPINDEX and
    5163              :    swaps two operands if it is profitable for binary operation
    5164              :    consuming OPINDEX + 1 and OPINDEX + 2 operands.
    5165              : 
    5166              :    We pair ops with the same rank if possible.  */
    5167              : 
    5168              : static void
    5169       130313 : swap_ops_for_binary_stmt (const vec<operand_entry *> &ops,
    5170              :                           unsigned int opindex)
    5171              : {
    5172       130313 :   operand_entry *oe1, *oe2, *oe3;
    5173              : 
    5174       130313 :   oe1 = ops[opindex];
    5175       130313 :   oe2 = ops[opindex + 1];
    5176       130313 :   oe3 = ops[opindex + 2];
    5177              : 
    5178       130313 :   if (oe1->rank == oe2->rank && oe2->rank != oe3->rank)
    5179        16858 :     std::swap (*oe1, *oe3);
    5180       113455 :   else if (oe1->rank == oe3->rank && oe2->rank != oe3->rank)
    5181          363 :     std::swap (*oe1, *oe2);
    5182       130313 : }
    5183              : 
    5184              : /* Return true if OP is set by a comparison in BB, which a conditional
    5185              :    compare can compute into the condition flags.  */
    5186              : 
    5187              : static bool
    5188            2 : ccmp_comparison_p (tree op, basic_block bb)
    5189              : {
    5190            2 :   if (TREE_CODE (op) != SSA_NAME)
    5191              :     return false;
    5192              : 
    5193            2 :   gimple *def = SSA_NAME_DEF_STMT (op);
    5194            2 :   if (!is_gimple_assign (def)
    5195            2 :       || gimple_bb (def) != bb
    5196            4 :       || TREE_CODE_CLASS (gimple_assign_rhs_code (def)) != tcc_comparison)
    5197              :     return false;
    5198              : 
    5199              :   /* A conditional compare sets the flags from one scalar comparison.  */
    5200            2 :   machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_assign_rhs1 (def)));
    5201            2 :   return SCALAR_INT_MODE_P (mode) || SCALAR_FLOAT_MODE_P (mode);
    5202              : }
    5203              : 
    5204              : /* Return how strongly OP should start a conditional compare sequence.
    5205              :    FOLDABLE is true if OP can be folded into the flags.  */
    5206              : 
    5207              : static ccmp_first_kind
    5208            2 : ccmp_first_preference (tree op, bool foldable)
    5209              : {
    5210            2 :   if (!foldable)
    5211              :     return CCMP_FIRST_NONE;
    5212              : 
    5213            2 :   gimple *def = SSA_NAME_DEF_STMT (op);
    5214            2 :   gcc_checking_assert (is_gimple_assign (def));
    5215              : 
    5216            2 :   tree op0 = gimple_assign_rhs1 (def);
    5217            2 :   tree op1 = gimple_assign_rhs2 (def);
    5218            2 :   ccmp_first_kind kind = targetm.ccmp_first_preference
    5219            2 :     (TYPE_MODE (TREE_TYPE (op0)), gimple_assign_rhs_code (def), op0, op1);
    5220            2 :   if (kind == CCMP_FIRST_UNSUPPORTED || !gimple_could_trap_p (def))
    5221            2 :     return kind;
    5222              :   return CCMP_FIRST_REQUIRED;
    5223              : }
    5224              : 
    5225              : /* Return true if all operands in OPS form one conditional compare sequence
    5226              :    for CODE in BB.  Put the operand that must start the sequence last in OPS,
    5227              :    where rewrite_expr_tree builds the innermost pair.  Every operand must be a
    5228              :    comparison, and at least one must be foldable into the flags.  */
    5229              : 
    5230              : static bool
    5231       182493 : prepare_ccmp_chain (vec<operand_entry *> &ops, tree_code code, basic_block bb)
    5232              : {
    5233       182493 :   if (!targetm.have_ccmp ()
    5234       182493 :       || (code != BIT_AND_EXPR && code != BIT_IOR_EXPR))
    5235              :     return false;
    5236              : 
    5237              :   bool seen_foldable = false;
    5238              :   int head = -1;
    5239              :   ccmp_first_kind head_kind = CCMP_FIRST_NONE;
    5240              : 
    5241            2 :   for (unsigned int i = 0; i < ops.length (); ++i)
    5242              :     {
    5243            2 :       const operand_entry *oe = ops[i];
    5244            2 :       if (!ccmp_comparison_p (oe->op, bb))
    5245              :         return false;
    5246              : 
    5247            2 :       bool foldable = has_single_use (oe->op);
    5248              : 
    5249            2 :       ccmp_first_kind kind
    5250            2 :         = ccmp_first_preference (oe->op, foldable);
    5251            2 :       if (kind == CCMP_FIRST_UNSUPPORTED)
    5252              :         return false;
    5253            0 :       if (kind == CCMP_FIRST_REQUIRED && head_kind == CCMP_FIRST_REQUIRED)
    5254              :         return false;
    5255            0 :       if (kind > head_kind)
    5256              :         {
    5257            0 :           head = i;
    5258            0 :           head_kind = kind;
    5259              :         }
    5260              : 
    5261              :       /* Comparison leaves can have uses outside this chain.  */
    5262            0 :       seen_foldable |= foldable;
    5263              :     }
    5264              : 
    5265            0 :   if (head >= 0)
    5266            0 :     std::swap (*ops[head], *ops.last ());
    5267              :   return seen_foldable;
    5268              : }
    5269              : 
    5270              : /* If definition of RHS1 or RHS2 dominates STMT, return the later of those
    5271              :    two definitions, otherwise return STMT.  Sets INSERT_BEFORE to indicate
    5272              :    whether RHS1 op RHS2 can be inserted before or needs to be inserted
    5273              :    after the returned stmt.  */
    5274              : 
    5275              : static inline gimple *
    5276       377322 : find_insert_point (gimple *stmt, tree rhs1, tree rhs2, bool &insert_before)
    5277              : {
    5278       377322 :   insert_before = true;
    5279       377322 :   if (TREE_CODE (rhs1) == SSA_NAME
    5280       377322 :       && reassoc_stmt_dominates_stmt_p (stmt, SSA_NAME_DEF_STMT (rhs1)))
    5281              :     {
    5282        11084 :       stmt = SSA_NAME_DEF_STMT (rhs1);
    5283        11084 :       insert_before = false;
    5284              :     }
    5285       377322 :   if (TREE_CODE (rhs2) == SSA_NAME
    5286       377322 :       && reassoc_stmt_dominates_stmt_p (stmt, SSA_NAME_DEF_STMT (rhs2)))
    5287              :     {
    5288        20094 :       stmt = SSA_NAME_DEF_STMT (rhs2);
    5289        20094 :       insert_before = false;
    5290              :     }
    5291       377322 :   return stmt;
    5292              : }
    5293              : 
    5294              : /* If the stmt that defines operand has to be inserted, insert it
    5295              :    before the use.  */
    5296              : static void
    5297           85 : insert_stmt_before_use (gimple *stmt, gimple *stmt_to_insert)
    5298              : {
    5299           85 :   gcc_assert (is_gimple_assign (stmt_to_insert));
    5300           85 :   tree rhs1 = gimple_assign_rhs1 (stmt_to_insert);
    5301           85 :   tree rhs2 = gimple_assign_rhs2 (stmt_to_insert);
    5302           85 :   bool insert_before;
    5303           85 :   gimple *insert_point = find_insert_point (stmt, rhs1, rhs2, insert_before);
    5304           85 :   gimple_stmt_iterator gsi = gsi_for_stmt (insert_point);
    5305           85 :   gimple_set_uid (stmt_to_insert, gimple_uid (insert_point));
    5306              : 
    5307              :   /* If the insert point is not stmt, then insert_point would be
    5308              :      the point where operand rhs1 or rhs2 is defined. In this case,
    5309              :      stmt_to_insert has to be inserted afterwards. This would
    5310              :      only happen when the stmt insertion point is flexible. */
    5311           85 :   if (insert_before)
    5312           85 :     gsi_insert_before (&gsi, stmt_to_insert, GSI_NEW_STMT);
    5313              :   else
    5314            0 :     insert_stmt_after (stmt_to_insert, insert_point);
    5315           85 : }
    5316              : 
    5317              : 
    5318              : /* Recursively rewrite our linearized statements so that the operators
    5319              :    match those in OPS[OPINDEX], putting the computation in rank
    5320              :    order.  Return new lhs.
    5321              :    CHANGED is true if we shouldn't reuse the lhs SSA_NAME both in
    5322              :    the current stmt and during recursive invocations.
    5323              :    NEXT_CHANGED is true if we shouldn't reuse the lhs SSA_NAME in
    5324              :    recursive invocations.  */
    5325              : 
    5326              : static tree
    5327      4973485 : rewrite_expr_tree (gimple *stmt, enum tree_code rhs_code, unsigned int opindex,
    5328              :                    const vec<operand_entry *> &ops, bool changed,
    5329              :                    bool next_changed)
    5330              : {
    5331      4973485 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    5332      4973485 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    5333      4973485 :   tree lhs = gimple_assign_lhs (stmt);
    5334      4973485 :   operand_entry *oe;
    5335              : 
    5336              :   /* The final recursion case for this function is that you have
    5337              :      exactly two operations left.
    5338              :      If we had exactly one op in the entire list to start with, we
    5339              :      would have never called this function, and the tail recursion
    5340              :      rewrites them one at a time.  */
    5341      9946970 :   if (opindex + 2 == ops.length ())
    5342              :     {
    5343      4743498 :       operand_entry *oe1, *oe2;
    5344              : 
    5345      4743498 :       oe1 = ops[opindex];
    5346      4743498 :       oe2 = ops[opindex + 1];
    5347      4743498 :       if (commutative_tree_code (rhs_code)
    5348      4743498 :           && tree_swap_operands_p (oe1->op, oe2->op))
    5349              :         std::swap (oe1, oe2);
    5350              : 
    5351      4743498 :       if (rhs1 != oe1->op || rhs2 != oe2->op)
    5352              :         {
    5353       247408 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    5354       247408 :           unsigned int uid = gimple_uid (stmt);
    5355              : 
    5356       247408 :           if (dump_file && (dump_flags & TDF_DETAILS))
    5357              :             {
    5358           31 :               fprintf (dump_file, "Transforming ");
    5359           31 :               print_gimple_stmt (dump_file, stmt, 0);
    5360              :             }
    5361              : 
    5362              :           /* If the stmt that defines operand has to be inserted, insert it
    5363              :              before the use.  */
    5364       247408 :           if (oe1->stmt_to_insert)
    5365           34 :             insert_stmt_before_use (stmt, oe1->stmt_to_insert);
    5366       247408 :           if (oe2->stmt_to_insert)
    5367           51 :             insert_stmt_before_use (stmt, oe2->stmt_to_insert);
    5368              :           /* Even when changed is false, reassociation could have e.g. removed
    5369              :              some redundant operations, so unless we are just swapping the
    5370              :              arguments or unless there is no change at all (then we just
    5371              :              return lhs), force creation of a new SSA_NAME.  */
    5372       247408 :           if (changed || ((rhs1 != oe2->op || rhs2 != oe1->op) && opindex))
    5373              :             {
    5374       110396 :               bool insert_before;
    5375       110396 :               gimple *insert_point
    5376       110396 :                 = find_insert_point (stmt, oe1->op, oe2->op, insert_before);
    5377       110396 :               lhs = make_ssa_name (TREE_TYPE (lhs));
    5378       110396 :               stmt
    5379       110396 :                 = gimple_build_assign (lhs, rhs_code,
    5380              :                                        oe1->op, oe2->op);
    5381       110396 :               gimple_set_uid (stmt, uid);
    5382       110396 :               gimple_set_visited (stmt, true);
    5383       110396 :               if (insert_before)
    5384        87875 :                 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
    5385              :               else
    5386        22521 :                 insert_stmt_after (stmt, insert_point);
    5387       110396 :             }
    5388              :           else
    5389              :             {
    5390       137012 :               bool insert_before;
    5391       137012 :               gcc_checking_assert (find_insert_point (stmt, oe1->op, oe2->op,
    5392              :                                                       insert_before)
    5393              :                                    == stmt);
    5394       137012 :               gimple_assign_set_rhs1 (stmt, oe1->op);
    5395       137012 :               gimple_assign_set_rhs2 (stmt, oe2->op);
    5396       137012 :               update_stmt (stmt);
    5397              :             }
    5398              : 
    5399       247408 :           if (rhs1 != oe1->op && rhs1 != oe2->op)
    5400        55372 :             remove_visited_stmt_chain (rhs1);
    5401              : 
    5402       247408 :           if (dump_file && (dump_flags & TDF_DETAILS))
    5403              :             {
    5404           31 :               fprintf (dump_file, " into ");
    5405           31 :               print_gimple_stmt (dump_file, stmt, 0);
    5406              :             }
    5407              :         }
    5408              :       return lhs;
    5409              :     }
    5410              : 
    5411              :   /* If we hit here, we should have 3 or more ops left.  */
    5412       229987 :   gcc_assert (opindex + 2 < ops.length ());
    5413              : 
    5414              :   /* Rewrite the next operator.  */
    5415       229987 :   oe = ops[opindex];
    5416              : 
    5417              :   /* If the stmt that defines operand has to be inserted, insert it
    5418              :      before the use.  */
    5419       229987 :   if (oe->stmt_to_insert)
    5420            0 :     insert_stmt_before_use (stmt, oe->stmt_to_insert);
    5421              : 
    5422              :   /* Recurse on the LHS of the binary operator, which is guaranteed to
    5423              :      be the non-leaf side.  */
    5424       229987 :   tree new_rhs1
    5425       441015 :     = rewrite_expr_tree (SSA_NAME_DEF_STMT (rhs1), rhs_code, opindex + 1, ops,
    5426       211028 :                          changed || oe->op != rhs2 || next_changed,
    5427              :                          false);
    5428              : 
    5429       229987 :   if (oe->op != rhs2 || new_rhs1 != rhs1)
    5430              :     {
    5431       129829 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5432              :         {
    5433            6 :           fprintf (dump_file, "Transforming ");
    5434            6 :           print_gimple_stmt (dump_file, stmt, 0);
    5435              :         }
    5436              : 
    5437              :       /* If changed is false, this is either opindex == 0
    5438              :          or all outer rhs2's were equal to corresponding oe->op,
    5439              :          and powi_result is NULL.
    5440              :          That means lhs is equivalent before and after reassociation.
    5441              :          Otherwise ensure the old lhs SSA_NAME is not reused and
    5442              :          create a new stmt as well, so that any debug stmts will be
    5443              :          properly adjusted.  */
    5444       129829 :       if (changed)
    5445              :         {
    5446        18724 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    5447        18724 :           unsigned int uid = gimple_uid (stmt);
    5448        18724 :           bool insert_before;
    5449        18724 :           gimple *insert_point = find_insert_point (stmt, new_rhs1, oe->op,
    5450              :                                                     insert_before);
    5451              : 
    5452        18724 :           lhs = make_ssa_name (TREE_TYPE (lhs));
    5453        18724 :           stmt = gimple_build_assign (lhs, rhs_code,
    5454              :                                       new_rhs1, oe->op);
    5455        18724 :           gimple_set_uid (stmt, uid);
    5456        18724 :           gimple_set_visited (stmt, true);
    5457        18724 :           if (insert_before)
    5458        11145 :             gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
    5459              :           else
    5460         7579 :             insert_stmt_after (stmt, insert_point);
    5461              :         }
    5462              :       else
    5463              :         {
    5464       111105 :           bool insert_before;
    5465       111105 :           gcc_checking_assert (find_insert_point (stmt, new_rhs1, oe->op,
    5466              :                                                   insert_before)
    5467              :                                == stmt);
    5468       111105 :           gimple_assign_set_rhs1 (stmt, new_rhs1);
    5469       111105 :           gimple_assign_set_rhs2 (stmt, oe->op);
    5470       111105 :           update_stmt (stmt);
    5471              :         }
    5472              : 
    5473       129829 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5474              :         {
    5475            6 :           fprintf (dump_file, " into ");
    5476            6 :           print_gimple_stmt (dump_file, stmt, 0);
    5477              :         }
    5478              :     }
    5479              :   return lhs;
    5480              : }
    5481              : 
    5482              : /* Find out how many cycles we need to compute statements chain.
    5483              :    OPS_NUM holds number os statements in a chain.  CPU_WIDTH is a
    5484              :    maximum number of independent statements we may execute per cycle.  */
    5485              : 
    5486              : static int
    5487        18844 : get_required_cycles (int ops_num, int cpu_width)
    5488              : {
    5489        18844 :   int res;
    5490        18844 :   int elog;
    5491        18844 :   unsigned int rest;
    5492              : 
    5493              :   /* While we have more than 2 * cpu_width operands
    5494              :      we may reduce number of operands by cpu_width
    5495              :      per cycle.  */
    5496        18844 :   res = ops_num / (2 * cpu_width);
    5497              : 
    5498              :   /* Remained operands count may be reduced twice per cycle
    5499              :      until we have only one operand.  */
    5500        18844 :   rest = (unsigned)(ops_num - res * cpu_width);
    5501        18844 :   elog = exact_log2 (rest);
    5502         8899 :   if (elog >= 0)
    5503         8899 :     res += elog;
    5504              :   else
    5505        19890 :     res += floor_log2 (rest) + 1;
    5506              : 
    5507        18844 :   return res;
    5508              : }
    5509              : 
    5510              : /* Given that the target fully pipelines FMA instructions, return the latency
    5511              :    of MULT_EXPRs that can't be hidden by the FMAs.  WIDTH is the number of
    5512              :    pipes.  */
    5513              : 
    5514              : static inline int
    5515            0 : get_mult_latency_consider_fma (int ops_num, int mult_num, int width)
    5516              : {
    5517            0 :   gcc_checking_assert (mult_num && mult_num <= ops_num);
    5518              : 
    5519              :   /* For each partition, if mult_num == ops_num, there's latency(MULT)*2.
    5520              :      e.g:
    5521              : 
    5522              :         A * B + C * D
    5523              :         =>
    5524              :         _1 = A * B;
    5525              :         _2 = .FMA (C, D, _1);
    5526              : 
    5527              :       Otherwise there's latency(MULT)*1 in the first FMA.  */
    5528            0 :   return CEIL (ops_num, width) == CEIL (mult_num, width) ? 2 : 1;
    5529              : }
    5530              : 
    5531              : /* Returns an optimal number of registers to use for computation of
    5532              :    given statements.
    5533              : 
    5534              :    LHS is the result ssa name of OPS.  MULT_NUM is number of sub-expressions
    5535              :    that are MULT_EXPRs, when OPS are PLUS_EXPRs or MINUS_EXPRs.  */
    5536              : 
    5537              : static int
    5538        19149 : get_reassociation_width (vec<operand_entry *> *ops, int mult_num, tree lhs,
    5539              :                          enum tree_code opc, machine_mode mode)
    5540              : {
    5541        19149 :   int param_width = param_tree_reassoc_width;
    5542        19149 :   int width;
    5543        19149 :   int width_min;
    5544        19149 :   int cycles_best;
    5545        19149 :   int ops_num = ops->length ();
    5546              : 
    5547        19149 :   if (param_width > 0)
    5548              :     width = param_width;
    5549              :   else
    5550        19105 :     width = targetm.sched.reassociation_width (opc, mode);
    5551              : 
    5552        19149 :   if (width == 1)
    5553              :     return width;
    5554              : 
    5555              :   /* Get the minimal time required for sequence computation.  */
    5556         7188 :   cycles_best = get_required_cycles (ops_num, width);
    5557              : 
    5558              :   /* Check if we may use less width and still compute sequence for
    5559              :      the same time.  It will allow us to reduce registers usage.
    5560              :      get_required_cycles is monotonically increasing with lower width
    5561              :      so we can perform a binary search for the minimal width that still
    5562              :      results in the optimal cycle count.  */
    5563         7188 :   width_min = 1;
    5564              : 
    5565              :   /* If the target fully pipelines FMA instruction, the multiply part can start
    5566              :      already if its operands are ready.  Assuming symmetric pipes are used for
    5567              :      FMUL/FADD/FMA, then for a sequence of FMA like:
    5568              : 
    5569              :         _8 = .FMA (_2, _3, _1);
    5570              :         _9 = .FMA (_5, _4, _8);
    5571              :         _10 = .FMA (_7, _6, _9);
    5572              : 
    5573              :      , if width=1, the latency is latency(MULT) + latency(ADD)*3.
    5574              :      While with width=2:
    5575              : 
    5576              :         _8 = _4 * _5;
    5577              :         _9 = .FMA (_2, _3, _1);
    5578              :         _10 = .FMA (_6, _7, _8);
    5579              :         _11 = _9 + _10;
    5580              : 
    5581              :      , it is latency(MULT)*2 + latency(ADD)*2.  Assuming latency(MULT) >=
    5582              :      latency(ADD), the first variant is preferred.
    5583              : 
    5584              :      Find out if we can get a smaller width considering FMA.
    5585              :      Assume FMUL and FMA use the same units that can also do FADD.
    5586              :      For other scenarios, such as when FMUL and FADD are using separated units,
    5587              :      the following code may not apply.  */
    5588              : 
    5589         7188 :   int width_mult = targetm.sched.reassociation_width (MULT_EXPR, mode);
    5590         7188 :   if (width > 1 && mult_num && param_fully_pipelined_fma
    5591            0 :       && width_mult <= width)
    5592              :     {
    5593              :       /* Latency of MULT_EXPRs.  */
    5594            0 :       int lat_mul
    5595            0 :         = get_mult_latency_consider_fma (ops_num, mult_num, width_mult);
    5596              : 
    5597              :       /* Quick search might not apply.  So start from 1.  */
    5598            0 :       for (int i = 1; i < width_mult; i++)
    5599              :         {
    5600            0 :           int lat_mul_new
    5601            0 :             = get_mult_latency_consider_fma (ops_num, mult_num, i);
    5602            0 :           int lat_add_new = get_required_cycles (ops_num, i);
    5603              : 
    5604              :           /* Assume latency(MULT) >= latency(ADD).  */
    5605            0 :           if (lat_mul - lat_mul_new >= lat_add_new - cycles_best)
    5606              :             {
    5607              :               width = i;
    5608              :               break;
    5609              :             }
    5610              :         }
    5611              :     }
    5612              :   else
    5613              :     {
    5614        16944 :       while (width > width_min)
    5615              :         {
    5616        11656 :           int width_mid = (width + width_min) / 2;
    5617              : 
    5618        11656 :           if (get_required_cycles (ops_num, width_mid) == cycles_best)
    5619              :             width = width_mid;
    5620         2097 :           else if (width_min < width_mid)
    5621              :             width_min = width_mid;
    5622              :           else
    5623              :             break;
    5624              :         }
    5625              :     }
    5626              : 
    5627              :   /* If there's loop dependent FMA result, return width=2 to avoid it.  This is
    5628              :      better than skipping these FMA candidates in widening_mul.  */
    5629         7188 :   if (width == 1
    5630         7188 :       && maybe_le (tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (lhs))),
    5631              :                    param_avoid_fma_max_bits))
    5632              :     {
    5633              :       /* Look for cross backedge dependency:
    5634              :         1. LHS is a phi argument in the same basic block it is defined.
    5635              :         2. And the result of the phi node is used in OPS.  */
    5636         4868 :       basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (lhs));
    5637              : 
    5638         4868 :       use_operand_p use_p;
    5639         4868 :       imm_use_iterator iter;
    5640        10943 :       FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
    5641        10310 :         if (gphi *phi = dyn_cast<gphi *> (USE_STMT (use_p)))
    5642              :           {
    5643         4235 :             if (gimple_phi_arg_edge (phi, phi_arg_index_from_use (use_p))->src
    5644              :                 != bb)
    5645            0 :               continue;
    5646         4235 :             tree phi_result = gimple_phi_result (phi);
    5647         4235 :             operand_entry *oe;
    5648         4235 :             unsigned int j;
    5649        20976 :             FOR_EACH_VEC_ELT (*ops, j, oe)
    5650              :               {
    5651        12705 :                 if (TREE_CODE (oe->op) != SSA_NAME)
    5652            0 :                   continue;
    5653              : 
    5654              :                 /* Result of phi is operand of PLUS_EXPR.  */
    5655        12705 :                 if (oe->op == phi_result)
    5656         2039 :                   return 2;
    5657              : 
    5658              :                 /* Check is result of phi is operand of MULT_EXPR.  */
    5659        10666 :                 gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
    5660        10666 :                 if (is_gimple_assign (def_stmt)
    5661        10666 :                     && gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR)
    5662              :                   {
    5663         2246 :                     tree rhs = gimple_assign_rhs1 (def_stmt);
    5664         2246 :                     if (TREE_CODE (rhs) == SSA_NAME)
    5665              :                       {
    5666         2246 :                         if (rhs == phi_result)
    5667              :                           return 2;
    5668         2246 :                         def_stmt = SSA_NAME_DEF_STMT (rhs);
    5669              :                       }
    5670              :                   }
    5671        10666 :                 if (is_gimple_assign (def_stmt)
    5672        10666 :                     && gimple_assign_rhs_code (def_stmt) == MULT_EXPR)
    5673              :                   {
    5674         8470 :                     if (gimple_assign_rhs1 (def_stmt) == phi_result
    5675         8470 :                         || gimple_assign_rhs2 (def_stmt) == phi_result)
    5676              :                       return 2;
    5677              :                   }
    5678              :               }
    5679         4868 :           }
    5680              :     }
    5681              : 
    5682              :   return width;
    5683              : }
    5684              : 
    5685              : #define SPECIAL_BIASED_END_STMT 0 /* It is the end stmt of all ops.  */
    5686              : #define BIASED_END_STMT 1 /* It is the end stmt of normal or biased ops.  */
    5687              : #define NORMAL_END_STMT 2 /* It is the end stmt of normal ops.  */
    5688              : 
    5689              : /* Rewrite statements with dependency chain with regard the chance to generate
    5690              :    FMA.
    5691              :    For the chain with FMA: Try to keep fma opportunity as much as possible.
    5692              :    For the chain without FMA: Putting the computation in rank order and trying
    5693              :    to allow operations to be executed in parallel.
    5694              :    E.g.
    5695              :    e + f + a * b + c * d;
    5696              : 
    5697              :    ssa1 = e + a * b;
    5698              :    ssa2 = f + c * d;
    5699              :    ssa3 = ssa1 + ssa2;
    5700              : 
    5701              :    This reassociation approach preserves the chance of fma generation as much
    5702              :    as possible.
    5703              : 
    5704              :    Another thing is to avoid adding loop-carried ops to long chains, otherwise
    5705              :    the whole chain will have dependencies across the loop iteration.  Just keep
    5706              :    loop-carried ops in a separate chain.
    5707              :    E.g.
    5708              :    x_1 = phi (x_0, x_2)
    5709              :    y_1 = phi (y_0, y_2)
    5710              : 
    5711              :    a + b + c + d + e + x1 + y1
    5712              : 
    5713              :    SSA1 = a + b;
    5714              :    SSA2 = c + d;
    5715              :    SSA3 = SSA1 + e;
    5716              :    SSA4 = SSA3 + SSA2;
    5717              :    SSA5 = x1 + y1;
    5718              :    SSA6 = SSA4 + SSA5;
    5719              :  */
    5720              : static void
    5721         1900 : rewrite_expr_tree_parallel (gassign *stmt, int width, bool has_fma,
    5722              :                             const vec<operand_entry *> &ops)
    5723              : {
    5724         1900 :   enum tree_code opcode = gimple_assign_rhs_code (stmt);
    5725         1900 :   int op_num = ops.length ();
    5726         1900 :   int op_normal_num = op_num;
    5727         1900 :   gcc_assert (op_num > 0);
    5728         1900 :   int stmt_num = op_num - 1;
    5729         1900 :   gimple **stmts = XALLOCAVEC (gimple *, stmt_num);
    5730         1900 :   int i = 0, j = 0;
    5731         1900 :   tree tmp_op[2], op1;
    5732         1900 :   operand_entry *oe;
    5733         1900 :   gimple *stmt1 = NULL;
    5734         1900 :   tree last_rhs1 = gimple_assign_rhs1 (stmt);
    5735         1900 :   int last_rhs1_stmt_index = 0, last_rhs2_stmt_index = 0;
    5736         1900 :   int width_active = 0, width_count = 0;
    5737         1900 :   bool has_biased = false, ops_changed = false;
    5738         1900 :   auto_vec<operand_entry *> ops_normal;
    5739         1900 :   auto_vec<operand_entry *> ops_biased;
    5740         1900 :   vec<operand_entry *> *ops1;
    5741              : 
    5742              :   /* We start expression rewriting from the top statements.
    5743              :      So, in this loop we create a full list of statements
    5744              :      we will work with.  */
    5745         1900 :   stmts[stmt_num - 1] = stmt;
    5746         9250 :   for (i = stmt_num - 2; i >= 0; i--)
    5747         7350 :     stmts[i] = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmts[i+1]));
    5748              : 
    5749              :   /* Avoid adding loop-carried ops to long chains, first filter out the
    5750              :      loop-carried.  But we need to make sure that the length of the remainder
    5751              :      is not less than 4, which is the smallest ops length we can break the
    5752              :      dependency.  */
    5753        14950 :   FOR_EACH_VEC_ELT (ops, i, oe)
    5754              :     {
    5755        11150 :       if (TREE_CODE (oe->op) == SSA_NAME
    5756        10984 :           && bitmap_bit_p (biased_names, SSA_NAME_VERSION (oe->op))
    5757        11407 :           && op_normal_num > 4)
    5758              :         {
    5759          223 :           ops_biased.safe_push (oe);
    5760          223 :           has_biased = true;
    5761          223 :           op_normal_num --;
    5762              :         }
    5763              :       else
    5764        10927 :         ops_normal.safe_push (oe);
    5765              :     }
    5766              : 
    5767              :   /* Width should not be larger than ops length / 2, since we can not create
    5768              :      more parallel dependency chains that exceeds such value.  */
    5769         1900 :   int width_normal = op_normal_num / 2;
    5770         1900 :   int width_biased = (op_num - op_normal_num) / 2;
    5771         1900 :   width_normal = width <= width_normal ? width : width_normal;
    5772         1900 :   width_biased = width <= width_biased ? width : width_biased;
    5773              : 
    5774         1900 :   ops1 = &ops_normal;
    5775         1900 :   width_count = width_active = width_normal;
    5776              : 
    5777              :   /* Build parallel dependency chain according to width.  */
    5778        11150 :   for (i = 0; i < stmt_num; i++)
    5779              :     {
    5780         9250 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5781              :         {
    5782            6 :           fprintf (dump_file, "Transforming ");
    5783            6 :           print_gimple_stmt (dump_file, stmts[i], 0);
    5784              :         }
    5785              : 
    5786              :       /* When the work of normal ops is over, but the loop is not over,
    5787              :          continue to do biased ops.  */
    5788         9250 :       if (width_count == 0 && ops1 == &ops_normal)
    5789              :         {
    5790          219 :           ops1 = &ops_biased;
    5791          219 :           width_count = width_active = width_biased;
    5792          219 :           ops_changed = true;
    5793              :         }
    5794              : 
    5795              :       /* Swap the operands if no FMA in the chain.  */
    5796         9250 :       if (ops1->length () > 2 && !has_fma)
    5797         4513 :         swap_ops_for_binary_stmt (*ops1, ops1->length () - 3);
    5798              : 
    5799         9250 :       if (i < width_active
    5800         5254 :           || (ops_changed && i <= (last_rhs1_stmt_index + width_active)))
    5801              :         {
    5802        11997 :           for (j = 0; j < 2; j++)
    5803              :             {
    5804         7998 :               oe = ops1->pop ();
    5805         7998 :               tmp_op[j] = oe->op;
    5806              :               /* If the stmt that defines operand has to be inserted, insert it
    5807              :                  before the use.  */
    5808         7998 :               stmt1 = oe->stmt_to_insert;
    5809         7998 :               if (stmt1)
    5810            0 :                 insert_stmt_before_use (stmts[i], stmt1);
    5811         7998 :               stmt1 = NULL;
    5812              :             }
    5813         3999 :           stmts[i] = build_and_add_sum (TREE_TYPE (last_rhs1),
    5814              :                                         tmp_op[1],
    5815              :                                         tmp_op[0],
    5816              :                                         opcode);
    5817         3999 :           gimple_set_visited (stmts[i], true);
    5818              : 
    5819              :         }
    5820              :       else
    5821              :         {
    5822              :           /* We keep original statement only for the last one.  All others are
    5823              :              recreated.  */
    5824         5251 :           if (!ops1->length ())
    5825              :             {
    5826              :               /* For biased length equal to 2.  */
    5827         2099 :               if (width_count == BIASED_END_STMT && !last_rhs2_stmt_index)
    5828            1 :                 last_rhs2_stmt_index = i - 1;
    5829              : 
    5830              :               /* When width_count == 2 and there is no biased, just finish.  */
    5831         2099 :               if (width_count == NORMAL_END_STMT && !has_biased)
    5832              :                 {
    5833         1681 :                   last_rhs1_stmt_index = i - 1;
    5834         1681 :                   last_rhs2_stmt_index = i - 2;
    5835              :                 }
    5836         2099 :               if (last_rhs1_stmt_index && (last_rhs2_stmt_index || !has_biased))
    5837              :                 {
    5838              :                   /* We keep original statement only for the last one.  All
    5839              :                      others are recreated.  */
    5840         1683 :                   gimple_assign_set_rhs1 (stmts[i], gimple_assign_lhs
    5841         1683 :                                           (stmts[last_rhs1_stmt_index]));
    5842         1683 :                   gimple_assign_set_rhs2 (stmts[i], gimple_assign_lhs
    5843         1683 :                                           (stmts[last_rhs2_stmt_index]));
    5844         1683 :                   update_stmt (stmts[i]);
    5845              :                 }
    5846              :               else
    5847              :                 {
    5848         1248 :                   stmts[i] =
    5849          416 :                     build_and_add_sum (TREE_TYPE (last_rhs1),
    5850          416 :                                        gimple_assign_lhs (stmts[i-width_count]),
    5851              :                                        gimple_assign_lhs
    5852          416 :                                        (stmts[i-width_count+1]),
    5853              :                                        opcode);
    5854          416 :                   gimple_set_visited (stmts[i], true);
    5855          416 :                   width_count--;
    5856              : 
    5857              :                   /* It is the end of normal or biased ops.
    5858              :                      last_rhs1_stmt_index used to record the last stmt index
    5859              :                      for normal ops.  last_rhs2_stmt_index used to record the
    5860              :                      last stmt index for biased ops.  */
    5861          416 :                   if (width_count == BIASED_END_STMT)
    5862              :                     {
    5863          220 :                       gcc_assert (has_biased);
    5864          220 :                       if (ops_biased.length ())
    5865              :                         last_rhs1_stmt_index = i;
    5866              :                       else
    5867            1 :                         last_rhs2_stmt_index = i;
    5868              :                       width_count--;
    5869              :                     }
    5870              :                 }
    5871              :             }
    5872              :           else
    5873              :             {
    5874              :               /* Attach the rest ops to the parallel dependency chain.  */
    5875         3152 :               oe = ops1->pop ();
    5876         3152 :               op1 = oe->op;
    5877         3152 :               stmt1 = oe->stmt_to_insert;
    5878         3152 :               if (stmt1)
    5879            0 :                 insert_stmt_before_use (stmts[i], stmt1);
    5880         3152 :               stmt1 = NULL;
    5881              : 
    5882              :               /* For only one biased ops.  */
    5883         3152 :               if (width_count == SPECIAL_BIASED_END_STMT)
    5884              :                 {
    5885              :                   /* We keep original statement only for the last one.  All
    5886              :                      others are recreated.  */
    5887          217 :                   gcc_assert (has_biased);
    5888          217 :                   gimple_assign_set_rhs1 (stmts[i], gimple_assign_lhs
    5889          217 :                                           (stmts[last_rhs1_stmt_index]));
    5890          217 :                   gimple_assign_set_rhs2 (stmts[i], op1);
    5891          217 :                   update_stmt (stmts[i]);
    5892              :                 }
    5893              :               else
    5894              :                 {
    5895         2935 :                   stmts[i] = build_and_add_sum (TREE_TYPE (last_rhs1),
    5896              :                                                 gimple_assign_lhs
    5897         2935 :                                                 (stmts[i-width_active]),
    5898              :                                                 op1,
    5899              :                                                 opcode);
    5900         2935 :                   gimple_set_visited (stmts[i], true);
    5901              :                 }
    5902              :             }
    5903              :         }
    5904              : 
    5905         9250 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5906              :         {
    5907            6 :           fprintf (dump_file, " into ");
    5908            6 :           print_gimple_stmt (dump_file, stmts[i], 0);
    5909              :         }
    5910              :     }
    5911              : 
    5912         1900 :   remove_visited_stmt_chain (last_rhs1);
    5913         1900 : }
    5914              : 
    5915              : /* Transform STMT, which is really (A +B) + (C + D) into the left
    5916              :    linear form, ((A+B)+C)+D.
    5917              :    Recurse on D if necessary.  */
    5918              : 
    5919              : static void
    5920         2630 : linearize_expr (gimple *stmt)
    5921              : {
    5922         2630 :   gimple_stmt_iterator gsi;
    5923         2630 :   gimple *binlhs = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
    5924         2630 :   gimple *binrhs = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
    5925         2630 :   gimple *oldbinrhs = binrhs;
    5926         2630 :   enum tree_code rhscode = gimple_assign_rhs_code (stmt);
    5927         2630 :   gimple *newbinrhs = NULL;
    5928         2630 :   class loop *loop = loop_containing_stmt (stmt);
    5929         2630 :   tree lhs = gimple_assign_lhs (stmt);
    5930              : 
    5931         2630 :   gcc_assert (is_reassociable_op (binlhs, rhscode, loop)
    5932              :               && is_reassociable_op (binrhs, rhscode, loop));
    5933              : 
    5934         2630 :   gsi = gsi_for_stmt (stmt);
    5935              : 
    5936         2630 :   gimple_assign_set_rhs2 (stmt, gimple_assign_rhs1 (binrhs));
    5937         2630 :   binrhs = gimple_build_assign (make_ssa_name (TREE_TYPE (lhs)),
    5938              :                                 gimple_assign_rhs_code (binrhs),
    5939              :                                 gimple_assign_lhs (binlhs),
    5940              :                                 gimple_assign_rhs2 (binrhs));
    5941         2630 :   gimple_assign_set_rhs1 (stmt, gimple_assign_lhs (binrhs));
    5942         2630 :   gsi_insert_before (&gsi, binrhs, GSI_SAME_STMT);
    5943         2630 :   gimple_set_uid (binrhs, gimple_uid (stmt));
    5944              : 
    5945         2630 :   if (TREE_CODE (gimple_assign_rhs2 (stmt)) == SSA_NAME)
    5946         2627 :     newbinrhs = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
    5947              : 
    5948         2630 :   if (dump_file && (dump_flags & TDF_DETAILS))
    5949              :     {
    5950            0 :       fprintf (dump_file, "Linearized: ");
    5951            0 :       print_gimple_stmt (dump_file, stmt, 0);
    5952              :     }
    5953              : 
    5954         2630 :   reassociate_stats.linearized++;
    5955         2630 :   update_stmt (stmt);
    5956              : 
    5957         2630 :   gsi = gsi_for_stmt (oldbinrhs);
    5958         2630 :   reassoc_remove_stmt (&gsi);
    5959         2630 :   release_defs (oldbinrhs);
    5960              : 
    5961         2630 :   gimple_set_visited (stmt, true);
    5962         2630 :   gimple_set_visited (binlhs, true);
    5963         2630 :   gimple_set_visited (binrhs, true);
    5964              : 
    5965              :   /* Tail recurse on the new rhs if it still needs reassociation.  */
    5966         2630 :   if (newbinrhs && is_reassociable_op (newbinrhs, rhscode, loop))
    5967              :     /* ??? This should probably be linearize_expr (newbinrhs) but I don't
    5968              :            want to change the algorithm while converting to tuples.  */
    5969          622 :     linearize_expr (stmt);
    5970         2630 : }
    5971              : 
    5972              : /* If LHS has a single immediate use that is a GIMPLE_ASSIGN statement, return
    5973              :    it.  Otherwise, return NULL.  */
    5974              : 
    5975              : static gimple *
    5976       413066 : get_single_immediate_use (tree lhs)
    5977              : {
    5978       413066 :   use_operand_p immuse;
    5979       413066 :   gimple *immusestmt;
    5980              : 
    5981       413066 :   if (TREE_CODE (lhs) == SSA_NAME
    5982       413066 :       && single_imm_use (lhs, &immuse, &immusestmt)
    5983       732347 :       && is_gimple_assign (immusestmt))
    5984       243057 :     return immusestmt;
    5985              : 
    5986              :   return NULL;
    5987              : }
    5988              : 
    5989              : /* Recursively negate the value of TONEGATE, and return the SSA_NAME
    5990              :    representing the negated value.  Insertions of any necessary
    5991              :    instructions go before GSI.
    5992              :    This function is recursive in that, if you hand it "a_5" as the
    5993              :    value to negate, and a_5 is defined by "a_5 = b_3 + b_4", it will
    5994              :    transform b_3 + b_4 into a_5 = -b_3 + -b_4.  */
    5995              : 
    5996              : static tree
    5997        78114 : negate_value (tree tonegate, gimple_stmt_iterator *gsip)
    5998              : {
    5999        78114 :   gimple *negatedefstmt = NULL;
    6000        78114 :   tree resultofnegate;
    6001        78114 :   gimple_stmt_iterator gsi;
    6002        78114 :   unsigned int uid;
    6003              : 
    6004              :   /* If we are trying to negate a name, defined by an add, negate the
    6005              :      add operands instead.  */
    6006        78114 :   if (TREE_CODE (tonegate) == SSA_NAME)
    6007        76254 :     negatedefstmt = SSA_NAME_DEF_STMT (tonegate);
    6008        78114 :   if (TREE_CODE (tonegate) == SSA_NAME
    6009        76254 :       && is_gimple_assign (negatedefstmt)
    6010        66523 :       && TREE_CODE (gimple_assign_lhs (negatedefstmt)) == SSA_NAME
    6011        66523 :       && has_single_use (gimple_assign_lhs (negatedefstmt))
    6012       127534 :       && gimple_assign_rhs_code (negatedefstmt) == PLUS_EXPR)
    6013              :     {
    6014          902 :       tree rhs1 = gimple_assign_rhs1 (negatedefstmt);
    6015          902 :       tree rhs2 = gimple_assign_rhs2 (negatedefstmt);
    6016          902 :       tree lhs = gimple_assign_lhs (negatedefstmt);
    6017          902 :       gimple *g;
    6018              : 
    6019          902 :       gsi = gsi_for_stmt (negatedefstmt);
    6020          902 :       rhs1 = negate_value (rhs1, &gsi);
    6021              : 
    6022          902 :       gsi = gsi_for_stmt (negatedefstmt);
    6023          902 :       rhs2 = negate_value (rhs2, &gsi);
    6024              : 
    6025          902 :       gsi = gsi_for_stmt (negatedefstmt);
    6026          902 :       lhs = make_ssa_name (TREE_TYPE (lhs));
    6027          902 :       gimple_set_visited (negatedefstmt, true);
    6028          902 :       g = gimple_build_assign (lhs, PLUS_EXPR, rhs1, rhs2);
    6029          902 :       gimple_set_uid (g, gimple_uid (negatedefstmt));
    6030          902 :       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    6031          902 :       return lhs;
    6032              :     }
    6033              : 
    6034        77212 :   tonegate = fold_build1 (NEGATE_EXPR, TREE_TYPE (tonegate), tonegate);
    6035        77212 :   resultofnegate = force_gimple_operand_gsi (gsip, tonegate, true,
    6036              :                                              NULL_TREE, true, GSI_SAME_STMT);
    6037        77212 :   gsi = *gsip;
    6038        77212 :   uid = gimple_uid (gsi_stmt (gsi));
    6039       157135 :   for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
    6040              :     {
    6041       147993 :       gimple *stmt = gsi_stmt (gsi);
    6042       147993 :       if (gimple_uid (stmt) != 0)
    6043              :         break;
    6044        75352 :       gimple_set_uid (stmt, uid);
    6045              :     }
    6046              :   return resultofnegate;
    6047              : }
    6048              : 
    6049              : /* Return true if we should break up the subtract in STMT into an add
    6050              :    with negate.  This is true when we the subtract operands are really
    6051              :    adds, or the subtract itself is used in an add expression.  In
    6052              :    either case, breaking up the subtract into an add with negate
    6053              :    exposes the adds to reassociation.  */
    6054              : 
    6055              : static bool
    6056       298387 : should_break_up_subtract (gimple *stmt)
    6057              : {
    6058       298387 :   tree lhs = gimple_assign_lhs (stmt);
    6059       298387 :   tree binlhs = gimple_assign_rhs1 (stmt);
    6060       298387 :   tree binrhs = gimple_assign_rhs2 (stmt);
    6061       298387 :   gimple *immusestmt;
    6062       298387 :   class loop *loop = loop_containing_stmt (stmt);
    6063              : 
    6064       298387 :   if (TREE_CODE (binlhs) == SSA_NAME
    6065       298387 :       && is_reassociable_op (SSA_NAME_DEF_STMT (binlhs), PLUS_EXPR, loop))
    6066              :     return true;
    6067              : 
    6068       280585 :   if (TREE_CODE (binrhs) == SSA_NAME
    6069       280585 :       && is_reassociable_op (SSA_NAME_DEF_STMT (binrhs), PLUS_EXPR, loop))
    6070              :     return true;
    6071              : 
    6072       279773 :   if (TREE_CODE (lhs) == SSA_NAME
    6073       279773 :       && (immusestmt = get_single_immediate_use (lhs))
    6074       131170 :       && is_gimple_assign (immusestmt)
    6075       410943 :       && (gimple_assign_rhs_code (immusestmt) == PLUS_EXPR
    6076        86791 :           || (gimple_assign_rhs_code (immusestmt) == MINUS_EXPR
    6077         2244 :               && gimple_assign_rhs1 (immusestmt) == lhs)
    6078        84557 :           || gimple_assign_rhs_code (immusestmt) == MULT_EXPR))
    6079        57696 :     return true;
    6080              :   return false;
    6081              : }
    6082              : 
    6083              : /* Transform STMT from A - B into A + -B.  */
    6084              : 
    6085              : static void
    6086        76310 : break_up_subtract (gimple *stmt, gimple_stmt_iterator *gsip)
    6087              : {
    6088        76310 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    6089        76310 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    6090              : 
    6091        76310 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6092              :     {
    6093            0 :       fprintf (dump_file, "Breaking up subtract ");
    6094            0 :       print_gimple_stmt (dump_file, stmt, 0);
    6095              :     }
    6096              : 
    6097        76310 :   rhs2 = negate_value (rhs2, gsip);
    6098        76310 :   gimple_assign_set_rhs_with_ops (gsip, PLUS_EXPR, rhs1, rhs2);
    6099        76310 :   update_stmt (stmt);
    6100        76310 : }
    6101              : 
    6102              : /* Determine whether STMT is a builtin call that raises an SSA name
    6103              :    to an integer power and has only one use.  If so, and this is early
    6104              :    reassociation and unsafe math optimizations are permitted, place
    6105              :    the SSA name in *BASE and the exponent in *EXPONENT, and return TRUE.
    6106              :    If any of these conditions does not hold, return FALSE.  */
    6107              : 
    6108              : static bool
    6109          131 : acceptable_pow_call (gcall *stmt, tree *base, HOST_WIDE_INT *exponent)
    6110              : {
    6111          131 :   tree arg1;
    6112          131 :   REAL_VALUE_TYPE c, cint;
    6113              : 
    6114          131 :   switch (gimple_call_combined_fn (stmt))
    6115              :     {
    6116           24 :     CASE_CFN_POW:
    6117           24 :       if (flag_errno_math)
    6118              :         return false;
    6119              : 
    6120           24 :       *base = gimple_call_arg (stmt, 0);
    6121           24 :       arg1 = gimple_call_arg (stmt, 1);
    6122              : 
    6123           24 :       if (TREE_CODE (arg1) != REAL_CST)
    6124              :         return false;
    6125              : 
    6126           19 :       c = TREE_REAL_CST (arg1);
    6127              : 
    6128           19 :       if (REAL_EXP (&c) > HOST_BITS_PER_WIDE_INT)
    6129              :         return false;
    6130              : 
    6131           19 :       *exponent = real_to_integer (&c);
    6132           19 :       real_from_integer (&cint, VOIDmode, *exponent, SIGNED);
    6133           19 :       if (!real_identical (&c, &cint))
    6134              :         return false;
    6135              : 
    6136              :       break;
    6137              : 
    6138            9 :     CASE_CFN_POWI:
    6139            9 :       *base = gimple_call_arg (stmt, 0);
    6140            9 :       arg1 = gimple_call_arg (stmt, 1);
    6141              : 
    6142            9 :       if (!tree_fits_shwi_p (arg1))
    6143              :         return false;
    6144              : 
    6145            7 :       *exponent = tree_to_shwi (arg1);
    6146            7 :       break;
    6147              : 
    6148              :     default:
    6149              :       return false;
    6150              :     }
    6151              : 
    6152              :   /* Expanding negative exponents is generally unproductive, so we don't
    6153              :      complicate matters with those.  Exponents of zero and one should
    6154              :      have been handled by expression folding.  */
    6155           16 :   if (*exponent < 2
    6156           16 :       || TREE_CODE (*base) != SSA_NAME
    6157           32 :       || !can_reassociate_op_p (*base))
    6158              :     return false;
    6159              : 
    6160              :   return true;
    6161              : }
    6162              : 
    6163              : /* Try to derive and add operand entry for OP to *OPS.  Return false if
    6164              :    unsuccessful.  */
    6165              : 
    6166              : static bool
    6167      9859616 : try_special_add_to_ops (vec<operand_entry *> *ops,
    6168              :                         enum tree_code code,
    6169              :                         tree op, gimple* def_stmt)
    6170              : {
    6171      9859616 :   tree base = NULL_TREE;
    6172      9859616 :   HOST_WIDE_INT exponent = 0;
    6173              : 
    6174      9859616 :   if (TREE_CODE (op) != SSA_NAME
    6175      9859616 :       || ! has_single_use (op))
    6176              :     return false;
    6177              : 
    6178      3698174 :   if (code == MULT_EXPR
    6179       800726 :       && reassoc_insert_powi_p
    6180       376912 :       && flag_unsafe_math_optimizations
    6181        30352 :       && is_gimple_call (def_stmt)
    6182      3698305 :       && acceptable_pow_call (as_a <gcall *> (def_stmt), &base, &exponent))
    6183              :     {
    6184           12 :       add_repeat_to_ops_vec (ops, base, exponent);
    6185           12 :       gimple_set_visited (def_stmt, true);
    6186           12 :       return true;
    6187              :     }
    6188      3698162 :   else if (code == MULT_EXPR
    6189       800714 :            && is_gimple_assign (def_stmt)
    6190       759100 :            && gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR
    6191          585 :            && !HONOR_SNANS (TREE_TYPE (op))
    6192          585 :            && (!HONOR_SIGNED_ZEROS (TREE_TYPE (op))
    6193            0 :                || !COMPLEX_FLOAT_TYPE_P (TREE_TYPE (op)))
    6194      3698747 :            && (!FLOAT_TYPE_P (TREE_TYPE (op))
    6195           50 :                || !DECIMAL_FLOAT_MODE_P (element_mode (op))))
    6196              :     {
    6197          578 :       tree rhs1 = gimple_assign_rhs1 (def_stmt);
    6198          578 :       if (!can_reassociate_op_p (rhs1))
    6199              :         return false;
    6200          568 :       tree cst = build_minus_one_cst (TREE_TYPE (op));
    6201          568 :       add_to_ops_vec (ops, rhs1);
    6202          568 :       add_to_ops_vec (ops, cst);
    6203          568 :       gimple_set_visited (def_stmt, true);
    6204          568 :       return true;
    6205              :     }
    6206              : 
    6207              :   return false;
    6208              : }
    6209              : 
    6210              : /* Recursively linearize a binary expression that is the RHS of STMT.
    6211              :    Place the operands of the expression tree in the vector named OPS.  */
    6212              : 
    6213              : static void
    6214      5059567 : linearize_expr_tree (vec<operand_entry *> *ops, gimple *stmt,
    6215              :                      bool is_associative, bool set_visited)
    6216              : {
    6217      5059567 :   tree binlhs = gimple_assign_rhs1 (stmt);
    6218      5059567 :   tree binrhs = gimple_assign_rhs2 (stmt);
    6219      5059567 :   gimple *binlhsdef = NULL, *binrhsdef = NULL;
    6220      5059567 :   bool binlhsisreassoc = false;
    6221      5059567 :   bool binrhsisreassoc = false;
    6222      5059567 :   enum tree_code rhscode = gimple_assign_rhs_code (stmt);
    6223      5059567 :   class loop *loop = loop_containing_stmt (stmt);
    6224              : 
    6225      5059567 :   if (set_visited)
    6226      5018844 :     gimple_set_visited (stmt, true);
    6227              : 
    6228      5059567 :   if (TREE_CODE (binlhs) == SSA_NAME)
    6229              :     {
    6230      5056941 :       binlhsdef = SSA_NAME_DEF_STMT (binlhs);
    6231      5056941 :       binlhsisreassoc = (is_reassociable_op (binlhsdef, rhscode, loop)
    6232      5056941 :                          && !stmt_could_throw_p (cfun, binlhsdef));
    6233              :     }
    6234              : 
    6235      5059567 :   if (TREE_CODE (binrhs) == SSA_NAME)
    6236              :     {
    6237      1598130 :       binrhsdef = SSA_NAME_DEF_STMT (binrhs);
    6238      1598130 :       binrhsisreassoc = (is_associative
    6239      1598113 :                          && is_reassociable_op (binrhsdef, rhscode, loop)
    6240      1664711 :                          && !stmt_could_throw_p (cfun, binrhsdef));
    6241              :     }
    6242              : 
    6243              :   /* If the LHS is not reassociable, but the RHS is, we need to swap
    6244              :      them.  If neither is reassociable, there is nothing we can do, so
    6245              :      just put them in the ops vector.  If the LHS is reassociable,
    6246              :      linearize it.  If both are reassociable, then linearize the RHS
    6247              :      and the LHS.  */
    6248              : 
    6249      5059567 :   if (!binlhsisreassoc)
    6250              :     {
    6251              :       /* If this is not a associative operation like division, give up.  */
    6252      4864656 :       if (!is_associative)
    6253              :         {
    6254           17 :           add_to_ops_vec (ops, binrhs);
    6255           17 :           return;
    6256              :         }
    6257              : 
    6258      4864639 :       if (!binrhsisreassoc)
    6259              :         {
    6260      4800066 :           bool swap = false;
    6261      4800066 :           if (try_special_add_to_ops (ops, rhscode, binrhs, binrhsdef))
    6262              :             /* If we add ops for the rhs we expect to be able to recurse
    6263              :                to it via the lhs during expression rewrite so swap
    6264              :                operands.  */
    6265              :             swap = true;
    6266              :           else
    6267      4799686 :             add_to_ops_vec (ops, binrhs);
    6268              : 
    6269      4800066 :           if (!try_special_add_to_ops (ops, rhscode, binlhs, binlhsdef))
    6270      4799871 :             add_to_ops_vec (ops, binlhs);
    6271              : 
    6272      4800066 :           if (!swap)
    6273              :             return;
    6274              :         }
    6275              : 
    6276        64953 :       if (dump_file && (dump_flags & TDF_DETAILS))
    6277              :         {
    6278            9 :           fprintf (dump_file, "swapping operands of ");
    6279            9 :           print_gimple_stmt (dump_file, stmt, 0);
    6280              :         }
    6281              : 
    6282        64953 :       swap_ssa_operands (stmt,
    6283              :                          gimple_assign_rhs1_ptr (stmt),
    6284              :                          gimple_assign_rhs2_ptr (stmt));
    6285        64953 :       update_stmt (stmt);
    6286              : 
    6287        64953 :       if (dump_file && (dump_flags & TDF_DETAILS))
    6288              :         {
    6289            9 :           fprintf (dump_file, " is now ");
    6290            9 :           print_gimple_stmt (dump_file, stmt, 0);
    6291              :         }
    6292        64953 :       if (!binrhsisreassoc)
    6293              :         return;
    6294              : 
    6295              :       /* We want to make it so the lhs is always the reassociative op,
    6296              :          so swap.  */
    6297              :       std::swap (binlhs, binrhs);
    6298              :     }
    6299       194911 :   else if (binrhsisreassoc)
    6300              :     {
    6301         2008 :       linearize_expr (stmt);
    6302         2008 :       binlhs = gimple_assign_rhs1 (stmt);
    6303         2008 :       binrhs = gimple_assign_rhs2 (stmt);
    6304              :     }
    6305              : 
    6306       259484 :   gcc_checking_assert (TREE_CODE (binrhs) != SSA_NAME
    6307              :                        || !is_associative
    6308              :                        || !is_reassociable_op (SSA_NAME_DEF_STMT (binrhs),
    6309              :                                                rhscode, loop));
    6310       259484 :   linearize_expr_tree (ops, SSA_NAME_DEF_STMT (binlhs),
    6311              :                        is_associative, set_visited);
    6312              : 
    6313       259484 :   if (!try_special_add_to_ops (ops, rhscode, binrhs, binrhsdef))
    6314       259479 :     add_to_ops_vec (ops, binrhs);
    6315              : }
    6316              : 
    6317              : /* Repropagate the negates back into subtracts, since no other pass
    6318              :    currently does it.  */
    6319              : 
    6320              : static void
    6321      2125670 : repropagate_negates (void)
    6322              : {
    6323      2125670 :   unsigned int i = 0;
    6324      2125670 :   tree negate;
    6325              : 
    6326      2258963 :   FOR_EACH_VEC_ELT (plus_negates, i, negate)
    6327              :     {
    6328       133293 :       gimple *user = get_single_immediate_use (negate);
    6329       133293 :       if (!user || !is_gimple_assign (user))
    6330        21406 :         continue;
    6331              : 
    6332       111887 :       tree negateop = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (negate));
    6333       111897 :       if (TREE_CODE (negateop) == SSA_NAME
    6334       111887 :           && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (negateop))
    6335           10 :         continue;
    6336              : 
    6337              :       /* The negate operand can be either operand of a PLUS_EXPR
    6338              :          (it can be the LHS if the RHS is a constant for example).
    6339              : 
    6340              :          Force the negate operand to the RHS of the PLUS_EXPR, then
    6341              :          transform the PLUS_EXPR into a MINUS_EXPR.  */
    6342       111877 :       if (gimple_assign_rhs_code (user) == PLUS_EXPR)
    6343              :         {
    6344              :           /* If the negated operand appears on the LHS of the
    6345              :              PLUS_EXPR, exchange the operands of the PLUS_EXPR
    6346              :              to force the negated operand to the RHS of the PLUS_EXPR.  */
    6347        84462 :           if (gimple_assign_rhs1 (user) == negate)
    6348              :             {
    6349        25751 :               swap_ssa_operands (user,
    6350              :                                  gimple_assign_rhs1_ptr (user),
    6351              :                                  gimple_assign_rhs2_ptr (user));
    6352              :             }
    6353              : 
    6354              :           /* Now transform the PLUS_EXPR into a MINUS_EXPR and replace
    6355              :              the RHS of the PLUS_EXPR with the operand of the NEGATE_EXPR.  */
    6356        84462 :           if (gimple_assign_rhs2 (user) == negate)
    6357              :             {
    6358        84462 :               tree rhs1 = gimple_assign_rhs1 (user);
    6359        84462 :               gimple_stmt_iterator gsi = gsi_for_stmt (user);
    6360        84462 :               gimple_assign_set_rhs_with_ops (&gsi, MINUS_EXPR, rhs1,
    6361              :                                               negateop);
    6362        84462 :               update_stmt (user);
    6363              :             }
    6364              :         }
    6365        27415 :       else if (gimple_assign_rhs_code (user) == MINUS_EXPR)
    6366              :         {
    6367         2052 :           if (gimple_assign_rhs1 (user) == negate)
    6368              :             {
    6369              :               /* We have
    6370              :                    x = -negateop
    6371              :                    y = x - b
    6372              :                  which we transform into
    6373              :                    x = negateop + b
    6374              :                    y = -x .
    6375              :                  This pushes down the negate which we possibly can merge
    6376              :                  into some other operation, hence insert it into the
    6377              :                  plus_negates vector.  */
    6378         2052 :               gimple *feed = SSA_NAME_DEF_STMT (negate);
    6379         2052 :               tree b = gimple_assign_rhs2 (user);
    6380         2052 :               gimple_stmt_iterator gsi = gsi_for_stmt (feed);
    6381         2052 :               gimple_stmt_iterator gsi2 = gsi_for_stmt (user);
    6382         2052 :               tree x = make_ssa_name (TREE_TYPE (gimple_assign_lhs (feed)));
    6383         2052 :               gimple *g = gimple_build_assign (x, PLUS_EXPR, negateop, b);
    6384         2052 :               gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
    6385         2052 :               gimple_assign_set_rhs_with_ops (&gsi2, NEGATE_EXPR, x);
    6386         2052 :               user = gsi_stmt (gsi2);
    6387         2052 :               update_stmt (user);
    6388         2052 :               reassoc_remove_stmt (&gsi);
    6389         2052 :               release_defs (feed);
    6390         2052 :               plus_negates.safe_push (gimple_assign_lhs (user));
    6391              :             }
    6392              :           else
    6393              :             {
    6394              :               /* Transform "x = -negateop; y = b - x" into "y = b + negateop",
    6395              :                  getting rid of one operation.  */
    6396            0 :               tree rhs1 = gimple_assign_rhs1 (user);
    6397            0 :               gimple_stmt_iterator gsi = gsi_for_stmt (user);
    6398            0 :               gimple_assign_set_rhs_with_ops (&gsi, PLUS_EXPR, rhs1, negateop);
    6399            0 :               update_stmt (gsi_stmt (gsi));
    6400              :             }
    6401              :         }
    6402              :     }
    6403      2125670 : }
    6404              : 
    6405              : /* Break up subtract operations in block BB.
    6406              : 
    6407              :    We do this top down because we don't know whether the subtract is
    6408              :    part of a possible chain of reassociation except at the top.
    6409              : 
    6410              :    IE given
    6411              :    d = f + g
    6412              :    c = a + e
    6413              :    b = c - d
    6414              :    q = b - r
    6415              :    k = t - q
    6416              : 
    6417              :    we want to break up k = t - q, but we won't until we've transformed q
    6418              :    = b - r, which won't be broken up until we transform b = c - d.
    6419              : 
    6420              :    En passant, clear the GIMPLE visited flag on every statement
    6421              :    and set UIDs within each basic block.  */
    6422              : 
    6423              : static void
    6424     19961050 : break_up_subtract_bb (basic_block bb)
    6425              : {
    6426     19961050 :   gimple_stmt_iterator gsi;
    6427     19961050 :   unsigned int uid = 1;
    6428              : 
    6429    220419082 :   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    6430              :     {
    6431    180496982 :       gimple *stmt = gsi_stmt (gsi);
    6432    180496982 :       gimple_set_visited (stmt, false);
    6433    180496982 :       gimple_set_uid (stmt, uid++);
    6434              : 
    6435    180496982 :       if (!is_gimple_assign (stmt)
    6436     48578130 :           || !can_reassociate_type_p (TREE_TYPE (gimple_assign_lhs (stmt)))
    6437    198311818 :           || !can_reassociate_op_p (gimple_assign_lhs (stmt)))
    6438    162682515 :         continue;
    6439              : 
    6440              :       /* Look for simple gimple subtract operations.  */
    6441     17814467 :       if (gimple_assign_rhs_code (stmt) == MINUS_EXPR)
    6442              :         {
    6443       298652 :           if (!can_reassociate_op_p (gimple_assign_rhs1 (stmt))
    6444       298652 :               || !can_reassociate_op_p (gimple_assign_rhs2 (stmt)))
    6445          265 :             continue;
    6446              : 
    6447              :           /* Check for a subtract used only in an addition.  If this
    6448              :              is the case, transform it into add of a negate for better
    6449              :              reassociation.  IE transform C = A-B into C = A + -B if C
    6450              :              is only used in an addition.  */
    6451       298387 :           if (should_break_up_subtract (stmt))
    6452        76310 :             break_up_subtract (stmt, &gsi);
    6453              :         }
    6454     17515815 :       else if (gimple_assign_rhs_code (stmt) == NEGATE_EXPR
    6455     17515815 :                && can_reassociate_op_p (gimple_assign_rhs1 (stmt)))
    6456        44889 :         plus_negates.safe_push (gimple_assign_lhs (stmt));
    6457              :     }
    6458     19961050 : }
    6459              : 
    6460              : /* Used for repeated factor analysis.  */
    6461              : struct repeat_factor
    6462              : {
    6463              :   /* An SSA name that occurs in a multiply chain.  */
    6464              :   tree factor;
    6465              : 
    6466              :   /* Cached rank of the factor.  */
    6467              :   unsigned rank;
    6468              : 
    6469              :   /* Number of occurrences of the factor in the chain.  */
    6470              :   HOST_WIDE_INT count;
    6471              : 
    6472              :   /* An SSA name representing the product of this factor and
    6473              :      all factors appearing later in the repeated factor vector.  */
    6474              :   tree repr;
    6475              : };
    6476              : 
    6477              : 
    6478              : static vec<repeat_factor> repeat_factor_vec;
    6479              : 
    6480              : /* Used for sorting the repeat factor vector.  Sort primarily by
    6481              :    ascending occurrence count, secondarily by descending rank.  */
    6482              : 
    6483              : static int
    6484       262338 : compare_repeat_factors (const void *x1, const void *x2)
    6485              : {
    6486       262338 :   const repeat_factor *rf1 = (const repeat_factor *) x1;
    6487       262338 :   const repeat_factor *rf2 = (const repeat_factor *) x2;
    6488              : 
    6489       262338 :   if (rf1->count < rf2->count)
    6490              :     return -1;
    6491       261707 :   else if (rf1->count > rf2->count)
    6492              :     return 1;
    6493              : 
    6494       261215 :   if (rf1->rank < rf2->rank)
    6495              :     return 1;
    6496       141948 :   else if (rf1->rank > rf2->rank)
    6497       118118 :     return -1;
    6498              : 
    6499              :   return 0;
    6500              : }
    6501              : 
    6502              : /* Look for repeated operands in OPS in the multiply tree rooted at
    6503              :    STMT.  Replace them with an optimal sequence of multiplies and powi
    6504              :    builtin calls, and remove the used operands from OPS.  Return an
    6505              :    SSA name representing the value of the replacement sequence.  */
    6506              : 
    6507              : static tree
    6508       504200 : attempt_builtin_powi (gimple *stmt, vec<operand_entry *> *ops)
    6509              : {
    6510       504200 :   unsigned i, j, vec_len;
    6511       504200 :   int ii;
    6512       504200 :   operand_entry *oe;
    6513       504200 :   repeat_factor *rf1, *rf2;
    6514       504200 :   repeat_factor rfnew;
    6515       504200 :   tree result = NULL_TREE;
    6516       504200 :   tree target_ssa, iter_result;
    6517       504200 :   tree type = TREE_TYPE (gimple_get_lhs (stmt));
    6518       504200 :   tree powi_fndecl = mathfn_built_in (type, BUILT_IN_POWI);
    6519       504200 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    6520       504200 :   gimple *mul_stmt, *pow_stmt;
    6521              : 
    6522              :   /* Expand powers as a series of multiplications if type is integral
    6523              :      or BUILT_IN_POWI doesn't exist for this type and target.  We cannot
    6524              :      simply give up in the latter case as OPS can contain entries with
    6525              :      an occurrence count larger than one for __builtin_pow calls.  */
    6526       504200 :   bool use_mults = INTEGRAL_TYPE_P (type) || !powi_fndecl;
    6527              : 
    6528              :   /* Allocate the repeated factor vector.  */
    6529       504200 :   repeat_factor_vec.create (10);
    6530              : 
    6531              :   /* Scan the OPS vector for all SSA names in the product and build
    6532              :      up a vector of occurrence counts for each factor.  */
    6533      2021153 :   FOR_EACH_VEC_ELT (*ops, i, oe)
    6534              :     {
    6535      1012753 :       if (TREE_CODE (oe->op) == SSA_NAME)
    6536              :         {
    6537       641325 :           FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
    6538              :             {
    6539        71033 :               if (rf1->factor == oe->op)
    6540              :                 {
    6541         2827 :                   rf1->count += oe->count;
    6542         2827 :                   break;
    6543              :                 }
    6544              :             }
    6545              : 
    6546       573119 :           if (j >= repeat_factor_vec.length ())
    6547              :             {
    6548       570292 :               rfnew.factor = oe->op;
    6549       570292 :               rfnew.rank = oe->rank;
    6550       570292 :               rfnew.count = oe->count;
    6551       570292 :               rfnew.repr = NULL_TREE;
    6552       570292 :               repeat_factor_vec.safe_push (rfnew);
    6553              :             }
    6554              :         }
    6555              :     }
    6556              : 
    6557              :   /* Sort the repeated factor vector by (a) increasing occurrence count,
    6558              :      and (b) decreasing rank.  */
    6559       504200 :   repeat_factor_vec.qsort (compare_repeat_factors);
    6560              : 
    6561              :   /* It is generally best to combine as many base factors as possible
    6562              :      into a product before applying __builtin_powi to the result.
    6563              :      However, the sort order chosen for the repeated factor vector
    6564              :      allows us to cache partial results for the product of the base
    6565              :      factors for subsequent use.  When we already have a cached partial
    6566              :      result from a previous iteration, it is best to make use of it
    6567              :      before looking for another __builtin_pow opportunity.
    6568              : 
    6569              :      As an example, consider x * x * y * y * y * z * z * z * z.
    6570              :      We want to first compose the product x * y * z, raise it to the
    6571              :      second power, then multiply this by y * z, and finally multiply
    6572              :      by z.  This can be done in 5 multiplies provided we cache y * z
    6573              :      for use in both expressions:
    6574              : 
    6575              :         t1 = y * z
    6576              :         t2 = t1 * x
    6577              :         t3 = t2 * t2
    6578              :         t4 = t1 * t3
    6579              :         result = t4 * z
    6580              : 
    6581              :      If we instead ignored the cached y * z and first multiplied by
    6582              :      the __builtin_pow opportunity z * z, we would get the inferior:
    6583              : 
    6584              :         t1 = y * z
    6585              :         t2 = t1 * x
    6586              :         t3 = t2 * t2
    6587              :         t4 = z * z
    6588              :         t5 = t3 * t4
    6589              :         result = t5 * y  */
    6590              : 
    6591      1008400 :   vec_len = repeat_factor_vec.length ();
    6592              : 
    6593              :   /* Repeatedly look for opportunities to create a builtin_powi call.  */
    6594       505997 :   while (true)
    6595              :     {
    6596       505997 :       HOST_WIDE_INT power;
    6597              : 
    6598              :       /* First look for the largest cached product of factors from
    6599              :          preceding iterations.  If found, create a builtin_powi for
    6600              :          it if the minimum occurrence count for its factors is at
    6601              :          least 2, or just use this cached product as our next
    6602              :          multiplicand if the minimum occurrence count is 1.  */
    6603      1078386 :       FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
    6604              :         {
    6605       572398 :           if (rf1->repr && rf1->count > 0)
    6606              :             break;
    6607              :         }
    6608              : 
    6609       505997 :       if (j < vec_len)
    6610              :         {
    6611            9 :           power = rf1->count;
    6612              : 
    6613            9 :           if (power == 1)
    6614              :             {
    6615            7 :               iter_result = rf1->repr;
    6616              : 
    6617            7 :               if (dump_file && (dump_flags & TDF_DETAILS))
    6618              :                 {
    6619            0 :                   unsigned elt;
    6620            0 :                   repeat_factor *rf;
    6621            0 :                   fputs ("Multiplying by cached product ", dump_file);
    6622            0 :                   for (elt = j; elt < vec_len; elt++)
    6623              :                     {
    6624            0 :                       rf = &repeat_factor_vec[elt];
    6625            0 :                       print_generic_expr (dump_file, rf->factor);
    6626            0 :                       if (elt < vec_len - 1)
    6627            0 :                         fputs (" * ", dump_file);
    6628              :                     }
    6629            0 :                   fputs ("\n", dump_file);
    6630              :                 }
    6631              :             }
    6632              :           else
    6633              :             {
    6634            2 :               if (use_mults)
    6635              :                 {
    6636            0 :                   gcc_assert (power > 1);
    6637            0 :                   gimple_stmt_iterator gsip = gsi;
    6638            0 :                   gsi_prev (&gsip);
    6639            0 :                   iter_result = powi_as_mults (&gsi, gimple_location (stmt),
    6640              :                                                rf1->repr, power);
    6641            0 :                   gimple_stmt_iterator gsic = gsi;
    6642            0 :                   while (gsi_stmt (gsic) != gsi_stmt (gsip))
    6643              :                     {
    6644            0 :                       gimple_set_uid (gsi_stmt (gsic), gimple_uid (stmt));
    6645            0 :                       gimple_set_visited (gsi_stmt (gsic), true);
    6646            0 :                       gsi_prev (&gsic);
    6647              :                     }
    6648              :                 }
    6649              :               else
    6650              :                 {
    6651            2 :                   iter_result = make_temp_ssa_name (type, NULL, "reassocpow");
    6652            2 :                   pow_stmt
    6653            2 :                     = gimple_build_call (powi_fndecl, 2, rf1->repr,
    6654              :                                          build_int_cst (integer_type_node,
    6655            2 :                                                         power));
    6656            2 :                   gimple_call_set_lhs (pow_stmt, iter_result);
    6657            2 :                   gimple_set_location (pow_stmt, gimple_location (stmt));
    6658            2 :                   gimple_set_uid (pow_stmt, gimple_uid (stmt));
    6659            2 :                   gsi_insert_before (&gsi, pow_stmt, GSI_SAME_STMT);
    6660              :                 }
    6661              : 
    6662            2 :               if (dump_file && (dump_flags & TDF_DETAILS))
    6663              :                 {
    6664            0 :                   unsigned elt;
    6665            0 :                   repeat_factor *rf;
    6666            0 :                   fputs ("Building __builtin_pow call for cached product (",
    6667              :                          dump_file);
    6668            0 :                   for (elt = j; elt < vec_len; elt++)
    6669              :                     {
    6670            0 :                       rf = &repeat_factor_vec[elt];
    6671            0 :                       print_generic_expr (dump_file, rf->factor);
    6672            0 :                       if (elt < vec_len - 1)
    6673            0 :                         fputs (" * ", dump_file);
    6674              :                     }
    6675            0 :                   fprintf (dump_file, ")^" HOST_WIDE_INT_PRINT_DEC"\n",
    6676              :                            power);
    6677              :                 }
    6678              :             }
    6679              :         }
    6680              :       else
    6681              :         {
    6682              :           /* Otherwise, find the first factor in the repeated factor
    6683              :              vector whose occurrence count is at least 2.  If no such
    6684              :              factor exists, there are no builtin_powi opportunities
    6685              :              remaining.  */
    6686      1076545 :           FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
    6687              :             {
    6688       572345 :               if (rf1->count >= 2)
    6689              :                 break;
    6690              :             }
    6691              : 
    6692       505988 :           if (j >= vec_len)
    6693              :             break;
    6694              : 
    6695         1788 :           power = rf1->count;
    6696              : 
    6697         1788 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6698              :             {
    6699            0 :               unsigned elt;
    6700            0 :               repeat_factor *rf;
    6701            0 :               fputs ("Building __builtin_pow call for (", dump_file);
    6702            0 :               for (elt = j; elt < vec_len; elt++)
    6703              :                 {
    6704            0 :                   rf = &repeat_factor_vec[elt];
    6705            0 :                   print_generic_expr (dump_file, rf->factor);
    6706            0 :                   if (elt < vec_len - 1)
    6707            0 :                     fputs (" * ", dump_file);
    6708              :                 }
    6709            0 :               fprintf (dump_file, ")^" HOST_WIDE_INT_PRINT_DEC"\n", power);
    6710              :             }
    6711              : 
    6712         1788 :           reassociate_stats.pows_created++;
    6713              : 
    6714              :           /* Visit each element of the vector in reverse order (so that
    6715              :              high-occurrence elements are visited first, and within the
    6716              :              same occurrence count, lower-ranked elements are visited
    6717              :              first).  Form a linear product of all elements in this order
    6718              :              whose occurrencce count is at least that of element J.
    6719              :              Record the SSA name representing the product of each element
    6720              :              with all subsequent elements in the vector.  */
    6721         1788 :           if (j == vec_len - 1)
    6722         1767 :             rf1->repr = rf1->factor;
    6723              :           else
    6724              :             {
    6725           50 :               for (ii = vec_len - 2; ii >= (int)j; ii--)
    6726              :                 {
    6727           29 :                   tree op1, op2;
    6728              : 
    6729           29 :                   rf1 = &repeat_factor_vec[ii];
    6730           29 :                   rf2 = &repeat_factor_vec[ii + 1];
    6731              : 
    6732              :                   /* Init the last factor's representative to be itself.  */
    6733           29 :                   if (!rf2->repr)
    6734           21 :                     rf2->repr = rf2->factor;
    6735              : 
    6736           29 :                   op1 = rf1->factor;
    6737           29 :                   op2 = rf2->repr;
    6738              : 
    6739           29 :                   target_ssa = make_temp_ssa_name (type, NULL, "reassocpow");
    6740           29 :                   mul_stmt = gimple_build_assign (target_ssa, MULT_EXPR,
    6741              :                                                   op1, op2);
    6742           29 :                   gimple_set_location (mul_stmt, gimple_location (stmt));
    6743           29 :                   gimple_set_uid (mul_stmt, gimple_uid (stmt));
    6744           29 :                   gsi_insert_before (&gsi, mul_stmt, GSI_SAME_STMT);
    6745           29 :                   rf1->repr = target_ssa;
    6746              : 
    6747              :                   /* Don't reprocess the multiply we just introduced.  */
    6748           29 :                   gimple_set_visited (mul_stmt, true);
    6749              :                 }
    6750              :             }
    6751              : 
    6752              :           /* Form a call to __builtin_powi for the maximum product
    6753              :              just formed, raised to the power obtained earlier.  */
    6754         1788 :           rf1 = &repeat_factor_vec[j];
    6755         1788 :           if (use_mults)
    6756              :             {
    6757         1226 :               gcc_assert (power > 1);
    6758         1226 :               gimple_stmt_iterator gsip = gsi;
    6759         1226 :               gsi_prev (&gsip);
    6760         1226 :               iter_result = powi_as_mults (&gsi, gimple_location (stmt),
    6761              :                                            rf1->repr, power);
    6762         1226 :               gimple_stmt_iterator gsic = gsi;
    6763         3715 :               while (gsi_stmt (gsic) != gsi_stmt (gsip))
    6764              :                 {
    6765         2489 :                   gimple_set_uid (gsi_stmt (gsic), gimple_uid (stmt));
    6766         2489 :                   gimple_set_visited (gsi_stmt (gsic), true);
    6767         2545 :                   gsi_prev (&gsic);
    6768              :                 }
    6769              :             }
    6770              :           else
    6771              :             {
    6772          562 :               iter_result = make_temp_ssa_name (type, NULL, "reassocpow");
    6773          562 :               pow_stmt = gimple_build_call (powi_fndecl, 2, rf1->repr,
    6774              :                                             build_int_cst (integer_type_node,
    6775          562 :                                                            power));
    6776          562 :               gimple_call_set_lhs (pow_stmt, iter_result);
    6777          562 :               gimple_set_location (pow_stmt, gimple_location (stmt));
    6778          562 :               gimple_set_uid (pow_stmt, gimple_uid (stmt));
    6779          562 :               gsi_insert_before (&gsi, pow_stmt, GSI_SAME_STMT);
    6780              :             }
    6781              :         }
    6782              : 
    6783              :       /* If we previously formed at least one other builtin_powi call,
    6784              :          form the product of this one and those others.  */
    6785         1797 :       if (result)
    6786              :         {
    6787            9 :           tree new_result = make_temp_ssa_name (type, NULL, "reassocpow");
    6788            9 :           mul_stmt = gimple_build_assign (new_result, MULT_EXPR,
    6789              :                                           result, iter_result);
    6790            9 :           gimple_set_location (mul_stmt, gimple_location (stmt));
    6791            9 :           gimple_set_uid (mul_stmt, gimple_uid (stmt));
    6792            9 :           gsi_insert_before (&gsi, mul_stmt, GSI_SAME_STMT);
    6793            9 :           gimple_set_visited (mul_stmt, true);
    6794            9 :           result = new_result;
    6795              :         }
    6796              :       else
    6797              :         result = iter_result;
    6798              : 
    6799              :       /* Decrement the occurrence count of each element in the product
    6800              :          by the count found above, and remove this many copies of each
    6801              :          factor from OPS.  */
    6802         3628 :       for (i = j; i < vec_len; i++)
    6803              :         {
    6804         1831 :           unsigned k = power;
    6805         1831 :           unsigned n;
    6806              : 
    6807         1831 :           rf1 = &repeat_factor_vec[i];
    6808         1831 :           rf1->count -= power;
    6809              : 
    6810         8726 :           FOR_EACH_VEC_ELT_REVERSE (*ops, n, oe)
    6811              :             {
    6812         5064 :               if (oe->op == rf1->factor)
    6813              :                 {
    6814         4650 :                   if (oe->count <= k)
    6815              :                     {
    6816         4644 :                       ops->ordered_remove (n);
    6817         4644 :                       k -= oe->count;
    6818              : 
    6819         4644 :                       if (k == 0)
    6820              :                         break;
    6821              :                     }
    6822              :                   else
    6823              :                     {
    6824            6 :                       oe->count -= k;
    6825            6 :                       break;
    6826              :                     }
    6827              :                 }
    6828              :             }
    6829              :         }
    6830              :     }
    6831              : 
    6832              :   /* At this point all elements in the repeated factor vector have a
    6833              :      remaining occurrence count of 0 or 1, and those with a count of 1
    6834              :      don't have cached representatives.  Re-sort the ops vector and
    6835              :      clean up.  */
    6836       504200 :   ops->qsort (sort_by_operand_rank);
    6837       504200 :   repeat_factor_vec.release ();
    6838              : 
    6839              :   /* Return the final product computed herein.  Note that there may
    6840              :      still be some elements with single occurrence count left in OPS;
    6841              :      those will be handled by the normal reassociation logic.  */
    6842       504200 :   return result;
    6843              : }
    6844              : 
    6845              : /* Attempt to optimize
    6846              :    CST1 * copysign (CST2, y) -> copysign (CST1 * CST2, y) if CST1 > 0, or
    6847              :    CST1 * copysign (CST2, y) -> -copysign (CST1 * CST2, y) if CST1 < 0.  */
    6848              : 
    6849              : static void
    6850      1108885 : attempt_builtin_copysign (vec<operand_entry *> *ops)
    6851              : {
    6852      1108885 :   operand_entry *oe;
    6853      1108885 :   unsigned int i;
    6854      1108885 :   unsigned int length = ops->length ();
    6855      1108885 :   tree cst = ops->last ()->op;
    6856              : 
    6857      1108885 :   if (length == 1 || TREE_CODE (cst) != REAL_CST)
    6858              :     return;
    6859              : 
    6860         4152 :   FOR_EACH_VEC_ELT (*ops, i, oe)
    6861              :     {
    6862         2961 :       if (TREE_CODE (oe->op) == SSA_NAME
    6863         2961 :           && has_single_use (oe->op))
    6864              :         {
    6865          865 :           gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
    6866         2999 :           if (gcall *old_call = dyn_cast <gcall *> (def_stmt))
    6867              :             {
    6868           54 :               tree arg0, arg1;
    6869           54 :               switch (gimple_call_combined_fn (old_call))
    6870              :                 {
    6871           20 :                 CASE_CFN_COPYSIGN:
    6872           20 :                 CASE_CFN_COPYSIGN_FN:
    6873           20 :                   arg0 = gimple_call_arg (old_call, 0);
    6874           20 :                   arg1 = gimple_call_arg (old_call, 1);
    6875              :                   /* The first argument of copysign must be a constant,
    6876              :                      otherwise there's nothing to do.  */
    6877           20 :                   if (TREE_CODE (arg0) == REAL_CST)
    6878              :                     {
    6879           20 :                       tree type = TREE_TYPE (arg0);
    6880           20 :                       tree mul = const_binop (MULT_EXPR, type, cst, arg0);
    6881              :                       /* If we couldn't fold to a single constant, skip it.
    6882              :                          That happens e.g. for inexact multiplication when
    6883              :                          -frounding-math.  */
    6884           20 :                       if (mul == NULL_TREE)
    6885              :                         break;
    6886              :                       /* Instead of adjusting OLD_CALL, let's build a new
    6887              :                          call to not leak the LHS and prevent keeping bogus
    6888              :                          debug statements.  DCE will clean up the old call.  */
    6889           16 :                       gcall *new_call;
    6890           16 :                       if (gimple_call_internal_p (old_call))
    6891            0 :                         new_call = gimple_build_call_internal
    6892            0 :                           (IFN_COPYSIGN, 2, mul, arg1);
    6893              :                       else
    6894           16 :                         new_call = gimple_build_call
    6895           16 :                           (gimple_call_fndecl (old_call), 2, mul, arg1);
    6896           16 :                       tree lhs = make_ssa_name (type);
    6897           16 :                       gimple_call_set_lhs (new_call, lhs);
    6898           16 :                       gimple_set_location (new_call,
    6899              :                                            gimple_location (old_call));
    6900           16 :                       insert_stmt_after (new_call, old_call);
    6901              :                       /* We've used the constant, get rid of it.  */
    6902           16 :                       ops->pop ();
    6903           16 :                       bool cst1_neg = real_isneg (TREE_REAL_CST_PTR (cst));
    6904              :                       /* Handle the CST1 < 0 case by negating the result.  */
    6905           16 :                       if (cst1_neg)
    6906              :                         {
    6907            7 :                           tree negrhs = make_ssa_name (TREE_TYPE (lhs));
    6908            7 :                           gimple *negate_stmt
    6909            7 :                             = gimple_build_assign (negrhs, NEGATE_EXPR, lhs);
    6910            7 :                           insert_stmt_after (negate_stmt, new_call);
    6911            7 :                           oe->op = negrhs;
    6912              :                         }
    6913              :                       else
    6914              :                         oe->op = lhs;
    6915           16 :                       if (dump_file && (dump_flags & TDF_DETAILS))
    6916              :                         {
    6917           14 :                           fprintf (dump_file, "Optimizing copysign: ");
    6918           14 :                           print_generic_expr (dump_file, cst);
    6919           14 :                           fprintf (dump_file, " * COPYSIGN (");
    6920           14 :                           print_generic_expr (dump_file, arg0);
    6921           14 :                           fprintf (dump_file, ", ");
    6922           14 :                           print_generic_expr (dump_file, arg1);
    6923           23 :                           fprintf (dump_file, ") into %sCOPYSIGN (",
    6924              :                                    cst1_neg ? "-" : "");
    6925           14 :                           print_generic_expr (dump_file, mul);
    6926           14 :                           fprintf (dump_file, ", ");
    6927           14 :                           print_generic_expr (dump_file, arg1);
    6928           14 :                           fprintf (dump_file, "\n");
    6929              :                         }
    6930              :                       return;
    6931              :                     }
    6932              :                   break;
    6933              :                 default:
    6934              :                   break;
    6935              :                 }
    6936              :             }
    6937              :         }
    6938              :     }
    6939              : }
    6940              : 
    6941              : /* Transform STMT at *GSI into a copy by replacing its rhs with NEW_RHS.  */
    6942              : 
    6943              : static void
    6944        14459 : transform_stmt_to_copy (gimple_stmt_iterator *gsi, gimple *stmt, tree new_rhs)
    6945              : {
    6946        14459 :   tree rhs1;
    6947              : 
    6948        14459 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6949              :     {
    6950           28 :       fprintf (dump_file, "Transforming ");
    6951           28 :       print_gimple_stmt (dump_file, stmt, 0);
    6952              :     }
    6953              : 
    6954        14459 :   rhs1 = gimple_assign_rhs1 (stmt);
    6955        14459 :   gimple_assign_set_rhs_from_tree (gsi, new_rhs);
    6956        14459 :   update_stmt (stmt);
    6957        14459 :   remove_visited_stmt_chain (rhs1);
    6958              : 
    6959        14459 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6960              :     {
    6961           28 :       fprintf (dump_file, " into ");
    6962           28 :       print_gimple_stmt (dump_file, stmt, 0);
    6963              :     }
    6964        14459 : }
    6965              : 
    6966              : /* Transform STMT at *GSI into a multiply of RHS1 and RHS2.  */
    6967              : 
    6968              : static void
    6969          193 : transform_stmt_to_multiply (gimple_stmt_iterator *gsi, gimple *stmt,
    6970              :                             tree rhs1, tree rhs2)
    6971              : {
    6972          193 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6973              :     {
    6974            0 :       fprintf (dump_file, "Transforming ");
    6975            0 :       print_gimple_stmt (dump_file, stmt, 0);
    6976              :     }
    6977              : 
    6978          193 :   gimple_assign_set_rhs_with_ops (gsi, MULT_EXPR, rhs1, rhs2);
    6979          193 :   update_stmt (gsi_stmt (*gsi));
    6980          193 :   remove_visited_stmt_chain (rhs1);
    6981              : 
    6982          193 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6983              :     {
    6984            0 :       fprintf (dump_file, " into ");
    6985            0 :       print_gimple_stmt (dump_file, stmt, 0);
    6986              :     }
    6987          193 : }
    6988              : 
    6989              : /* Rearrange ops may have more FMA when the chain may has more than 2 FMAs.
    6990              :    Put no-mult ops and mult ops alternately at the end of the queue, which is
    6991              :    conducive to generating more FMA and reducing the loss of FMA when breaking
    6992              :    the chain.
    6993              :    E.g.
    6994              :    a * b + c * d + e generates:
    6995              : 
    6996              :    _4  = c_9(D) * d_10(D);
    6997              :    _12 = .FMA (a_7(D), b_8(D), _4);
    6998              :    _11 = e_6(D) + _12;
    6999              : 
    7000              :    Rearrange ops to -> e + a * b + c * d generates:
    7001              : 
    7002              :    _4  = .FMA (c_7(D), d_8(D), _3);
    7003              :    _11 = .FMA (a_5(D), b_6(D), _4);
    7004              : 
    7005              :    Return the number of MULT_EXPRs in the chain.  */
    7006              : static int
    7007        16826 : rank_ops_for_fma (vec<operand_entry *> *ops)
    7008              : {
    7009        16826 :   operand_entry *oe;
    7010        16826 :   unsigned int i;
    7011        16826 :   unsigned int ops_length = ops->length ();
    7012        16826 :   auto_vec<operand_entry *> ops_mult;
    7013        16826 :   auto_vec<operand_entry *> ops_others;
    7014              : 
    7015        73006 :   FOR_EACH_VEC_ELT (*ops, i, oe)
    7016              :     {
    7017        39354 :       if (TREE_CODE (oe->op) == SSA_NAME)
    7018              :         {
    7019        39338 :           gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
    7020        39338 :           if (is_gimple_assign (def_stmt))
    7021              :             {
    7022        28093 :               if (gimple_assign_rhs_code (def_stmt) == MULT_EXPR)
    7023        13046 :                 ops_mult.safe_push (oe);
    7024              :               /* A negate on the multiplication leads to FNMA.  */
    7025        15047 :               else if (gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR
    7026        15047 :                        && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
    7027              :                 {
    7028         2839 :                   gimple *neg_def_stmt
    7029         2839 :                     = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
    7030         2839 :                   if (is_gimple_assign (neg_def_stmt)
    7031         2836 :                       && gimple_bb (neg_def_stmt) == gimple_bb (def_stmt)
    7032         5670 :                       && gimple_assign_rhs_code (neg_def_stmt) == MULT_EXPR)
    7033         2792 :                     ops_mult.safe_push (oe);
    7034              :                   else
    7035           47 :                     ops_others.safe_push (oe);
    7036              :                 }
    7037              :               else
    7038        12208 :                 ops_others.safe_push (oe);
    7039              :             }
    7040              :           else
    7041        11245 :             ops_others.safe_push (oe);
    7042              :         }
    7043              :       else
    7044           16 :         ops_others.safe_push (oe);
    7045              :     }
    7046              :   /* 1. When ops_mult.length == 2, like the following case,
    7047              : 
    7048              :      a * b + c * d + e.
    7049              : 
    7050              :      we need to rearrange the ops.
    7051              : 
    7052              :      Putting ops that not def from mult in front can generate more FMAs.
    7053              : 
    7054              :      2. If all ops are defined with mult, we don't need to rearrange them.  */
    7055        16826 :   unsigned mult_num = ops_mult.length ();
    7056        16826 :   if (mult_num >= 2 && mult_num != ops_length)
    7057              :     {
    7058              :       /* Put no-mult ops and mult ops alternately at the end of the
    7059              :          queue, which is conducive to generating more FMA and reducing the
    7060              :          loss of FMA when breaking the chain.  */
    7061         5292 :       ops->truncate (0);
    7062         5292 :       ops->splice (ops_mult);
    7063         5292 :       int j, opindex = ops->length ();
    7064         5292 :       int others_length = ops_others.length ();
    7065        10589 :       for (j = 0; j < others_length; j++)
    7066              :         {
    7067         5297 :           oe = ops_others.pop ();
    7068         5297 :           ops->quick_insert (opindex, oe);
    7069         5297 :           if (opindex > 0)
    7070         5296 :             opindex--;
    7071              :         }
    7072              :     }
    7073        16826 :   return mult_num;
    7074        16826 : }
    7075              : /* Reassociate expressions in basic block BB and its post-dominator as
    7076              :    children.
    7077              : 
    7078              :    Bubble up return status from maybe_optimize_range_tests.  */
    7079              : 
    7080              : static bool
    7081     19960999 : reassociate_bb (basic_block bb)
    7082              : {
    7083     19960999 :   gimple_stmt_iterator gsi;
    7084     19960999 :   gimple *stmt = last_nondebug_stmt (bb);
    7085     19960999 :   bool cfg_cleanup_needed = false;
    7086              : 
    7087     19960999 :   if (stmt && !gimple_visited_p (stmt))
    7088     19325763 :     cfg_cleanup_needed |= maybe_optimize_range_tests (stmt);
    7089              : 
    7090     19960999 :   bool do_prev = false;
    7091     39921998 :   for (gsi = gsi_last_bb (bb);
    7092    381490168 :        !gsi_end_p (gsi); do_prev ? gsi_prev (&gsi) : (void) 0)
    7093              :     {
    7094    180764813 :       do_prev = true;
    7095    180764813 :       stmt = gsi_stmt (gsi);
    7096              : 
    7097    180764813 :       if (is_gimple_assign (stmt)
    7098    180764813 :           && !stmt_could_throw_p (cfun, stmt))
    7099              :         {
    7100     46814228 :           tree lhs, rhs1, rhs2;
    7101     46814228 :           enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
    7102              : 
    7103              :           /* If this was part of an already processed statement,
    7104              :              we don't need to touch it again. */
    7105     46814228 :           if (gimple_visited_p (stmt))
    7106              :             {
    7107              :               /* This statement might have become dead because of previous
    7108              :                  reassociations.  */
    7109       403995 :               if (has_zero_uses (gimple_get_lhs (stmt)))
    7110              :                 {
    7111       138182 :                   reassoc_remove_stmt (&gsi);
    7112       138182 :                   release_defs (stmt);
    7113              :                   /* We might end up removing the last stmt above which
    7114              :                      places the iterator to the end of the sequence.
    7115              :                      Reset it to the last stmt in this case and make sure
    7116              :                      we don't do gsi_prev in that case.  */
    7117       138182 :                   if (gsi_end_p (gsi))
    7118              :                     {
    7119          457 :                       gsi = gsi_last_bb (bb);
    7120          457 :                       do_prev = false;
    7121              :                     }
    7122              :                 }
    7123       403995 :               continue;
    7124              :             }
    7125              : 
    7126              :           /* If this is not a gimple binary expression, there is
    7127              :              nothing for us to do with it.  */
    7128     46410233 :           if (get_gimple_rhs_class (rhs_code) != GIMPLE_BINARY_RHS)
    7129     34386796 :             continue;
    7130              : 
    7131     12023437 :           lhs = gimple_assign_lhs (stmt);
    7132     12023437 :           rhs1 = gimple_assign_rhs1 (stmt);
    7133     12023437 :           rhs2 = gimple_assign_rhs2 (stmt);
    7134              : 
    7135              :           /* For non-bit or min/max operations we can't associate
    7136              :              all types.  Verify that here.  */
    7137     17680194 :           if ((rhs_code != BIT_IOR_EXPR
    7138     12023437 :                && rhs_code != BIT_AND_EXPR
    7139     11010079 :                && rhs_code != BIT_XOR_EXPR
    7140     11010079 :                && rhs_code != MIN_EXPR
    7141     10864775 :                && rhs_code != MAX_EXPR
    7142     10725199 :                && !can_reassociate_type_p (TREE_TYPE (lhs)))
    7143      6371344 :               || !can_reassociate_op_p (rhs1)
    7144     18391695 :               || !can_reassociate_op_p (rhs2))
    7145      5656757 :             continue;
    7146              : 
    7147      6366680 :           if (associative_tree_code (rhs_code))
    7148              :             {
    7149      4807880 :               auto_vec<operand_entry *> ops;
    7150      4807880 :               tree powi_result = NULL_TREE;
    7151      4807880 :               bool is_vector = VECTOR_TYPE_P (TREE_TYPE (lhs));
    7152              : 
    7153              :               /* There may be no immediate uses left by the time we
    7154              :                  get here because we may have eliminated them all.  */
    7155      4807880 :               if (TREE_CODE (lhs) == SSA_NAME && has_zero_uses (lhs))
    7156        47830 :                 continue;
    7157              : 
    7158      4760050 :               gimple_set_visited (stmt, true);
    7159      4760050 :               linearize_expr_tree (&ops, stmt, true, true);
    7160      4760050 :               ops.qsort (sort_by_operand_rank);
    7161      4760050 :               int orig_len = ops.length ();
    7162      4760050 :               optimize_ops_list (rhs_code, &ops);
    7163      9520100 :               if (undistribute_ops_list (rhs_code, &ops,
    7164              :                                          loop_containing_stmt (stmt)))
    7165              :                 {
    7166          315 :                   ops.qsort (sort_by_operand_rank);
    7167          315 :                   optimize_ops_list (rhs_code, &ops);
    7168              :                 }
    7169      9520100 :               if (undistribute_bitref_for_vector (rhs_code, &ops,
    7170              :                                                   loop_containing_stmt (stmt)))
    7171              :                 {
    7172           36 :                   ops.qsort (sort_by_operand_rank);
    7173           36 :                   optimize_ops_list (rhs_code, &ops);
    7174              :                 }
    7175      4760050 :               if (rhs_code == PLUS_EXPR
    7176      4760050 :                   && transform_add_to_multiply (&ops))
    7177           73 :                 ops.qsort (sort_by_operand_rank);
    7178              : 
    7179      4760050 :               if (rhs_code == BIT_IOR_EXPR || rhs_code == BIT_AND_EXPR)
    7180              :                 {
    7181       998033 :                   if (is_vector)
    7182        26638 :                     optimize_vec_cond_expr (rhs_code, &ops);
    7183              :                   else
    7184       971395 :                     optimize_range_tests (rhs_code, &ops, NULL);
    7185              :                 }
    7186              : 
    7187      4760050 :               if (rhs_code == MULT_EXPR && !is_vector)
    7188              :                 {
    7189      1108885 :                   attempt_builtin_copysign (&ops);
    7190              : 
    7191      1108885 :                   if (reassoc_insert_powi_p
    7192      1108885 :                       && (flag_unsafe_math_optimizations
    7193       460767 :                           || (INTEGRAL_TYPE_P (TREE_TYPE (lhs)))))
    7194       504200 :                     powi_result = attempt_builtin_powi (stmt, &ops);
    7195              :                 }
    7196              : 
    7197      4760050 :               operand_entry *last;
    7198      4760050 :               bool negate_result = false;
    7199      4760050 :               if (ops.length () > 1
    7200      4760050 :                   && rhs_code == MULT_EXPR)
    7201              :                 {
    7202      1132784 :                   last = ops.last ();
    7203      1132784 :                   if ((integer_minus_onep (last->op)
    7204      1132263 :                        || real_minus_onep (last->op))
    7205          548 :                       && !HONOR_SNANS (TREE_TYPE (lhs))
    7206      1133332 :                       && (!HONOR_SIGNED_ZEROS (TREE_TYPE (lhs))
    7207            0 :                           || !COMPLEX_FLOAT_TYPE_P (TREE_TYPE (lhs))))
    7208              :                     {
    7209          548 :                       ops.pop ();
    7210          548 :                       negate_result = true;
    7211              :                     }
    7212              :                 }
    7213              : 
    7214      4760050 :               tree new_lhs = lhs;
    7215              :               /* If the operand vector is now empty, all operands were
    7216              :                  consumed by the __builtin_powi optimization.  */
    7217      4760050 :               if (ops.length () == 0)
    7218         1466 :                 transform_stmt_to_copy (&gsi, stmt, powi_result);
    7219      4758584 :               else if (ops.length () == 1)
    7220              :                 {
    7221        13186 :                   tree last_op = ops.last ()->op;
    7222              : 
    7223              :                   /* If the stmt that defines operand has to be inserted, insert it
    7224              :                      before the use.  */
    7225        13186 :                   if (ops.last ()->stmt_to_insert)
    7226            0 :                     insert_stmt_before_use (stmt, ops.last ()->stmt_to_insert);
    7227        13186 :                   if (powi_result)
    7228          193 :                     transform_stmt_to_multiply (&gsi, stmt, last_op,
    7229              :                                                 powi_result);
    7230              :                   else
    7231        12993 :                     transform_stmt_to_copy (&gsi, stmt, last_op);
    7232              :                 }
    7233              :               else
    7234              :                 {
    7235      4745398 :                   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
    7236      4745398 :                   int ops_num = ops.length ();
    7237      4745398 :                   int width = 0;
    7238      4745398 :                   int mult_num = 0;
    7239              : 
    7240              :                   /* For binary bit operations, if there are at least 3
    7241              :                      operands and the last operand in OPS is a constant,
    7242              :                      move it to the front.  This helps ensure that we generate
    7243              :                      (X & Y) & C rather than (X & C) & Y.  The former will
    7244              :                      often match a canonical bit test when we get to RTL.  */
    7245      4745398 :                   if (ops.length () > 2
    7246       182493 :                       && (rhs_code == BIT_AND_EXPR
    7247              :                           || rhs_code == BIT_IOR_EXPR
    7248       159274 :                           || rhs_code == BIT_XOR_EXPR)
    7249      4773209 :                       && TREE_CODE (ops.last ()->op) == INTEGER_CST)
    7250         1775 :                     std::swap (*ops[0], *ops[ops_num - 1]);
    7251              : 
    7252      4745398 :                   optimization_type opt_type = bb_optimization_type (bb);
    7253              : 
    7254              :                   /* If the target support FMA, rank_ops_for_fma will detect if
    7255              :                      the chain has fmas and rearrange the ops if so.  */
    7256      4745398 :                   if (!reassoc_insert_powi_p
    7257      2928062 :                       && direct_internal_fn_supported_p (IFN_FMA,
    7258      2928062 :                                                          TREE_TYPE (lhs),
    7259              :                                                          opt_type)
    7260      4780028 :                       && (rhs_code == PLUS_EXPR || rhs_code == MINUS_EXPR))
    7261              :                     {
    7262        16826 :                       mult_num = rank_ops_for_fma (&ops);
    7263              :                     }
    7264              : 
    7265              :                   /* A chain that expands to a conditional compare sequence
    7266              :                      has to stay linear.  Every conditional compare tests the
    7267              :                      flags that the previous comparison set, so the sequence
    7268              :                      cannot be split into independent parts.  */
    7269      4745398 :                   bool ccmp_chain = (ops_num > 2
    7270      4745398 :                                      && prepare_ccmp_chain (ops, rhs_code, bb));
    7271              : 
    7272              :                   /* Only rewrite the expression tree to parallel in the
    7273              :                      last reassoc pass to avoid useless work back-and-forth
    7274              :                      with initial linearization.  */
    7275      4745398 :                   bool has_fma = mult_num >= 2 && mult_num != ops_num;
    7276      4745398 :                   if (!reassoc_insert_powi_p
    7277      2928062 :                       && !ccmp_chain
    7278      2928062 :                       && ops.length () > 3
    7279      4759259 :                       && (width = get_reassociation_width (&ops, mult_num, lhs,
    7280              :                                                            rhs_code, mode))
    7281              :                            > 1)
    7282              :                     {
    7283         1900 :                       if (dump_file && (dump_flags & TDF_DETAILS))
    7284            2 :                         fprintf (dump_file,
    7285              :                                  "Width = %d was chosen for reassociation\n",
    7286              :                                  width);
    7287         1900 :                       rewrite_expr_tree_parallel (as_a <gassign *> (stmt),
    7288              :                                                   width,
    7289              :                                                   has_fma,
    7290              :                                                   ops);
    7291              :                     }
    7292              :                   else
    7293              :                     {
    7294              :                       /* When there are three operands left, we want
    7295              :                          to make sure the ones that get the double
    7296              :                          binary op are chosen wisely.  */
    7297      4743498 :                       int len = ops.length ();
    7298      7669660 :                       if (!reassoc_insert_powi_p && !ccmp_chain
    7299      2926162 :                           && len >= 3
    7300      4872547 :                           && (!has_fma
    7301              :                               /* width > 1 means ranking ops results in better
    7302              :                                  parallelism.  Check current value to avoid
    7303              :                                  calling get_reassociation_width again.  */
    7304         5288 :                               || (width != 1
    7305         5288 :                                   && get_reassociation_width (
    7306              :                                        &ops, mult_num, lhs, rhs_code, mode)
    7307              :                                        > 1)))
    7308       125800 :                         swap_ops_for_binary_stmt (ops, len - 3);
    7309              : 
    7310      4743498 :                       new_lhs = rewrite_expr_tree (stmt, rhs_code, 0, ops,
    7311      4743498 :                                                    powi_result != NULL
    7312              :                                                    || negate_result,
    7313              :                                                    len != orig_len);
    7314              :                     }
    7315              : 
    7316              :                   /* If we combined some repeated factors into a
    7317              :                      __builtin_powi call, multiply that result by the
    7318              :                      reassociated operands.  */
    7319      4745398 :                   if (powi_result)
    7320              :                     {
    7321          129 :                       gimple *mul_stmt, *lhs_stmt = SSA_NAME_DEF_STMT (lhs);
    7322          129 :                       tree type = TREE_TYPE (lhs);
    7323          129 :                       tree target_ssa = make_temp_ssa_name (type, NULL,
    7324              :                                                             "reassocpow");
    7325          129 :                       gimple_set_lhs (lhs_stmt, target_ssa);
    7326          129 :                       update_stmt (lhs_stmt);
    7327          129 :                       if (lhs != new_lhs)
    7328              :                         {
    7329          129 :                           target_ssa = new_lhs;
    7330          129 :                           new_lhs = lhs;
    7331              :                         }
    7332          129 :                       mul_stmt = gimple_build_assign (lhs, MULT_EXPR,
    7333              :                                                       powi_result, target_ssa);
    7334          129 :                       gimple_set_location (mul_stmt, gimple_location (stmt));
    7335          129 :                       gimple_set_uid (mul_stmt, gimple_uid (stmt));
    7336          129 :                       gsi_insert_after (&gsi, mul_stmt, GSI_NEW_STMT);
    7337              :                     }
    7338              :                 }
    7339              : 
    7340      4760050 :               if (negate_result)
    7341              :                 {
    7342          548 :                   stmt = SSA_NAME_DEF_STMT (lhs);
    7343          548 :                   tree tmp = make_ssa_name (TREE_TYPE (lhs));
    7344          548 :                   gimple_set_lhs (stmt, tmp);
    7345          548 :                   if (lhs != new_lhs)
    7346          538 :                     tmp = new_lhs;
    7347          548 :                   gassign *neg_stmt = gimple_build_assign (lhs, NEGATE_EXPR,
    7348              :                                                            tmp);
    7349          548 :                   gimple_set_uid (neg_stmt, gimple_uid (stmt));
    7350          548 :                   gsi_insert_after (&gsi, neg_stmt, GSI_NEW_STMT);
    7351          548 :                   update_stmt (stmt);
    7352              :                 }
    7353      4807880 :             }
    7354              :         }
    7355              :     }
    7356              : 
    7357     19960999 :   return cfg_cleanup_needed;
    7358              : }
    7359              : 
    7360              : /* Add jumps around shifts for range tests turned into bit tests.
    7361              :    For each SSA_NAME VAR we have code like:
    7362              :    VAR = ...; // final stmt of range comparison
    7363              :    // bit test here...;
    7364              :    OTHERVAR = ...; // final stmt of the bit test sequence
    7365              :    RES = VAR | OTHERVAR;
    7366              :    Turn the above into:
    7367              :    VAR = ...;
    7368              :    if (VAR != 0)
    7369              :      goto <l3>;
    7370              :    else
    7371              :      goto <l2>;
    7372              :    <l2>:
    7373              :    // bit test here...;
    7374              :    OTHERVAR = ...;
    7375              :    <l3>:
    7376              :    # RES = PHI<1(l1), OTHERVAR(l2)>;  */
    7377              : 
    7378              : static void
    7379      2125670 : branch_fixup (void)
    7380              : {
    7381      2125670 :   tree var;
    7382      2125670 :   unsigned int i;
    7383              : 
    7384      2126057 :   FOR_EACH_VEC_ELT (reassoc_branch_fixups, i, var)
    7385              :     {
    7386          387 :       gimple *def_stmt = SSA_NAME_DEF_STMT (var);
    7387          387 :       gimple *use_stmt;
    7388          387 :       use_operand_p use;
    7389          387 :       bool ok = single_imm_use (var, &use, &use_stmt);
    7390          387 :       gcc_assert (ok
    7391              :                   && is_gimple_assign (use_stmt)
    7392              :                   && gimple_assign_rhs_code (use_stmt) == BIT_IOR_EXPR
    7393              :                   && gimple_bb (def_stmt) == gimple_bb (use_stmt));
    7394              : 
    7395          387 :       basic_block cond_bb = gimple_bb (def_stmt);
    7396          387 :       basic_block then_bb = split_block (cond_bb, def_stmt)->dest;
    7397          387 :       basic_block merge_bb = split_block (then_bb, use_stmt)->dest;
    7398              : 
    7399          387 :       gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
    7400          387 :       gimple *g = gimple_build_cond (NE_EXPR, var,
    7401          387 :                                      build_zero_cst (TREE_TYPE (var)),
    7402              :                                      NULL_TREE, NULL_TREE);
    7403          387 :       location_t loc = gimple_location (use_stmt);
    7404          387 :       gimple_set_location (g, loc);
    7405          387 :       gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    7406              : 
    7407          387 :       edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE);
    7408          387 :       etrue->probability = profile_probability::even ();
    7409          387 :       edge efalse = find_edge (cond_bb, then_bb);
    7410          387 :       efalse->flags = EDGE_FALSE_VALUE;
    7411          387 :       efalse->probability -= etrue->probability;
    7412          387 :       then_bb->count -= etrue->count ();
    7413              : 
    7414          387 :       tree othervar = NULL_TREE;
    7415          387 :       if (gimple_assign_rhs1 (use_stmt) == var)
    7416          284 :         othervar = gimple_assign_rhs2 (use_stmt);
    7417          103 :       else if (gimple_assign_rhs2 (use_stmt) == var)
    7418              :         othervar = gimple_assign_rhs1 (use_stmt);
    7419              :       else
    7420            0 :         gcc_unreachable ();
    7421          387 :       tree lhs = gimple_assign_lhs (use_stmt);
    7422          387 :       gphi *phi = create_phi_node (lhs, merge_bb);
    7423          387 :       add_phi_arg (phi, build_one_cst (TREE_TYPE (lhs)), etrue, loc);
    7424          387 :       add_phi_arg (phi, othervar, single_succ_edge (then_bb), loc);
    7425          387 :       gsi = gsi_for_stmt (use_stmt);
    7426          387 :       gsi_remove (&gsi, true);
    7427              : 
    7428          387 :       set_immediate_dominator (CDI_DOMINATORS, merge_bb, cond_bb);
    7429          387 :       set_immediate_dominator (CDI_POST_DOMINATORS, cond_bb, merge_bb);
    7430              :     }
    7431      2125670 :   reassoc_branch_fixups.release ();
    7432      2125670 : }
    7433              : 
    7434              : void dump_ops_vector (FILE *file, vec<operand_entry *> ops);
    7435              : void debug_ops_vector (vec<operand_entry *> ops);
    7436              : 
    7437              : /* Dump the operand entry vector OPS to FILE.  */
    7438              : 
    7439              : void
    7440            0 : dump_ops_vector (FILE *file, vec<operand_entry *> ops)
    7441              : {
    7442            0 :   operand_entry *oe;
    7443            0 :   unsigned int i;
    7444              : 
    7445            0 :   FOR_EACH_VEC_ELT (ops, i, oe)
    7446              :     {
    7447            0 :       fprintf (file, "Op %d -> rank: %d, tree: ", i, oe->rank);
    7448            0 :       print_generic_expr (file, oe->op);
    7449            0 :       fprintf (file, "\n");
    7450              :     }
    7451            0 : }
    7452              : 
    7453              : /* Dump the operand entry vector OPS to STDERR.  */
    7454              : 
    7455              : DEBUG_FUNCTION void
    7456            0 : debug_ops_vector (vec<operand_entry *> ops)
    7457              : {
    7458            0 :   dump_ops_vector (stderr, ops);
    7459            0 : }
    7460              : 
    7461              : /* Bubble up return status from reassociate_bb.  */
    7462              : 
    7463              : static bool
    7464      2125670 : do_reassoc ()
    7465              : {
    7466      2125670 :   bool cfg_cleanup_needed = false;
    7467      2125670 :   basic_block *worklist = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
    7468              : 
    7469      2125670 :   unsigned sp = 0;
    7470      2125670 :   for (auto son = first_dom_son (CDI_DOMINATORS, ENTRY_BLOCK_PTR_FOR_FN (cfun));
    7471      4251340 :        son; son = next_dom_son (CDI_DOMINATORS, son))
    7472      2125670 :     worklist[sp++] = son;
    7473     22086720 :   while (sp)
    7474              :     {
    7475     19961050 :       basic_block bb = worklist[--sp];
    7476     19961050 :       break_up_subtract_bb (bb);
    7477     19961050 :       for (auto son = first_dom_son (CDI_DOMINATORS, bb);
    7478     37796430 :            son; son = next_dom_son (CDI_DOMINATORS, son))
    7479     17835380 :         worklist[sp++] = son;
    7480              :     }
    7481              : 
    7482     11004736 :   for (auto son = first_dom_son (CDI_POST_DOMINATORS,
    7483      2125670 :                                  EXIT_BLOCK_PTR_FOR_FN (cfun));
    7484     11004736 :        son; son = next_dom_son (CDI_POST_DOMINATORS, son))
    7485      8879066 :     worklist[sp++] = son;
    7486     22086669 :   while (sp)
    7487              :     {
    7488     19960999 :       basic_block bb = worklist[--sp];
    7489     19960999 :       cfg_cleanup_needed |= reassociate_bb (bb);
    7490     19960999 :       for (auto son = first_dom_son (CDI_POST_DOMINATORS, bb);
    7491     31042932 :            son; son = next_dom_son (CDI_POST_DOMINATORS, son))
    7492     11081933 :         worklist[sp++] = son;
    7493              :     }
    7494              : 
    7495      2125670 :   free (worklist);
    7496      2125670 :   return cfg_cleanup_needed;
    7497              : }
    7498              : 
    7499              : /* Initialize the reassociation pass.  */
    7500              : 
    7501              : static void
    7502      2125670 : init_reassoc (void)
    7503              : {
    7504      2125670 :   int i;
    7505      2125670 :   int64_t rank = 2;
    7506      2125670 :   int *bbs = XNEWVEC (int, n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS);
    7507              : 
    7508              :   /* Find the loops, so that we can prevent moving calculations in
    7509              :      them.  */
    7510      2125670 :   loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
    7511              : 
    7512      2125670 :   memset (&reassociate_stats, 0, sizeof (reassociate_stats));
    7513              : 
    7514      2125670 :   next_operand_entry_id = 0;
    7515              : 
    7516              :   /* Reverse RPO (Reverse Post Order) will give us something where
    7517              :      deeper loops come later.  */
    7518      2125670 :   pre_and_rev_post_order_compute (NULL, bbs, false);
    7519      2125670 :   bb_rank = XCNEWVEC (int64_t, last_basic_block_for_fn (cfun));
    7520      2125670 :   operand_rank = new hash_map<tree, int64_t>;
    7521              : 
    7522              :   /* Give each default definition a distinct rank.  This includes
    7523              :      parameters and the static chain.  Walk backwards over all
    7524              :      SSA names so that we get proper rank ordering according
    7525              :      to tree_swap_operands_p.  */
    7526    112809916 :   for (i = num_ssa_names - 1; i > 0; --i)
    7527              :     {
    7528    108558576 :       tree name = ssa_name (i);
    7529    185414706 :       if (name && SSA_NAME_IS_DEFAULT_DEF (name))
    7530      6308902 :         insert_operand_rank (name, ++rank);
    7531              :     }
    7532              : 
    7533              :   /* Set up rank for each BB  */
    7534     22086669 :   for (i = 0; i < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; i++)
    7535     19960999 :     bb_rank[bbs[i]] = ++rank << 16;
    7536              : 
    7537      2125670 :   free (bbs);
    7538      2125670 :   calculate_dominance_info (CDI_POST_DOMINATORS);
    7539      2125670 :   plus_negates = vNULL;
    7540      2125670 :   mark_ssa_maybe_undefs ();
    7541      2125670 : }
    7542              : 
    7543              : /* Cleanup after the reassociation pass, and print stats if
    7544              :    requested.  */
    7545              : 
    7546              : static void
    7547      2125670 : fini_reassoc (void)
    7548              : {
    7549      2125670 :   statistics_counter_event (cfun, "Linearized",
    7550              :                             reassociate_stats.linearized);
    7551      2125670 :   statistics_counter_event (cfun, "Constants eliminated",
    7552              :                             reassociate_stats.constants_eliminated);
    7553      2125670 :   statistics_counter_event (cfun, "Ops eliminated",
    7554              :                             reassociate_stats.ops_eliminated);
    7555      2125670 :   statistics_counter_event (cfun, "Statements rewritten",
    7556              :                             reassociate_stats.rewritten);
    7557      2125670 :   statistics_counter_event (cfun, "Built-in pow[i] calls encountered",
    7558              :                             reassociate_stats.pows_encountered);
    7559      2125670 :   statistics_counter_event (cfun, "Built-in powi calls created",
    7560              :                             reassociate_stats.pows_created);
    7561              : 
    7562      4251340 :   delete operand_rank;
    7563      2125670 :   bitmap_clear (biased_names);
    7564      2125670 :   operand_entry_pool.release ();
    7565      2125670 :   free (bb_rank);
    7566      2125670 :   plus_negates.release ();
    7567      2125670 :   free_dominance_info (CDI_POST_DOMINATORS);
    7568      2125670 :   loop_optimizer_finalize ();
    7569      2125670 : }
    7570              : 
    7571              : /* Gate and execute functions for Reassociation.  If INSERT_POWI_P, enable
    7572              :    insertion of __builtin_powi calls.
    7573              : 
    7574              :    Returns TODO_cfg_cleanup if a CFG cleanup pass is desired due to
    7575              :    optimization of a gimple conditional.  Otherwise returns zero.  */
    7576              : 
    7577              : static unsigned int
    7578      2125670 : execute_reassoc (bool insert_powi_p, bool bias_loop_carried_phi_ranks_p)
    7579              : {
    7580      2125670 :   reassoc_insert_powi_p = insert_powi_p;
    7581      2125670 :   reassoc_bias_loop_carried_phi_ranks_p = bias_loop_carried_phi_ranks_p;
    7582              : 
    7583      2125670 :   init_reassoc ();
    7584              : 
    7585      2125670 :   bool cfg_cleanup_needed;
    7586      2125670 :   cfg_cleanup_needed = do_reassoc ();
    7587      2125670 :   repropagate_negates ();
    7588      2125670 :   branch_fixup ();
    7589              : 
    7590      2125670 :   fini_reassoc ();
    7591      2125670 :   return cfg_cleanup_needed ? TODO_cleanup_cfg : 0;
    7592              : }
    7593              : 
    7594              : namespace {
    7595              : 
    7596              : const pass_data pass_data_reassoc =
    7597              : {
    7598              :   GIMPLE_PASS, /* type */
    7599              :   "reassoc", /* name */
    7600              :   OPTGROUP_NONE, /* optinfo_flags */
    7601              :   TV_TREE_REASSOC, /* tv_id */
    7602              :   ( PROP_cfg | PROP_ssa ), /* properties_required */
    7603              :   0, /* properties_provided */
    7604              :   0, /* properties_destroyed */
    7605              :   0, /* todo_flags_start */
    7606              :   TODO_update_ssa_only_virtuals, /* todo_flags_finish */
    7607              : };
    7608              : 
    7609              : class pass_reassoc : public gimple_opt_pass
    7610              : {
    7611              : public:
    7612       594360 :   pass_reassoc (gcc::context *ctxt)
    7613      1188720 :     : gimple_opt_pass (pass_data_reassoc, ctxt), insert_powi_p (false)
    7614              :   {}
    7615              : 
    7616              :   /* opt_pass methods: */
    7617       297180 :   opt_pass * clone () final override { return new pass_reassoc (m_ctxt); }
    7618       594360 :   void set_pass_param (unsigned int n, bool param) final override
    7619              :     {
    7620       594360 :       gcc_assert (n == 0);
    7621       594360 :       insert_powi_p = param;
    7622       594360 :       bias_loop_carried_phi_ranks_p = !param;
    7623       594360 :     }
    7624      2126036 :   bool gate (function *) final override { return flag_tree_reassoc != 0; }
    7625      2125670 :   unsigned int execute (function *) final override
    7626              :   {
    7627      2125670 :     return execute_reassoc (insert_powi_p, bias_loop_carried_phi_ranks_p);
    7628              :   }
    7629              : 
    7630              :  private:
    7631              :   /* Enable insertion of __builtin_powi calls during execute_reassoc.  See
    7632              :      point 3a in the pass header comment.  */
    7633              :   bool insert_powi_p;
    7634              :   bool bias_loop_carried_phi_ranks_p;
    7635              : }; // class pass_reassoc
    7636              : 
    7637              : } // anon namespace
    7638              : 
    7639              : gimple_opt_pass *
    7640       297180 : make_pass_reassoc (gcc::context *ctxt)
    7641              : {
    7642       297180 :   return new pass_reassoc (ctxt);
    7643              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.