LCOV - code coverage report
Current view: top level - gcc - tree-ssa-forwprop.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 95.6 % 3626 3468
Test Date: 2026-11-14 16:39:50 Functions: 100.0 % 95 95
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Forward propagation of expressions for single use variables.
       2              :    Copyright (C) 2004-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify
       7              : it under the terms of the GNU General Public License as published by
       8              : the Free Software Foundation; either version 3, or (at your option)
       9              : any later version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful,
      12              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      13              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      14              : GNU General Public License for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : #include "config.h"
      21              : #include "system.h"
      22              : #include "coretypes.h"
      23              : #include "backend.h"
      24              : #include "rtl.h"
      25              : #include "tree.h"
      26              : #include "gimple.h"
      27              : #include "cfghooks.h"
      28              : #include "tree-pass.h"
      29              : #include "ssa.h"
      30              : #include "expmed.h"
      31              : #include "optabs-query.h"
      32              : #include "gimple-pretty-print.h"
      33              : #include "fold-const.h"
      34              : #include "stor-layout.h"
      35              : #include "gimple-iterator.h"
      36              : #include "gimple-fold.h"
      37              : #include "tree-eh.h"
      38              : #include "gimplify.h"
      39              : #include "gimplify-me.h"
      40              : #include "tree-cfg.h"
      41              : #include "expr.h"
      42              : #include "tree-dfa.h"
      43              : #include "tree-ssa-propagate.h"
      44              : #include "tree-ssa-dom.h"
      45              : #include "tree-ssa-strlen.h"
      46              : #include "builtins.h"
      47              : #include "tree-cfgcleanup.h"
      48              : #include "cfganal.h"
      49              : #include "optabs-tree.h"
      50              : #include "insn-config.h"
      51              : #include "recog.h"
      52              : #include "cfgloop.h"
      53              : #include "tree-vectorizer.h"
      54              : #include "tree-vector-builder.h"
      55              : #include "vec-perm-indices.h"
      56              : #include "internal-fn.h"
      57              : #include "cgraph.h"
      58              : #include "tree-ssa.h"
      59              : #include "gimple-range.h"
      60              : #include "tree-ssa-dce.h"
      61              : #include "tree-ssa-math-opts.h"
      62              : 
      63              : /* This pass propagates the RHS of assignment statements into use
      64              :    sites of the LHS of the assignment.  It's basically a specialized
      65              :    form of tree combination.   It is hoped all of this can disappear
      66              :    when we have a generalized tree combiner.
      67              : 
      68              :    One class of common cases we handle is forward propagating a single use
      69              :    variable into a COND_EXPR.
      70              : 
      71              :      bb0:
      72              :        x = a COND b;
      73              :        if (x) goto ... else goto ...
      74              : 
      75              :    Will be transformed into:
      76              : 
      77              :      bb0:
      78              :        if (a COND b) goto ... else goto ...
      79              : 
      80              :    Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
      81              : 
      82              :    Or (assuming c1 and c2 are constants):
      83              : 
      84              :      bb0:
      85              :        x = a + c1;
      86              :        if (x EQ/NEQ c2) goto ... else goto ...
      87              : 
      88              :    Will be transformed into:
      89              : 
      90              :      bb0:
      91              :         if (a EQ/NEQ (c2 - c1)) goto ... else goto ...
      92              : 
      93              :    Similarly for x = a - c1.
      94              : 
      95              :    Or
      96              : 
      97              :      bb0:
      98              :        x = !a
      99              :        if (x) goto ... else goto ...
     100              : 
     101              :    Will be transformed into:
     102              : 
     103              :      bb0:
     104              :         if (a == 0) goto ... else goto ...
     105              : 
     106              :    Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
     107              :    For these cases, we propagate A into all, possibly more than one,
     108              :    COND_EXPRs that use X.
     109              : 
     110              :    Or
     111              : 
     112              :      bb0:
     113              :        x = (typecast) a
     114              :        if (x) goto ... else goto ...
     115              : 
     116              :    Will be transformed into:
     117              : 
     118              :      bb0:
     119              :         if (a != 0) goto ... else goto ...
     120              : 
     121              :    (Assuming a is an integral type and x is a boolean or x is an
     122              :     integral and a is a boolean.)
     123              : 
     124              :    Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
     125              :    For these cases, we propagate A into all, possibly more than one,
     126              :    COND_EXPRs that use X.
     127              : 
     128              :    In addition to eliminating the variable and the statement which assigns
     129              :    a value to the variable, we may be able to later thread the jump without
     130              :    adding insane complexity in the dominator optimizer.
     131              : 
     132              :    Also note these transformations can cascade.  We handle this by having
     133              :    a worklist of COND_EXPR statements to examine.  As we make a change to
     134              :    a statement, we put it back on the worklist to examine on the next
     135              :    iteration of the main loop.
     136              : 
     137              :    A second class of propagation opportunities arises for ADDR_EXPR
     138              :    nodes.
     139              : 
     140              :      ptr = &x->y->z;
     141              :      res = *ptr;
     142              : 
     143              :    Will get turned into
     144              : 
     145              :      res = x->y->z;
     146              : 
     147              :    Or
     148              :      ptr = (type1*)&type2var;
     149              :      res = *ptr
     150              : 
     151              :    Will get turned into (if type1 and type2 are the same size
     152              :    and neither have volatile on them):
     153              :      res = VIEW_CONVERT_EXPR<type1>(type2var)
     154              : 
     155              :    Or
     156              : 
     157              :      ptr = &x[0];
     158              :      ptr2 = ptr + <constant>;
     159              : 
     160              :    Will get turned into
     161              : 
     162              :      ptr2 = &x[constant/elementsize];
     163              : 
     164              :   Or
     165              : 
     166              :      ptr = &x[0];
     167              :      offset = index * element_size;
     168              :      offset_p = (pointer) offset;
     169              :      ptr2 = ptr + offset_p
     170              : 
     171              :   Will get turned into:
     172              : 
     173              :      ptr2 = &x[index];
     174              : 
     175              :   Or
     176              :     ssa = (int) decl
     177              :     res = ssa & 1
     178              : 
     179              :   Provided that decl has known alignment >= 2, will get turned into
     180              : 
     181              :     res = 0
     182              : 
     183              :   We also propagate casts into SWITCH_EXPR and COND_EXPR conditions to
     184              :   allow us to remove the cast and {NOT_EXPR,NEG_EXPR} into a subsequent
     185              :   {NOT_EXPR,NEG_EXPR}.
     186              : 
     187              :    This will (of course) be extended as other needs arise.  */
     188              : 
     189              : /* Data structure that contains simplifiable vectorized permute sequences.
     190              :    See recognise_vec_perm_simplify_seq () for a description of the sequence.  */
     191              : 
     192              : struct _vec_perm_simplify_seq
     193              : {
     194              :   /* Defining stmts of vectors in the sequence.  */
     195              :   gassign *v_1_stmt;
     196              :   gassign *v_2_stmt;
     197              :   gassign *v_x_stmt;
     198              :   gassign *v_y_stmt;
     199              :   /* Final permute statement.  */
     200              :   gassign *stmt;
     201              :   /* New selector indices for stmt.  */
     202              :   tree new_sel;
     203              :   /* Elements of each vector and selector.  */
     204              :   unsigned int nelts;
     205              : };
     206              : typedef struct _vec_perm_simplify_seq *vec_perm_simplify_seq;
     207              : 
     208              : static bool forward_propagate_addr_expr (tree, tree, bool);
     209              : 
     210              : /* Set to true if we delete dead edges during the optimization.  */
     211              : static bool cfg_changed;
     212              : 
     213              : static tree rhs_to_tree (tree type, gimple *stmt);
     214              : 
     215              : static bitmap to_purge;
     216              : 
     217              : /* Const-and-copy lattice.  */
     218              : static vec<tree> lattice;
     219              : 
     220              : /* Set the lattice entry for NAME to VAL.  */
     221              : static void
     222     33526974 : fwprop_set_lattice_val (tree name, tree val)
     223              : {
     224     33526974 :   if (TREE_CODE (name) == SSA_NAME)
     225              :     {
     226     33526974 :       if (SSA_NAME_VERSION (name) >= lattice.length ())
     227              :         {
     228        31518 :           lattice.reserve (num_ssa_names - lattice.length ());
     229        21012 :           lattice.quick_grow_cleared (num_ssa_names);
     230              :         }
     231     33526974 :       lattice[SSA_NAME_VERSION (name)] = val;
     232              :       /* As this now constitutes a copy duplicate points-to
     233              :          and range info appropriately.  */
     234     33526974 :       if (TREE_CODE (val) == SSA_NAME)
     235     33030107 :         maybe_duplicate_ssa_info_at_copy (name, val);
     236              :     }
     237     33526974 : }
     238              : 
     239              : /* Invalidate the lattice entry for NAME, done when releasing SSA names.  */
     240              : static void
     241       941733 : fwprop_invalidate_lattice (tree name)
     242              : {
     243       941733 :   if (name
     244       939192 :       && TREE_CODE (name) == SSA_NAME
     245      1880797 :       && SSA_NAME_VERSION (name) < lattice.length ())
     246       939033 :     lattice[SSA_NAME_VERSION (name)] = NULL_TREE;
     247       941733 : }
     248              : 
     249              : /* Get the statement we can propagate from into NAME skipping
     250              :    trivial copies.  Returns the statement which defines the
     251              :    propagation source or NULL_TREE if there is no such one.
     252              :    If SINGLE_USE_ONLY is set considers only sources which have
     253              :    a single use chain up to NAME.  If SINGLE_USE_P is non-null,
     254              :    it is set to whether the chain to NAME is a single use chain
     255              :    or not.  SINGLE_USE_P is not written to if SINGLE_USE_ONLY is set.  */
     256              : 
     257              : static gimple *
     258     28484311 : get_prop_source_stmt (tree name, bool single_use_only, bool *single_use_p)
     259              : {
     260     28484311 :   bool single_use = true;
     261              : 
     262     28485329 :   do {
     263     28484820 :     gimple *def_stmt = SSA_NAME_DEF_STMT (name);
     264              : 
     265     28484820 :     if (!has_single_use (name))
     266              :       {
     267     15521247 :         single_use = false;
     268     15521247 :         if (single_use_only)
     269              :           return NULL;
     270              :       }
     271              : 
     272              :     /* If name is defined by a PHI node or is the default def, bail out.  */
     273     28483262 :     if (!is_gimple_assign (def_stmt))
     274              :       return NULL;
     275              : 
     276              :     /* If def_stmt is a simple copy, continue looking.  */
     277     20055048 :     if (gimple_assign_rhs_code (def_stmt) == SSA_NAME)
     278          509 :       name = gimple_assign_rhs1 (def_stmt);
     279              :     else
     280              :       {
     281     20054539 :         if (!single_use_only && single_use_p)
     282     19726959 :           *single_use_p = single_use;
     283              : 
     284              :         return def_stmt;
     285              :       }
     286          509 :   } while (1);
     287              : }
     288              : 
     289              : /* Checks if the destination ssa name in DEF_STMT can be used as
     290              :    propagation source.  Returns true if so, otherwise false.  */
     291              : 
     292              : static bool
     293     28166711 : can_propagate_from (gimple *def_stmt)
     294              : {
     295     28166711 :   gcc_assert (is_gimple_assign (def_stmt));
     296              : 
     297              :   /* If the rhs has side-effects we cannot propagate from it.  */
     298     28166711 :   if (gimple_has_volatile_ops (def_stmt))
     299              :     return false;
     300              : 
     301              :   /* If the rhs is a load we cannot propagate from it.  */
     302     27571738 :   if (TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_reference
     303     27571738 :       || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_declaration)
     304              :     return false;
     305              : 
     306              :   /* Constants can be always propagated.  */
     307     13569555 :   if (gimple_assign_single_p (def_stmt)
     308     13569555 :       && is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
     309              :     return true;
     310              : 
     311              :   /* We cannot propagate ssa names that occur in abnormal phi nodes.  */
     312     13569555 :   if (stmt_references_abnormal_ssa_name (def_stmt))
     313              :     return false;
     314              : 
     315              :   /* If the definition is a conversion of a pointer to a function type,
     316              :      then we cannot apply optimizations as some targets require
     317              :      function pointers to be canonicalized and in this case this
     318              :      optimization could eliminate a necessary canonicalization.  */
     319     13568865 :   if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
     320              :     {
     321      3285643 :       tree rhs = gimple_assign_rhs1 (def_stmt);
     322      3285643 :       if (FUNCTION_POINTER_TYPE_P (TREE_TYPE (rhs)))
     323          818 :         return false;
     324              :     }
     325              : 
     326              :   return true;
     327              : }
     328              : 
     329              : /* Remove a chain of dead statements starting at the definition of
     330              :    NAME.  The chain is linked via the first operand of the defining statements.
     331              :    If NAME was replaced in its only use then this function can be used
     332              :    to clean up dead stmts.  The function handles already released SSA
     333              :    names gracefully.  */
     334              : 
     335              : static void
     336       233091 : remove_prop_source_from_use (tree name)
     337              : {
     338       295401 :   gimple_stmt_iterator gsi;
     339       295401 :   gimple *stmt;
     340              : 
     341       295401 :   do {
     342       295401 :     basic_block bb;
     343              : 
     344       295401 :     if (SSA_NAME_IN_FREE_LIST (name)
     345       295358 :         || SSA_NAME_IS_DEFAULT_DEF (name)
     346       589486 :         || !has_zero_uses (name))
     347              :       break;
     348              : 
     349        62770 :     stmt = SSA_NAME_DEF_STMT (name);
     350        62770 :     if (gimple_code (stmt) == GIMPLE_PHI
     351        62770 :         || gimple_has_side_effects (stmt))
     352              :       break;
     353              : 
     354        62770 :     bb = gimple_bb (stmt);
     355        62770 :     gsi = gsi_for_stmt (stmt);
     356        62770 :     unlink_stmt_vdef (stmt);
     357        62770 :     if (gsi_remove (&gsi, true))
     358            6 :       bitmap_set_bit (to_purge, bb->index);
     359        62770 :     fwprop_invalidate_lattice (gimple_get_lhs (stmt));
     360        62770 :     release_defs (stmt);
     361              : 
     362        62770 :     name = is_gimple_assign (stmt) ? gimple_assign_rhs1 (stmt) : NULL_TREE;
     363        62770 :   } while (name && TREE_CODE (name) == SSA_NAME);
     364              : 
     365       233091 : }
     366              : 
     367              : /* Return the rhs of a gassign *STMT in a form of a single tree,
     368              :    converted to type TYPE.
     369              : 
     370              :    This should disappear, but is needed so we can combine expressions and use
     371              :    the fold() interfaces. Long term, we need to develop folding and combine
     372              :    routines that deal with gimple exclusively . */
     373              : 
     374              : static tree
     375      7394938 : rhs_to_tree (tree type, gimple *stmt)
     376              : {
     377      7394938 :   location_t loc = gimple_location (stmt);
     378      7394938 :   enum tree_code code = gimple_assign_rhs_code (stmt);
     379      7394938 :   switch (get_gimple_rhs_class (code))
     380              :     {
     381        13290 :     case GIMPLE_TERNARY_RHS:
     382        13290 :       return fold_build3_loc (loc, code, type, gimple_assign_rhs1 (stmt),
     383              :                               gimple_assign_rhs2 (stmt),
     384        13290 :                               gimple_assign_rhs3 (stmt));
     385      5038226 :     case GIMPLE_BINARY_RHS:
     386      5038226 :       return fold_build2_loc (loc, code, type, gimple_assign_rhs1 (stmt),
     387      5038226 :                               gimple_assign_rhs2 (stmt));
     388      2076168 :     case GIMPLE_UNARY_RHS:
     389      2076168 :       return build1 (code, type, gimple_assign_rhs1 (stmt));
     390       267254 :     case GIMPLE_SINGLE_RHS:
     391       267254 :       return gimple_assign_rhs1 (stmt);
     392            0 :     default:
     393            0 :       gcc_unreachable ();
     394              :     }
     395              : }
     396              : 
     397              : /* Combine OP0 CODE OP1 in the context of a COND_EXPR.  Returns
     398              :    the folded result in a form suitable for COND_EXPR_COND or
     399              :    NULL_TREE, if there is no suitable simplified form.  If
     400              :    INVARIANT_ONLY is true only gimple_min_invariant results are
     401              :    considered simplified.  */
     402              : 
     403              : static tree
     404      8331384 : combine_cond_expr_cond (gimple *stmt, enum tree_code code, tree type,
     405              :                         tree op0, tree op1, bool invariant_only)
     406              : {
     407      8331384 :   tree t;
     408              : 
     409      8331384 :   gcc_assert (TREE_CODE_CLASS (code) == tcc_comparison);
     410              : 
     411      8331384 :   t = fold_binary_loc (gimple_location (stmt), code, type, op0, op1);
     412      8331384 :   if (!t)
     413              :     return NULL_TREE;
     414              : 
     415              :   /* Require that we got a boolean type out if we put one in.  */
     416      3646653 :   gcc_assert (TREE_CODE (TREE_TYPE (t)) == TREE_CODE (type));
     417              : 
     418              :   /* Canonicalize the combined condition for use in a COND_EXPR.  */
     419      3646653 :   t = canonicalize_cond_expr_cond (t);
     420              : 
     421              :   /* Bail out if we required an invariant but didn't get one.  */
     422      3646653 :   if (!t || (invariant_only && !is_gimple_min_invariant (t)))
     423      3416026 :     return NULL_TREE;
     424              : 
     425              :   return t;
     426              : }
     427              : 
     428              : /* Combine the comparison OP0 CODE OP1 at LOC with the defining statements
     429              :    of its operand.  Return a new comparison tree or NULL_TREE if there
     430              :    were no simplifying combines.  */
     431              : 
     432              : static tree
     433     22432887 : forward_propagate_into_comparison_1 (gimple *stmt,
     434              :                                      enum tree_code code, tree type,
     435              :                                      tree op0, tree op1)
     436              : {
     437     22432887 :   tree tmp = NULL_TREE;
     438     22432887 :   tree rhs0 = NULL_TREE, rhs1 = NULL_TREE;
     439     22432887 :   bool single_use0_p = false, single_use1_p = false;
     440              : 
     441              :   /* For comparisons use the first operand, that is likely to
     442              :      simplify comparisons against constants.  */
     443     22432887 :   if (TREE_CODE (op0) == SSA_NAME)
     444              :     {
     445     22362689 :       gimple *def_stmt = get_prop_source_stmt (op0, false, &single_use0_p);
     446     22362689 :       if (def_stmt && can_propagate_from (def_stmt))
     447              :         {
     448      5606267 :           enum tree_code def_code = gimple_assign_rhs_code (def_stmt);
     449      5606267 :           bool invariant_only_p = !single_use0_p;
     450              : 
     451      5606267 :           rhs0 = rhs_to_tree (TREE_TYPE (op1), def_stmt);
     452              : 
     453              :           /* Always combine comparisons or conversions from booleans.  */
     454      5606267 :           if (TREE_CODE (op1) == INTEGER_CST
     455      5606267 :               && ((CONVERT_EXPR_CODE_P (def_code)
     456       896858 :                    && TREE_CODE (TREE_TYPE (TREE_OPERAND (rhs0, 0)))
     457              :                       == BOOLEAN_TYPE)
     458      3583135 :                   || TREE_CODE_CLASS (def_code) == tcc_comparison))
     459              :             invariant_only_p = false;
     460              : 
     461      5606267 :           tmp = combine_cond_expr_cond (stmt, code, type,
     462              :                                         rhs0, op1, invariant_only_p);
     463      5606267 :           if (tmp)
     464              :             return tmp;
     465              :         }
     466              :     }
     467              : 
     468              :   /* If that wasn't successful, try the second operand.  */
     469     22210891 :   if (TREE_CODE (op1) == SSA_NAME)
     470              :     {
     471      5522820 :       gimple *def_stmt = get_prop_source_stmt (op1, false, &single_use1_p);
     472      5522820 :       if (def_stmt && can_propagate_from (def_stmt))
     473              :         {
     474      1788671 :           rhs1 = rhs_to_tree (TREE_TYPE (op0), def_stmt);
     475      1788671 :           tmp = combine_cond_expr_cond (stmt, code, type,
     476              :                                         op0, rhs1, !single_use1_p);
     477      1788671 :           if (tmp)
     478              :             return tmp;
     479              :         }
     480              :     }
     481              : 
     482              :   /* If that wasn't successful either, try both operands.  */
     483     22204091 :   if (rhs0 != NULL_TREE
     484     22204091 :       && rhs1 != NULL_TREE)
     485       936446 :     tmp = combine_cond_expr_cond (stmt, code, type,
     486              :                                   rhs0, rhs1,
     487       936446 :                                   !(single_use0_p && single_use1_p));
     488              : 
     489              :   return tmp;
     490              : }
     491              : 
     492              : /* Propagate from the ssa name definition statements of the assignment
     493              :    from a comparison at *GSI into the conditional if that simplifies it.
     494              :    Returns true if the stmt was modified.  */
     495              : 
     496              : static bool
     497      2640354 : forward_propagate_into_comparison (gimple_stmt_iterator *gsi)
     498              : {
     499      2640354 :   gimple *stmt = gsi_stmt (*gsi);
     500      2640354 :   tree tmp;
     501      2640354 :   tree type = TREE_TYPE (gimple_assign_lhs (stmt));
     502      2640354 :   tree rhs1 = gimple_assign_rhs1 (stmt);
     503      2640354 :   tree rhs2 = gimple_assign_rhs2 (stmt);
     504              : 
     505              :   /* Combine the comparison with defining statements.  */
     506      2640354 :   tmp = forward_propagate_into_comparison_1 (stmt,
     507              :                                              gimple_assign_rhs_code (stmt),
     508              :                                              type, rhs1, rhs2);
     509      2640354 :   if (tmp && useless_type_conversion_p (type, TREE_TYPE (tmp)))
     510              :     {
     511         7360 :       if (dump_file)
     512              :         {
     513            0 :           fprintf (dump_file, "  Replaced '");
     514            0 :           print_gimple_expr (dump_file, stmt, 0);
     515            0 :           fprintf (dump_file, "' with '");
     516            0 :           print_generic_expr (dump_file, tmp);
     517            0 :           fprintf (dump_file, "'\n");
     518              :         }
     519         7360 :       gimple_assign_set_rhs_from_tree (gsi, tmp);
     520         7360 :       fold_stmt (gsi);
     521         7360 :       update_stmt (gsi_stmt (*gsi));
     522              : 
     523         7360 :       if (TREE_CODE (rhs1) == SSA_NAME)
     524         7360 :         remove_prop_source_from_use (rhs1);
     525         7360 :       if (TREE_CODE (rhs2) == SSA_NAME)
     526         3123 :         remove_prop_source_from_use (rhs2);
     527              :       return true;
     528              :     }
     529              : 
     530              :   return false;
     531              : }
     532              : 
     533              : /* Propagate from the ssa name definition statements of COND_EXPR
     534              :    in GIMPLE_COND statement STMT into the conditional if that simplifies it.
     535              :    Returns zero if no statement was changed, one if there were
     536              :    changes and two if cfg_cleanup needs to run.  */
     537              : 
     538              : static int
     539     19792533 : forward_propagate_into_gimple_cond (gcond *stmt)
     540              : {
     541     19792533 :   tree tmp;
     542     19792533 :   enum tree_code code = gimple_cond_code (stmt);
     543     19792533 :   tree rhs1 = gimple_cond_lhs (stmt);
     544     19792533 :   tree rhs2 = gimple_cond_rhs (stmt);
     545              : 
     546              :   /* GIMPLE_COND will always be a comparison.  */
     547     19792533 :   gcc_assert (TREE_CODE_CLASS (gimple_cond_code (stmt)) == tcc_comparison);
     548              : 
     549     19792533 :   tmp = forward_propagate_into_comparison_1 (stmt, code,
     550              :                                              boolean_type_node,
     551              :                                              rhs1, rhs2);
     552     19792533 :   if (tmp
     553     19792533 :       && is_gimple_condexpr_for_cond (tmp))
     554              :     {
     555       216804 :       if (dump_file)
     556              :         {
     557            9 :           fprintf (dump_file, "  Replaced '");
     558            9 :           print_gimple_expr (dump_file, stmt, 0);
     559            9 :           fprintf (dump_file, "' with '");
     560            9 :           print_generic_expr (dump_file, tmp);
     561            9 :           fprintf (dump_file, "'\n");
     562              :         }
     563              : 
     564       216804 :       gimple_cond_set_condition_from_tree (stmt, unshare_expr (tmp));
     565       216804 :       update_stmt (stmt);
     566              : 
     567       216804 :       if (TREE_CODE (rhs1) == SSA_NAME)
     568       216804 :         remove_prop_source_from_use (rhs1);
     569       216804 :       if (TREE_CODE (rhs2) == SSA_NAME)
     570         5803 :         remove_prop_source_from_use (rhs2);
     571       216804 :       return is_gimple_min_invariant (tmp) ? 2 : 1;
     572              :     }
     573              : 
     574     19575729 :   if (canonicalize_bool_cond (stmt, gimple_bb (stmt)))
     575              :     return 1;
     576              : 
     577              :   return 0;
     578              : }
     579              : 
     580              : /* We've just substituted an ADDR_EXPR into stmt.  Update all the
     581              :    relevant data structures to match.  */
     582              : 
     583              : static void
     584      2005377 : tidy_after_forward_propagate_addr (gimple *stmt)
     585              : {
     586              :   /* We may have turned a trapping insn into a non-trapping insn.  */
     587      2005377 :   if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
     588           39 :     bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
     589              : 
     590      2005377 :   if (TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
     591       250930 :      recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
     592      2005377 : }
     593              : 
     594              : /* NAME is a SSA_NAME representing DEF_RHS which is of the form
     595              :    ADDR_EXPR <whatever>.
     596              : 
     597              :    Try to forward propagate the ADDR_EXPR into the use USE_STMT.
     598              :    Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
     599              :    node or for recovery of array indexing from pointer arithmetic.
     600              : 
     601              :    Return true if the propagation was successful (the propagation can
     602              :    be not totally successful, yet things may have been changed).  */
     603              : 
     604              : static bool
     605      2863108 : forward_propagate_addr_expr_1 (tree name, tree def_rhs,
     606              :                                gimple_stmt_iterator *use_stmt_gsi,
     607              :                                bool single_use_p)
     608              : {
     609      2863108 :   tree lhs, rhs, rhs2, array_ref;
     610      2863108 :   gimple *use_stmt = gsi_stmt (*use_stmt_gsi);
     611      2863108 :   enum tree_code rhs_code;
     612      2863108 :   bool res = true;
     613              : 
     614      2863108 :   gcc_assert (TREE_CODE (def_rhs) == ADDR_EXPR);
     615              : 
     616      2863108 :   lhs = gimple_assign_lhs (use_stmt);
     617      2863108 :   rhs_code = gimple_assign_rhs_code (use_stmt);
     618      2863108 :   rhs = gimple_assign_rhs1 (use_stmt);
     619              : 
     620              :   /* Do not perform copy-propagation but recurse through copy chains.  */
     621      2863108 :   if (TREE_CODE (lhs) == SSA_NAME
     622      1424276 :       && rhs_code == SSA_NAME)
     623         8807 :     return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
     624              : 
     625              :   /* The use statement could be a conversion.  Recurse to the uses of the
     626              :      lhs as copyprop does not copy through pointer to integer to pointer
     627              :      conversions and FRE does not catch all cases either.
     628              :      Treat the case of a single-use name and
     629              :      a conversion to def_rhs type separate, though.  */
     630      2854301 :   if (TREE_CODE (lhs) == SSA_NAME
     631      1415469 :       && CONVERT_EXPR_CODE_P (rhs_code))
     632              :     {
     633              :       /* If there is a point in a conversion chain where the types match
     634              :          so we can remove a conversion re-materialize the address here
     635              :          and stop.  */
     636        25065 :       if (single_use_p
     637        25065 :           && useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (def_rhs)))
     638              :         {
     639            2 :           gimple_assign_set_rhs1 (use_stmt, unshare_expr (def_rhs));
     640            2 :           gimple_assign_set_rhs_code (use_stmt, TREE_CODE (def_rhs));
     641            2 :           return true;
     642              :         }
     643              : 
     644              :       /* Else recurse if the conversion preserves the address value.  */
     645        50126 :       if ((INTEGRAL_TYPE_P (TREE_TYPE (lhs))
     646            2 :            || POINTER_TYPE_P (TREE_TYPE (lhs)))
     647        50126 :           && (TYPE_PRECISION (TREE_TYPE (lhs))
     648        25063 :               >= TYPE_PRECISION (TREE_TYPE (def_rhs))))
     649        24996 :         return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
     650              : 
     651              :       return false;
     652              :     }
     653              : 
     654              :   /* If this isn't a conversion chain from this on we only can propagate
     655              :      into compatible pointer contexts.  */
     656      2829236 :   if (!types_compatible_p (TREE_TYPE (name), TREE_TYPE (def_rhs)))
     657              :     return false;
     658              : 
     659              :   /* Propagate through constant pointer adjustments.  */
     660      2807832 :   if (TREE_CODE (lhs) == SSA_NAME
     661      1370786 :       && rhs_code == POINTER_PLUS_EXPR
     662      1370786 :       && rhs == name
     663      2978851 :       && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
     664              :     {
     665       120214 :       tree new_def_rhs;
     666              :       /* As we come here with non-invariant addresses in def_rhs we need
     667              :          to make sure we can build a valid constant offsetted address
     668              :          for further propagation.  Simply rely on fold building that
     669              :          and check after the fact.  */
     670       120214 :       new_def_rhs = fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (rhs)),
     671              :                                  def_rhs,
     672              :                                  fold_convert (ptr_type_node,
     673              :                                                gimple_assign_rhs2 (use_stmt)));
     674       120214 :       if (TREE_CODE (new_def_rhs) == MEM_REF
     675       120214 :           && !is_gimple_mem_ref_addr (TREE_OPERAND (new_def_rhs, 0)))
     676              :         return false;
     677       116098 :       new_def_rhs = build1 (ADDR_EXPR, TREE_TYPE (rhs), new_def_rhs);
     678              : 
     679              :       /* Recurse.  If we could propagate into all uses of lhs do not
     680              :          bother to replace into the current use but just pretend we did.  */
     681       116098 :       if (forward_propagate_addr_expr (lhs, new_def_rhs, single_use_p))
     682              :         return true;
     683              : 
     684        39951 :       if (useless_type_conversion_p (TREE_TYPE (lhs),
     685        39951 :                                      TREE_TYPE (new_def_rhs)))
     686        39951 :         gimple_assign_set_rhs_with_ops (use_stmt_gsi, TREE_CODE (new_def_rhs),
     687              :                                         new_def_rhs);
     688            0 :       else if (is_gimple_min_invariant (new_def_rhs))
     689            0 :         gimple_assign_set_rhs_with_ops (use_stmt_gsi, NOP_EXPR, new_def_rhs);
     690              :       else
     691              :         return false;
     692        39951 :       gcc_assert (gsi_stmt (*use_stmt_gsi) == use_stmt);
     693        39951 :       update_stmt (use_stmt);
     694        39951 :       return true;
     695              :     }
     696              : 
     697              :   /* Now strip away any outer COMPONENT_REF/ARRAY_REF nodes from the LHS.
     698              :      ADDR_EXPR will not appear on the LHS.  */
     699      2687618 :   tree *lhsp = gimple_assign_lhs_ptr (use_stmt);
     700      4078602 :   while (handled_component_p (*lhsp))
     701      1390984 :     lhsp = &TREE_OPERAND (*lhsp, 0);
     702      2687618 :   lhs = *lhsp;
     703              : 
     704              :   /* Now see if the LHS node is a MEM_REF using NAME.  If so,
     705              :      propagate the ADDR_EXPR into the use of NAME and fold the result.  */
     706      2687618 :   if (TREE_CODE (lhs) == MEM_REF
     707      2687618 :       && TREE_OPERAND (lhs, 0) == name)
     708              :     {
     709       903098 :       tree def_rhs_base;
     710       903098 :       poly_int64 def_rhs_offset;
     711              :       /* If the address is invariant we can always fold it.  */
     712       903098 :       if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
     713              :                                                          &def_rhs_offset)))
     714              :         {
     715       854029 :           poly_offset_int off = mem_ref_offset (lhs);
     716       854029 :           tree new_ptr;
     717       854029 :           off += def_rhs_offset;
     718       854029 :           if (TREE_CODE (def_rhs_base) == MEM_REF)
     719              :             {
     720       832864 :               off += mem_ref_offset (def_rhs_base);
     721       832864 :               new_ptr = TREE_OPERAND (def_rhs_base, 0);
     722              :             }
     723              :           else
     724        21165 :             new_ptr = build_fold_addr_expr (def_rhs_base);
     725       854029 :           TREE_OPERAND (lhs, 0) = new_ptr;
     726       854029 :           TREE_OPERAND (lhs, 1)
     727       854029 :             = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (lhs, 1)), off);
     728       854029 :           tidy_after_forward_propagate_addr (use_stmt);
     729              :           /* Continue propagating into the RHS if this was not the only use.  */
     730       854029 :           if (single_use_p)
     731       229508 :             return true;
     732              :         }
     733              :       /* If the LHS is a plain dereference and the value type is the same as
     734              :          that of the pointed-to type of the address we can put the
     735              :          dereferenced address on the LHS preserving the original alias-type.  */
     736        49069 :       else if (integer_zerop (TREE_OPERAND (lhs, 1))
     737        19049 :                && ((gimple_assign_lhs (use_stmt) == lhs
     738        15431 :                     && useless_type_conversion_p
     739        15431 :                          (TREE_TYPE (TREE_OPERAND (def_rhs, 0)),
     740        15431 :                           TREE_TYPE (gimple_assign_rhs1 (use_stmt))))
     741        14103 :                    || types_compatible_p (TREE_TYPE (lhs),
     742        14103 :                                           TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
     743              :                /* Don't forward anything into clobber stmts if it would result
     744              :                   in the lhs no longer being a MEM_REF.  */
     745        57270 :                && (!gimple_clobber_p (use_stmt)
     746          166 :                    || TREE_CODE (TREE_OPERAND (def_rhs, 0)) == MEM_REF))
     747              :         {
     748         8035 :           tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
     749         8035 :           tree new_offset, new_base, saved, new_lhs;
     750        29143 :           while (handled_component_p (*def_rhs_basep))
     751        13073 :             def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
     752         8035 :           saved = *def_rhs_basep;
     753         8035 :           if (TREE_CODE (*def_rhs_basep) == MEM_REF)
     754              :             {
     755         3944 :               new_base = TREE_OPERAND (*def_rhs_basep, 0);
     756         3944 :               new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (lhs, 1)),
     757              :                                          TREE_OPERAND (*def_rhs_basep, 1));
     758              :             }
     759              :           else
     760              :             {
     761         4091 :               new_base = build_fold_addr_expr (*def_rhs_basep);
     762         4091 :               new_offset = TREE_OPERAND (lhs, 1);
     763              :             }
     764         8035 :           tree atype = TREE_TYPE (*def_rhs_basep);
     765         8035 :           if (TYPE_ALIGN (TREE_TYPE (lhs)) < TYPE_ALIGN (atype))
     766          312 :             atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (lhs)));
     767         8035 :           *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
     768         8035 :           TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (lhs);
     769         8035 :           TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (lhs);
     770         8035 :           TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (lhs);
     771         8035 :           new_lhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
     772         8035 :           *lhsp = new_lhs;
     773         8035 :           TREE_THIS_VOLATILE (new_lhs) = TREE_THIS_VOLATILE (lhs);
     774         8035 :           TREE_SIDE_EFFECTS (new_lhs) = TREE_SIDE_EFFECTS (lhs);
     775         8035 :           *def_rhs_basep = saved;
     776         8035 :           tidy_after_forward_propagate_addr (use_stmt);
     777              :           /* Continue propagating into the RHS if this was not the
     778              :              only use.  */
     779         8035 :           if (single_use_p)
     780              :             return true;
     781              :         }
     782              :       else
     783              :         /* We can have a struct assignment dereferencing our name twice.
     784              :            Note that we didn't propagate into the lhs to not falsely
     785              :            claim we did when propagating into the rhs.  */
     786              :         res = false;
     787              :     }
     788              : 
     789              :   /* Strip away any outer COMPONENT_REF, ARRAY_REF or ADDR_EXPR
     790              :      nodes from the RHS.  */
     791      2453899 :   tree *rhsp = gimple_assign_rhs1_ptr (use_stmt);
     792      2453899 :   if (TREE_CODE (*rhsp) == ADDR_EXPR)
     793       239291 :     rhsp = &TREE_OPERAND (*rhsp, 0);
     794      3473934 :   while (handled_component_p (*rhsp))
     795      1020035 :     rhsp = &TREE_OPERAND (*rhsp, 0);
     796      2453899 :   rhs = *rhsp;
     797              : 
     798              :   /* Now see if the RHS node is a MEM_REF using NAME.  If so,
     799              :      propagate the ADDR_EXPR into the use of NAME and fold the result.  */
     800      2453899 :   if (TREE_CODE (rhs) == MEM_REF
     801      2453899 :       && TREE_OPERAND (rhs, 0) == name)
     802              :     {
     803      1165732 :       tree def_rhs_base;
     804      1165732 :       poly_int64 def_rhs_offset;
     805      1165732 :       if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
     806              :                                                          &def_rhs_offset)))
     807              :         {
     808      1127916 :           poly_offset_int off = mem_ref_offset (rhs);
     809      1127916 :           tree new_ptr;
     810      1127916 :           off += def_rhs_offset;
     811      1127916 :           if (TREE_CODE (def_rhs_base) == MEM_REF)
     812              :             {
     813      1100807 :               off += mem_ref_offset (def_rhs_base);
     814      1100807 :               new_ptr = TREE_OPERAND (def_rhs_base, 0);
     815              :             }
     816              :           else
     817        27109 :             new_ptr = build_fold_addr_expr (def_rhs_base);
     818      1127916 :           TREE_OPERAND (rhs, 0) = new_ptr;
     819      1127916 :           TREE_OPERAND (rhs, 1)
     820      1127916 :             = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (rhs, 1)), off);
     821      1127916 :           fold_stmt_inplace (use_stmt_gsi);
     822      1127916 :           tidy_after_forward_propagate_addr (use_stmt);
     823      1127916 :           return res;
     824              :         }
     825              :       /* If the RHS is a plain dereference and the value type is the same as
     826              :          that of the pointed-to type of the address we can put the
     827              :          dereferenced address on the RHS preserving the original alias-type.  */
     828        37816 :       else if (integer_zerop (TREE_OPERAND (rhs, 1))
     829        37816 :                && ((gimple_assign_rhs1 (use_stmt) == rhs
     830        20495 :                     && useless_type_conversion_p
     831        20495 :                          (TREE_TYPE (gimple_assign_lhs (use_stmt)),
     832        20495 :                           TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
     833        22710 :                    || types_compatible_p (TREE_TYPE (rhs),
     834        22710 :                                           TREE_TYPE (TREE_OPERAND (def_rhs, 0)))))
     835              :         {
     836        15397 :           tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
     837        15397 :           tree new_offset, new_base, saved, new_rhs;
     838        54760 :           while (handled_component_p (*def_rhs_basep))
     839        23966 :             def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
     840        15397 :           saved = *def_rhs_basep;
     841        15397 :           if (TREE_CODE (*def_rhs_basep) == MEM_REF)
     842              :             {
     843         7415 :               new_base = TREE_OPERAND (*def_rhs_basep, 0);
     844         7415 :               new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (rhs, 1)),
     845              :                                          TREE_OPERAND (*def_rhs_basep, 1));
     846              :             }
     847              :           else
     848              :             {
     849         7982 :               new_base = build_fold_addr_expr (*def_rhs_basep);
     850         7982 :               new_offset = TREE_OPERAND (rhs, 1);
     851              :             }
     852        15397 :           tree atype = TREE_TYPE (*def_rhs_basep);
     853        15397 :           if (TYPE_ALIGN (TREE_TYPE (rhs)) < TYPE_ALIGN (atype))
     854          526 :             atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (rhs)));
     855        15397 :           *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
     856        15397 :           TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (rhs);
     857        15397 :           TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (rhs);
     858        15397 :           TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (rhs);
     859        15397 :           new_rhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
     860        15397 :           *rhsp = new_rhs;
     861        15397 :           TREE_THIS_VOLATILE (new_rhs) = TREE_THIS_VOLATILE (rhs);
     862        15397 :           TREE_SIDE_EFFECTS (new_rhs) = TREE_SIDE_EFFECTS (rhs);
     863        15397 :           *def_rhs_basep = saved;
     864        15397 :           fold_stmt_inplace (use_stmt_gsi);
     865        15397 :           tidy_after_forward_propagate_addr (use_stmt);
     866        15397 :           return res;
     867              :         }
     868              :     }
     869              : 
     870              :   /* If the use of the ADDR_EXPR is not a POINTER_PLUS_EXPR, there
     871              :      is nothing to do. */
     872      1310586 :   if (gimple_assign_rhs_code (use_stmt) != POINTER_PLUS_EXPR
     873      1310586 :       || gimple_assign_rhs1 (use_stmt) != name)
     874              :     return false;
     875              : 
     876              :   /* The remaining cases are all for turning pointer arithmetic into
     877              :      array indexing.  They only apply when we have the address of
     878              :      element zero in an array.  If that is not the case then there
     879              :      is nothing to do.  */
     880        50805 :   array_ref = TREE_OPERAND (def_rhs, 0);
     881        50805 :   if ((TREE_CODE (array_ref) != ARRAY_REF
     882         4669 :        || TREE_CODE (TREE_TYPE (TREE_OPERAND (array_ref, 0))) != ARRAY_TYPE
     883         4669 :        || TREE_CODE (TREE_OPERAND (array_ref, 1)) != INTEGER_CST)
     884        52323 :       && TREE_CODE (TREE_TYPE (array_ref)) != ARRAY_TYPE)
     885              :     return false;
     886              : 
     887        27322 :   rhs2 = gimple_assign_rhs2 (use_stmt);
     888              :   /* Optimize &x[C1] p+ C2 to  &x p+ C3 with C3 = C1 * element_size + C2.  */
     889        27322 :   if (TREE_CODE (rhs2) == INTEGER_CST)
     890              :     {
     891            0 :       tree new_rhs = build1_loc (gimple_location (use_stmt),
     892            0 :                                  ADDR_EXPR, TREE_TYPE (def_rhs),
     893            0 :                                  fold_build2 (MEM_REF,
     894              :                                               TREE_TYPE (TREE_TYPE (def_rhs)),
     895              :                                               unshare_expr (def_rhs),
     896              :                                               fold_convert (ptr_type_node,
     897              :                                                             rhs2)));
     898            0 :       gimple_assign_set_rhs_from_tree (use_stmt_gsi, new_rhs);
     899            0 :       use_stmt = gsi_stmt (*use_stmt_gsi);
     900            0 :       update_stmt (use_stmt);
     901            0 :       tidy_after_forward_propagate_addr (use_stmt);
     902            0 :       return true;
     903              :     }
     904              : 
     905              :   return false;
     906              : }
     907              : 
     908              : /* STMT is a statement of the form SSA_NAME = ADDR_EXPR <whatever>.
     909              : 
     910              :    Try to forward propagate the ADDR_EXPR into all uses of the SSA_NAME.
     911              :    Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
     912              :    node or for recovery of array indexing from pointer arithmetic.
     913              : 
     914              :    PARENT_SINGLE_USE_P tells if, when in a recursive invocation, NAME was
     915              :    the single use in the previous invocation.  Pass true when calling
     916              :    this as toplevel.
     917              : 
     918              :    Returns true, if all uses have been propagated into.  */
     919              : 
     920              : static bool
     921      3318841 : forward_propagate_addr_expr (tree name, tree rhs, bool parent_single_use_p)
     922              : {
     923      3318841 :   bool all = true;
     924      3318841 :   bool single_use_p = parent_single_use_p && has_single_use (name);
     925              : 
     926     17519381 :   for (gimple *use_stmt : gather_imm_use_stmts (name))
     927              :     {
     928      7562858 :       bool result;
     929      7562858 :       tree use_rhs;
     930              : 
     931              :       /* If the use is not in a simple assignment statement, then
     932              :          there is nothing we can do.  */
     933      7562858 :       if (!is_gimple_assign (use_stmt))
     934              :         {
     935      4699750 :           if (!is_gimple_debug (use_stmt))
     936      1932735 :             all = false;
     937      4699750 :           continue;
     938              :         }
     939              : 
     940      2863108 :       gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
     941      2863108 :       result = forward_propagate_addr_expr_1 (name, rhs, &gsi,
     942              :                                               single_use_p);
     943              :       /* If the use has moved to a different statement adjust
     944              :          the update machinery for the old statement too.  */
     945      2863108 :       if (use_stmt != gsi_stmt (gsi))
     946              :         {
     947            0 :           update_stmt (use_stmt);
     948            0 :           use_stmt = gsi_stmt (gsi);
     949              :         }
     950      2863108 :       update_stmt (use_stmt);
     951      2863108 :       all &= result;
     952              : 
     953              :       /* Remove intermediate now unused copy and conversion chains.  */
     954      2863108 :       use_rhs = gimple_assign_rhs1 (use_stmt);
     955      2863108 :       if (result
     956      1498394 :           && TREE_CODE (gimple_assign_lhs (use_stmt)) == SSA_NAME
     957      1251739 :           && TREE_CODE (use_rhs) == SSA_NAME
     958      2944517 :           && has_zero_uses (gimple_assign_lhs (use_stmt)))
     959              :         {
     960        81409 :           gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
     961        81409 :           fwprop_invalidate_lattice (gimple_get_lhs (use_stmt));
     962        81409 :           release_defs (use_stmt);
     963        81409 :           gsi_remove (&gsi, true);
     964              :         }
     965      3318841 :     }
     966              : 
     967      3318841 :   return all && has_zero_uses (name);
     968              : }
     969              : 
     970              : 
     971              : /* Helper function for simplify_gimple_switch.  Remove case labels that
     972              :    have values outside the range of the new type.  */
     973              : 
     974              : static void
     975        11802 : simplify_gimple_switch_label_vec (gswitch *stmt, tree index_type,
     976              :                                   vec<std::pair<int, int> > &edges_to_remove)
     977              : {
     978        11802 :   unsigned int branch_num = gimple_switch_num_labels (stmt);
     979        11802 :   auto_vec<tree> labels (branch_num);
     980        11802 :   unsigned int i, len;
     981              : 
     982              :   /* Collect the existing case labels in a VEC, and preprocess it as if
     983              :      we are gimplifying a GENERIC SWITCH_EXPR.  */
     984        75206 :   for (i = 1; i < branch_num; i++)
     985        51602 :     labels.quick_push (gimple_switch_label (stmt, i));
     986        11802 :   preprocess_case_label_vec_for_gimple (labels, index_type, NULL);
     987              : 
     988              :   /* If any labels were removed, replace the existing case labels
     989              :      in the GIMPLE_SWITCH statement with the correct ones.
     990              :      Note that the type updates were done in-place on the case labels,
     991              :      so we only have to replace the case labels in the GIMPLE_SWITCH
     992              :      if the number of labels changed.  */
     993        11802 :   len = labels.length ();
     994        11802 :   if (len < branch_num - 1)
     995              :     {
     996            0 :       bitmap target_blocks;
     997            0 :       edge_iterator ei;
     998            0 :       edge e;
     999              : 
    1000              :       /* Corner case: *all* case labels have been removed as being
    1001              :          out-of-range for INDEX_TYPE.  Push one label and let the
    1002              :          CFG cleanups deal with this further.  */
    1003            0 :       if (len == 0)
    1004              :         {
    1005            0 :           tree label, elt;
    1006              : 
    1007            0 :           label = CASE_LABEL (gimple_switch_default_label (stmt));
    1008            0 :           elt = build_case_label (build_int_cst (index_type, 0), NULL, label);
    1009            0 :           labels.quick_push (elt);
    1010            0 :           len = 1;
    1011              :         }
    1012              : 
    1013            0 :       for (i = 0; i < labels.length (); i++)
    1014            0 :         gimple_switch_set_label (stmt, i + 1, labels[i]);
    1015            0 :       for (i++ ; i < branch_num; i++)
    1016            0 :         gimple_switch_set_label (stmt, i, NULL_TREE);
    1017            0 :       gimple_switch_set_num_labels (stmt, len + 1);
    1018              : 
    1019              :       /* Cleanup any edges that are now dead.  */
    1020            0 :       target_blocks = BITMAP_ALLOC (NULL);
    1021            0 :       for (i = 0; i < gimple_switch_num_labels (stmt); i++)
    1022              :         {
    1023            0 :           tree elt = gimple_switch_label (stmt, i);
    1024            0 :           basic_block target = label_to_block (cfun, CASE_LABEL (elt));
    1025            0 :           bitmap_set_bit (target_blocks, target->index);
    1026              :         }
    1027            0 :       for (ei = ei_start (gimple_bb (stmt)->succs); (e = ei_safe_edge (ei)); )
    1028              :         {
    1029            0 :           if (! bitmap_bit_p (target_blocks, e->dest->index))
    1030            0 :             edges_to_remove.safe_push (std::make_pair (e->src->index,
    1031            0 :                                                        e->dest->index));
    1032              :           else
    1033            0 :             ei_next (&ei);
    1034              :         }
    1035            0 :       BITMAP_FREE (target_blocks);
    1036              :     }
    1037        11802 : }
    1038              : 
    1039              : /* STMT is a SWITCH_EXPR for which we attempt to find equivalent forms of
    1040              :    the condition which we may be able to optimize better.  */
    1041              : 
    1042              : static bool
    1043       111600 : simplify_gimple_switch (gswitch *stmt,
    1044              :                         vec<std::pair<int, int> > &edges_to_remove,
    1045              :                         bitmap simple_dce_worklist)
    1046              : {
    1047              :   /* The optimization that we really care about is removing unnecessary
    1048              :      casts.  That will let us do much better in propagating the inferred
    1049              :      constant at the switch target.  */
    1050       111600 :   tree cond = gimple_switch_index (stmt);
    1051       111600 :   if (TREE_CODE (cond) == SSA_NAME)
    1052              :     {
    1053       111599 :       gimple *def_stmt = SSA_NAME_DEF_STMT (cond);
    1054       111599 :       if (gimple_assign_cast_p (def_stmt))
    1055              :         {
    1056        12779 :           tree def = gimple_assign_rhs1 (def_stmt);
    1057        12779 :           if (TREE_CODE (def) != SSA_NAME)
    1058              :             return false;
    1059              : 
    1060              :           /* If we have an extension or sign-change that preserves the
    1061              :              values we check against then we can copy the source value into
    1062              :              the switch.  */
    1063        12779 :           tree ti = TREE_TYPE (def);
    1064        12779 :           if (INTEGRAL_TYPE_P (ti)
    1065        12779 :               && TYPE_PRECISION (ti) <= TYPE_PRECISION (TREE_TYPE (cond)))
    1066              :             {
    1067        12509 :               size_t n = gimple_switch_num_labels (stmt);
    1068        12509 :               tree min = NULL_TREE, max = NULL_TREE;
    1069        12509 :               if (n > 1)
    1070              :                 {
    1071        12509 :                   min = CASE_LOW (gimple_switch_label (stmt, 1));
    1072        12509 :                   if (CASE_HIGH (gimple_switch_label (stmt, n - 1)))
    1073          186 :                     max = CASE_HIGH (gimple_switch_label (stmt, n - 1));
    1074              :                   else
    1075        12323 :                     max = CASE_LOW (gimple_switch_label (stmt, n - 1));
    1076              :                 }
    1077        12509 :               if ((!min || int_fits_type_p (min, ti))
    1078        12505 :                   && (!max || int_fits_type_p (max, ti)))
    1079              :                 {
    1080        11802 :                   bitmap_set_bit (simple_dce_worklist,
    1081        11802 :                                   SSA_NAME_VERSION (cond));
    1082        11802 :                   gimple_switch_set_index (stmt, def);
    1083        11802 :                   simplify_gimple_switch_label_vec (stmt, ti,
    1084              :                                                     edges_to_remove);
    1085        11802 :                   update_stmt (stmt);
    1086        11802 :                   return true;
    1087              :                 }
    1088              :             }
    1089              :         }
    1090              :     }
    1091              : 
    1092              :   return false;
    1093              : }
    1094              : 
    1095              : /* For pointers p2 and p1 return p2 - p1 if the
    1096              :    difference is known and constant, otherwise return NULL.  */
    1097              : 
    1098              : static tree
    1099         5505 : constant_pointer_difference (tree p1, tree p2)
    1100              : {
    1101         5505 :   int i, j;
    1102              : #define CPD_ITERATIONS 5
    1103         5505 :   tree exps[2][CPD_ITERATIONS];
    1104         5505 :   tree offs[2][CPD_ITERATIONS];
    1105         5505 :   int cnt[2];
    1106              : 
    1107        16515 :   for (i = 0; i < 2; i++)
    1108              :     {
    1109        11010 :       tree p = i ? p1 : p2;
    1110        11010 :       tree off = size_zero_node;
    1111        11010 :       gimple *stmt;
    1112        11010 :       enum tree_code code;
    1113              : 
    1114              :       /* For each of p1 and p2 we need to iterate at least
    1115              :          twice, to handle ADDR_EXPR directly in p1/p2,
    1116              :          SSA_NAME with ADDR_EXPR or POINTER_PLUS_EXPR etc.
    1117              :          on definition's stmt RHS.  Iterate a few extra times.  */
    1118        11010 :       j = 0;
    1119        12800 :       do
    1120              :         {
    1121        12800 :           if (!POINTER_TYPE_P (TREE_TYPE (p)))
    1122              :             break;
    1123        12794 :           if (TREE_CODE (p) == ADDR_EXPR)
    1124              :             {
    1125         9647 :               tree q = TREE_OPERAND (p, 0);
    1126         9647 :               poly_int64 offset;
    1127         9647 :               tree base = get_addr_base_and_unit_offset (q, &offset);
    1128         9647 :               if (base)
    1129              :                 {
    1130         8855 :                   q = base;
    1131         8855 :                   if (maybe_ne (offset, 0))
    1132         3778 :                     off = size_binop (PLUS_EXPR, off, size_int (offset));
    1133              :                 }
    1134         9647 :               if (TREE_CODE (q) == MEM_REF
    1135         9647 :                   && TREE_CODE (TREE_OPERAND (q, 0)) == SSA_NAME)
    1136              :                 {
    1137          155 :                   p = TREE_OPERAND (q, 0);
    1138          155 :                   off = size_binop (PLUS_EXPR, off,
    1139              :                                     wide_int_to_tree (sizetype,
    1140              :                                                       mem_ref_offset (q)));
    1141              :                 }
    1142              :               else
    1143              :                 {
    1144         9492 :                   exps[i][j] = q;
    1145         9492 :                   offs[i][j++] = off;
    1146         9492 :                   break;
    1147              :                 }
    1148              :             }
    1149         3302 :           if (TREE_CODE (p) != SSA_NAME)
    1150              :             break;
    1151         3302 :           exps[i][j] = p;
    1152         3302 :           offs[i][j++] = off;
    1153         3302 :           if (j == CPD_ITERATIONS)
    1154              :             break;
    1155         3302 :           stmt = SSA_NAME_DEF_STMT (p);
    1156         3302 :           if (!is_gimple_assign (stmt) || gimple_assign_lhs (stmt) != p)
    1157              :             break;
    1158         2643 :           code = gimple_assign_rhs_code (stmt);
    1159         2643 :           if (code == POINTER_PLUS_EXPR)
    1160              :             {
    1161         1390 :               if (TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)
    1162              :                 break;
    1163          863 :               off = size_binop (PLUS_EXPR, off, gimple_assign_rhs2 (stmt));
    1164          863 :               p = gimple_assign_rhs1 (stmt);
    1165              :             }
    1166              :           else if (code == ADDR_EXPR || CONVERT_EXPR_CODE_P (code))
    1167          927 :             p = gimple_assign_rhs1 (stmt);
    1168              :           else
    1169              :             break;
    1170              :         }
    1171              :       while (1);
    1172        11010 :       cnt[i] = j;
    1173              :     }
    1174              : 
    1175         7489 :   for (i = 0; i < cnt[0]; i++)
    1176         9744 :     for (j = 0; j < cnt[1]; j++)
    1177         7760 :       if (exps[0][i] == exps[1][j])
    1178         4620 :         return size_binop (MINUS_EXPR, offs[0][i], offs[1][j]);
    1179              : 
    1180              :   return NULL_TREE;
    1181              : }
    1182              : 
    1183              : /* Helper function for optimize_aggr_zeroprop.
    1184              :    Props the zeroing (memset, VAL) that was done in DEST+OFFSET:LEN
    1185              :    (DEFSTMT) into the STMT.  Returns true if the STMT was updated.  */
    1186              : static void
    1187     22640795 : optimize_aggr_zeroprop_1 (gimple *defstmt, gimple *stmt,
    1188              :                           tree dest, poly_int64 offset, tree val,
    1189              :                           poly_offset_int len)
    1190              : {
    1191     22640795 :   tree src2;
    1192     22640795 :   tree len2 = NULL_TREE;
    1193     22640795 :   poly_int64 offset2;
    1194              : 
    1195     22640795 :   if (gimple_call_builtin_p (stmt, BUILT_IN_MEMCPY)
    1196        22324 :       && TREE_CODE (gimple_call_arg (stmt, 1)) == ADDR_EXPR
    1197     22654968 :       && poly_int_tree_p (gimple_call_arg (stmt, 2)))
    1198              :     {
    1199        13056 :       src2 = TREE_OPERAND (gimple_call_arg (stmt, 1), 0);
    1200        13056 :       len2 = gimple_call_arg (stmt, 2);
    1201              :     }
    1202     22627739 :    else if (gimple_assign_load_p (stmt) && gimple_store_p (stmt))
    1203              :      {
    1204      1914741 :         src2 = gimple_assign_rhs1 (stmt);
    1205      1914741 :         len2 = (TREE_CODE (src2) == COMPONENT_REF
    1206      1914741 :                 ? DECL_SIZE_UNIT (TREE_OPERAND (src2, 1))
    1207      1746391 :                 : TYPE_SIZE_UNIT (TREE_TYPE (src2)));
    1208              :         /* Can only handle zero memsets. */
    1209      1914741 :         if (!integer_zerop (val))
    1210     22619537 :           return;
    1211              :      }
    1212              :    else
    1213              :      return;
    1214              : 
    1215      1926674 :   if (len2 == NULL_TREE
    1216      1926674 :       || !poly_int_tree_p (len2))
    1217              :     return;
    1218              : 
    1219      1926674 :   src2 = get_addr_base_and_unit_offset (src2, &offset2);
    1220      1926674 :   if (src2 == NULL_TREE
    1221      1926674 :       || maybe_lt (offset2, offset))
    1222              :     return;
    1223              : 
    1224       877339 :   if (!operand_equal_p (dest, src2, 0))
    1225              :     return;
    1226              : 
    1227              :   /* [ dest + offset, dest + offset + len - 1 ] is set to val.
    1228              :      Make sure that
    1229              :      [ dest + offset2, dest + offset2 + len2 - 1 ] is a subset of that.  */
    1230       131196 :   if (maybe_gt (wi::to_poly_offset (len2) + (offset2 - offset),
    1231              :                 len))
    1232              :     return;
    1233              : 
    1234        21258 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1235              :     {
    1236           32 :       fprintf (dump_file, "Simplified\n  ");
    1237           32 :       print_gimple_stmt (dump_file, stmt, 0, dump_flags);
    1238           32 :       fprintf (dump_file, "after previous\n  ");
    1239           32 :       print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
    1240              :     }
    1241        21258 :   gimple *orig_stmt = stmt;
    1242              :   /* For simplicity, don't change the kind of the stmt,
    1243              :      turn dest = src; into dest = {}; and memcpy (&dest, &src, len);
    1244              :      into memset (&dest, val, len);
    1245              :      In theory we could change dest = src into memset if dest
    1246              :      is addressable (maybe beneficial if val is not 0), or
    1247              :      memcpy (&dest, &src, len) into dest = {} if len is the size
    1248              :      of dest, dest isn't volatile.  */
    1249        21258 :   if (is_gimple_assign (stmt))
    1250              :     {
    1251        21253 :       tree ctor_type = TREE_TYPE (gimple_assign_lhs (stmt));
    1252        21253 :       tree ctor = build_constructor (ctor_type, NULL);
    1253        21253 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    1254        21253 :       gimple_assign_set_rhs_from_tree (&gsi, ctor);
    1255        21253 :       update_stmt (stmt);
    1256        21253 :       statistics_counter_event (cfun, "copy zeroing propagation of aggregate", 1);
    1257              :     }
    1258              :   else /* If stmt is memcpy, transform it into memset.  */
    1259              :     {
    1260            5 :       gcall *call = as_a <gcall *> (stmt);
    1261            5 :       tree fndecl = builtin_decl_implicit (BUILT_IN_MEMSET);
    1262            5 :       gimple_call_set_fndecl (call, fndecl);
    1263            5 :       gimple_call_set_fntype (call, TREE_TYPE (fndecl));
    1264            5 :       gimple_call_set_arg (call, 1, val);
    1265            5 :       update_stmt (stmt);
    1266            5 :       statistics_counter_event (cfun, "memcpy to memset changed", 1);
    1267              :     }
    1268              : 
    1269        21258 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1270              :     {
    1271           32 :       fprintf (dump_file, "into\n  ");
    1272           32 :       print_gimple_stmt (dump_file, stmt, 0, dump_flags);
    1273              :     }
    1274              : 
    1275              :   /* Mark the bb for eh cleanup if needed.  */
    1276        21258 :   if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
    1277            6 :     bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
    1278              : }
    1279              : 
    1280              : /* Optimize
    1281              :    a = {}; // DEST = value ;; LEN(nullptr)
    1282              :    b = a;
    1283              :    into
    1284              :    a = {};
    1285              :    b = {};
    1286              :    Similarly for memset (&a, ..., sizeof (a)); instead of a = {};
    1287              :    and/or memcpy (&b, &a, sizeof (a)); instead of b = a;  */
    1288              : 
    1289              : static void
    1290     31975584 : optimize_aggr_zeroprop (gimple *stmt, bool full_walk)
    1291              : {
    1292     31975584 :   ao_ref read;
    1293     63951168 :   if (gimple_has_volatile_ops (stmt))
    1294     27907779 :     return;
    1295              : 
    1296     30818560 :   tree dest = NULL_TREE;
    1297     30818560 :   tree val = integer_zero_node;
    1298     30818560 :   tree len = NULL_TREE;
    1299     30818560 :   bool can_use_tbba = true;
    1300              : 
    1301     30818560 :   if (gimple_call_builtin_p (stmt, BUILT_IN_MEMSET)
    1302       113245 :       && TREE_CODE (gimple_call_arg (stmt, 0)) == ADDR_EXPR
    1303        58030 :       && TREE_CODE (gimple_call_arg (stmt, 1)) == INTEGER_CST
    1304     30874287 :       && poly_int_tree_p (gimple_call_arg (stmt, 2)))
    1305              :     {
    1306        52960 :       dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
    1307        52960 :       len = gimple_call_arg (stmt, 2);
    1308        52960 :       val = gimple_call_arg (stmt, 1);
    1309        52960 :       ao_ref_init_from_ptr_and_size (&read, gimple_call_arg (stmt, 0), len);
    1310        52960 :       can_use_tbba = false;
    1311              :     }
    1312     30765600 :   else if (gimple_store_p (stmt)
    1313     30705155 :            && gimple_assign_single_p (stmt)
    1314     61470755 :            && TREE_CODE (gimple_assign_rhs1 (stmt)) == STRING_CST)
    1315              :     {
    1316        31000 :       tree str = gimple_assign_rhs1 (stmt);
    1317        31000 :       dest = gimple_assign_lhs (stmt);
    1318        31000 :       ao_ref_init (&read, dest);
    1319              :       /* The string must contain all null char's for now.  */
    1320        67086 :       for (int i = 0; i < TREE_STRING_LENGTH (str); i++)
    1321              :         {
    1322        33699 :           if (TREE_STRING_POINTER (str)[i] != 0)
    1323              :             {
    1324              :               dest = NULL_TREE;
    1325              :               break;
    1326              :             }
    1327              :         }
    1328              :     }
    1329              :   /* A store of integer (scalar, vector or complex) zeros is
    1330              :      a zero store. */
    1331     30734600 :   else if (gimple_store_p (stmt)
    1332     30674155 :            && gimple_assign_single_p (stmt)
    1333     61408755 :            && integer_zerop (gimple_assign_rhs1 (stmt)))
    1334              :     {
    1335      3577264 :       tree rhs = gimple_assign_rhs1 (stmt);
    1336      3577264 :       tree type = TREE_TYPE (rhs);
    1337      3577264 :       dest = gimple_assign_lhs (stmt);
    1338      3577264 :       ao_ref_init (&read, dest);
    1339              :       /* For integral types, the type precision needs to be a multiply of BITS_PER_UNIT. */
    1340      3577264 :       if (INTEGRAL_TYPE_P (type)
    1341      3577264 :           && (TYPE_PRECISION (type) % BITS_PER_UNIT) != 0)
    1342              :         dest = NULL_TREE;
    1343              :     }
    1344     27157336 :   else if (gimple_store_p (stmt)
    1345     27096891 :            && gimple_assign_single_p (stmt)
    1346     27096891 :            && TREE_CODE (gimple_assign_rhs1 (stmt)) == CONSTRUCTOR
    1347     27891958 :            && !gimple_clobber_p (stmt))
    1348              :     {
    1349       734622 :       dest = gimple_assign_lhs (stmt);
    1350       734622 :       ao_ref_init (&read, dest);
    1351              :     }
    1352              : 
    1353      4178584 :   if (dest == NULL_TREE)
    1354              :     return;
    1355              : 
    1356      4149971 :   if (len == NULL_TREE)
    1357      4097011 :     len = (TREE_CODE (dest) == COMPONENT_REF
    1358      4097011 :            ? DECL_SIZE_UNIT (TREE_OPERAND (dest, 1))
    1359      1788033 :            : TYPE_SIZE_UNIT (TREE_TYPE (dest)));
    1360      4097011 :   if (len == NULL_TREE
    1361      4149971 :       || !poly_int_tree_p (len))
    1362              :     return;
    1363              : 
    1364              :   /* Sometimes memset can have no vdef due to invalid declaration of memset (const, etc.).  */
    1365     36207697 :   if (!gimple_vdef (stmt))
    1366              :     return;
    1367              : 
    1368              :   /* This store needs to be on the byte boundary and pointing to an object.  */
    1369      4149947 :   poly_int64 offset;
    1370      4149947 :   tree dest_base = get_addr_base_and_unit_offset (dest, &offset);
    1371      4149947 :   if (dest_base == NULL_TREE)
    1372              :     return;
    1373              : 
    1374              :   /* Setup the worklist.  */
    1375      4067805 :   auto_vec<std::pair<tree, unsigned>> worklist;
    1376      4067805 :   unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 0;
    1377      8135610 :   worklist.safe_push (std::make_pair (gimple_vdef (stmt), limit));
    1378              : 
    1379     28240890 :   while (!worklist.is_empty ())
    1380              :     {
    1381     20105280 :       std::pair<tree, unsigned> top = worklist.pop ();
    1382     20105280 :       tree vdef = top.first;
    1383     20105280 :       limit = top.second;
    1384     20105280 :       gimple *use_stmt;
    1385     20105280 :       imm_use_iterator iter;
    1386     44794284 :       FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
    1387              :         {
    1388              :           /* Handling PHI nodes might not be worth it so don't.  */
    1389     24689004 :           if (is_a <gphi*> (use_stmt))
    1390      2048209 :             continue;
    1391              : 
    1392              :           /* If this statement does not clobber add the vdef stmt to the
    1393              :              worklist.
    1394              :              After hitting the limit, allow clobbers to able to pass through.  */
    1395      2059137 :           if ((limit != 0 || gimple_clobber_p (use_stmt))
    1396     20618171 :               && gimple_vdef (use_stmt)
    1397     40123648 :               && !stmt_may_clobber_ref_p_1 (use_stmt, &read,
    1398              :                                            /* tbaa_p = */ can_use_tbba))
    1399              :               {
    1400     16037475 :                 unsigned new_limit = limit == 0 ? 0 : limit - 1;
    1401     32074950 :                 worklist.safe_push (std::make_pair (gimple_vdef (use_stmt),
    1402              :                                                     new_limit));
    1403              :               }
    1404              : 
    1405     22640795 :            optimize_aggr_zeroprop_1 (stmt, use_stmt, dest_base, offset,
    1406     22640795 :                                      val, wi::to_poly_offset (len));
    1407     20105280 :         }
    1408              :     }
    1409              : 
    1410      4067805 : }
    1411              : 
    1412              : /* Returns the pointer to the base of the object of the
    1413              :    reference EXPR and extracts the information about
    1414              :    the offset of the access, storing it to PBYTESIZE,
    1415              :    PBYTEPOS and PREVERSEP.
    1416              :    If the access is not a byte sized or position is not
    1417              :    on the byte, return NULL.  */
    1418              : static tree
    1419      5366798 : split_core_and_offset_size (tree expr,
    1420              :                             poly_int64 *pbytesize, poly_int64 *pbytepos,
    1421              :                             tree *poffset, int *preversep)
    1422              : {
    1423      5366798 :   tree core;
    1424      5366798 :   machine_mode mode;
    1425      5366798 :   int unsignedp, volatilep;
    1426      5366798 :   poly_int64 bitsize;
    1427      5366798 :   poly_int64 bitpos;
    1428      5366798 :   location_t loc = EXPR_LOCATION (expr);
    1429              : 
    1430      5366798 :   core = get_inner_reference (expr, &bitsize, &bitpos,
    1431              :                               poffset, &mode, &unsignedp, preversep,
    1432              :                               &volatilep);
    1433     10733596 :   if (!multiple_p (bitsize, BITS_PER_UNIT, pbytesize))
    1434              :     return NULL_TREE;
    1435      5366798 :   if (!multiple_p (bitpos, BITS_PER_UNIT, pbytepos))
    1436              :     return NULL_TREE;
    1437              :   /* If we are left with MEM[a + CST] strip that and add it to the
    1438              :      pbytepos and return a. */
    1439      5366798 :   if (TREE_CODE (core) == MEM_REF)
    1440              :     {
    1441      1261545 :       poly_offset_int tem;
    1442      1261545 :       tem = wi::to_poly_offset (TREE_OPERAND (core, 1));
    1443      1261545 :       tem += *pbytepos;
    1444      1261545 :       if (tem.to_shwi (pbytepos))
    1445      1259306 :         return TREE_OPERAND (core, 0);
    1446              :     }
    1447      4107492 :   core = build_fold_addr_expr_loc (loc, core);
    1448      4107492 :   STRIP_NOPS (core);
    1449      4107492 :   return core;
    1450              : }
    1451              : 
    1452              : /* Returns a new src based on the
    1453              :    copy `DEST = SRC` and for the old SRC2.
    1454              :    Returns null if SRC2 is not related to DEST.  */
    1455              : 
    1456              : static tree
    1457      1282157 : new_src_based_on_copy (tree src2, tree dest, tree src)
    1458              : {
    1459              :   /* If the second src is not exactly the same as dest,
    1460              :      try to handle it separately; see it is address/size equivalent.
    1461              :      Handles `a` and `a.b` and `MEM<char[N]>(&a)` which all have
    1462              :      the same size and offsets as address/size equivalent.
    1463              :      This allows copying over a memcpy and also one for copying
    1464              :      where one field is the same size as the whole struct.  */
    1465      1282157 :   if (operand_equal_p (dest, src2))
    1466              :     return src;
    1467              :   /* if both dest and src2 are decls, then we know these 2
    1468              :      accesses can't be the same.  */
    1469       731307 :   if (DECL_P (dest) && DECL_P (src2))
    1470              :     return NULL_TREE;
    1471              :   /* A VCE can't be used with imag/real or BFR so reject them early. */
    1472       383284 :   if (TREE_CODE (src) == IMAGPART_EXPR
    1473       383284 :       || TREE_CODE (src) == REALPART_EXPR
    1474       383284 :       || TREE_CODE (src) == BIT_FIELD_REF)
    1475              :     return NULL_TREE;
    1476       383284 :   tree core1, core2;
    1477       383284 :   poly_int64 bytepos1, bytepos2;
    1478       383284 :   poly_int64 bytesize1, bytesize2;
    1479       383284 :   tree toffset1, toffset2;
    1480       383284 :   int reversep1 = 0;
    1481       383284 :   int reversep2 = 0;
    1482       383284 :   poly_int64 diff = 0;
    1483       383284 :   core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
    1484              :                                           &toffset1, &reversep1);
    1485       383284 :   core2 = split_core_and_offset_size (src2, &bytesize2, &bytepos2,
    1486              :                                           &toffset2, &reversep2);
    1487       383284 :   if (!core1 || !core2)
    1488              :     return NULL_TREE;
    1489       383284 :   if (reversep1 != reversep2)
    1490              :     return NULL_TREE;
    1491              :   /* The sizes of the 2 accesses need to be the same. */
    1492       383284 :   if (!known_eq (bytesize1, bytesize2))
    1493              :     return NULL_TREE;
    1494       175875 :   if (!operand_equal_p (core1, core2, 0))
    1495              :     return NULL_TREE;
    1496              : 
    1497        23612 :   if (toffset1 && toffset2)
    1498              :     {
    1499            2 :       tree type = TREE_TYPE (toffset1);
    1500            2 :       if (type != TREE_TYPE (toffset2))
    1501            0 :         toffset2 = fold_convert (type, toffset2);
    1502              : 
    1503            2 :       tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
    1504            2 :       if (!cst_and_fits_in_hwi (tdiff))
    1505              :         return NULL_TREE;
    1506              : 
    1507            0 :       diff = int_cst_value (tdiff);
    1508            0 :     }
    1509        23610 :   else if (toffset1 || toffset2)
    1510              :     {
    1511              :       /* If only one of the offsets is non-constant, the difference cannot
    1512              :          be a constant.  */
    1513              :       return NULL_TREE;
    1514              :     }
    1515        23578 :   diff += bytepos1 - bytepos2;
    1516              :   /* The offset between the 2 need to be 0. */
    1517        23578 :   if (!known_eq (diff, 0))
    1518              :     return NULL_TREE;
    1519        22740 :   return fold_build1 (VIEW_CONVERT_EXPR,TREE_TYPE (src2), src);
    1520              : }
    1521              : 
    1522              : /* Returns true if SRC and DEST are the same address such that
    1523              :    `SRC == DEST;` is considered a nop. This is more than an
    1524              :    operand_equal_p check as it needs to be similar to
    1525              :    new_src_based_on_copy.  */
    1526              : 
    1527              : static bool
    1528      4553242 : same_for_assignment (tree src, tree dest)
    1529              : {
    1530      4553242 :   if (operand_equal_p (dest, src, 0))
    1531              :     return true;
    1532              :   /* if both dest and src2 are decls, then we know these 2
    1533              :      accesses can't be the same.  */
    1534      4550333 :   if (DECL_P (dest) && DECL_P (src))
    1535              :     return false;
    1536              : 
    1537      2300115 :   tree core1, core2;
    1538      2300115 :   poly_int64 bytepos1, bytepos2;
    1539      2300115 :   poly_int64 bytesize1, bytesize2;
    1540      2300115 :   tree toffset1, toffset2;
    1541      2300115 :   int reversep1 = 0;
    1542      2300115 :   int reversep2 = 0;
    1543      2300115 :   poly_int64 diff = 0;
    1544      2300115 :   core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
    1545              :                                       &toffset1, &reversep1);
    1546      2300115 :   core2 = split_core_and_offset_size (src, &bytesize2, &bytepos2,
    1547              :                                       &toffset2, &reversep2);
    1548      2300115 :   if (!core1 || !core2)
    1549              :     return false;
    1550      2300115 :   if (reversep1 != reversep2)
    1551              :     return false;
    1552              :   /* The sizes of the 2 accesses need to be the same. */
    1553      2300115 :   if (!known_eq (bytesize1, bytesize2))
    1554              :     return false;
    1555      2299206 :   if (!operand_equal_p (core1, core2, 0))
    1556              :     return false;
    1557         6194 :   if (toffset1 && toffset2)
    1558              :     {
    1559          313 :       tree type = TREE_TYPE (toffset1);
    1560          313 :       if (type != TREE_TYPE (toffset2))
    1561            0 :         toffset2 = fold_convert (type, toffset2);
    1562              : 
    1563          313 :       tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
    1564          313 :       if (!cst_and_fits_in_hwi (tdiff))
    1565              :         return false;
    1566              : 
    1567            0 :       diff = int_cst_value (tdiff);
    1568            0 :     }
    1569         5881 :   else if (toffset1 || toffset2)
    1570              :     {
    1571              :       /* If only one of the offsets is non-constant, the difference cannot
    1572              :          be a constant.  */
    1573              :       return false;
    1574              :     }
    1575         5881 :   diff += bytepos1 - bytepos2;
    1576              :   /* The offset between the 2 need to be 0. */
    1577         5881 :   if (!known_eq (diff, 0))
    1578         5635 :     return false;
    1579              :   return true;
    1580              : }
    1581              : 
    1582              : /* Helper function for optimize_agr_copyprop.
    1583              :    For aggregate copies in USE_STMT, see if DEST
    1584              :    is on the lhs of USE_STMT and replace it with SRC. */
    1585              : static void
    1586      1036597 : optimize_agr_copyprop_1 (gimple *stmt, gimple *use_stmt,
    1587              :                          tree dest, tree src)
    1588              : {
    1589      1036597 :   gcc_assert (gimple_assign_load_p (use_stmt)
    1590              :               && gimple_store_p (use_stmt));
    1591      2073194 :   if (gimple_has_volatile_ops (use_stmt))
    1592       621053 :     return;
    1593      1036596 :   tree dest2 = gimple_assign_lhs (use_stmt);
    1594      1036596 :   tree src2 = gimple_assign_rhs1 (use_stmt);
    1595              :   /* If the new store is `src2 = src2;` skip over it. */
    1596      1036596 :   if (same_for_assignment (src2, dest2))
    1597              :     return;
    1598      1036033 :   src = new_src_based_on_copy (src2, dest, src);
    1599      1036033 :   if (!src)
    1600              :     return;
    1601              :   /* For 2 memory references and using a temporary to do the copy,
    1602              :      don't remove the temporary as the 2 memory references might overlap.
    1603              :      Note t does not need to be decl as it could be field.
    1604              :      See PR 22237 for full details.
    1605              :      E.g.
    1606              :      t = *a; #DEST = SRC;
    1607              :      *b = t; #DEST2 = SRC2;
    1608              :      Cannot be convert into
    1609              :      t = *a;
    1610              :      *b = *a;
    1611              :      Though the following is allowed to be done:
    1612              :      t = *a;
    1613              :      *a = t;
    1614              :      And convert it into:
    1615              :      t = *a;
    1616              :      *a = *a;
    1617              :      */
    1618       444545 :   if (!operand_equal_p (dest2, src, 0)
    1619       444545 :       && !DECL_P (dest2) && !DECL_P (src))
    1620              :     {
    1621              :       /* If *a and *b have the same base see if
    1622              :          the offset between the two is greater than
    1623              :          or equal to the size of the type. */
    1624        32379 :       poly_int64 offset1, offset2;
    1625        32379 :       tree len = TYPE_SIZE_UNIT (TREE_TYPE (src));
    1626        32379 :       if (len == NULL_TREE
    1627        32379 :           || !tree_fits_poly_int64_p (len))
    1628        29001 :         return;
    1629        32379 :       tree base1 = get_addr_base_and_unit_offset (dest2, &offset1);
    1630        32379 :       tree base2 = get_addr_base_and_unit_offset (src, &offset2);
    1631        32379 :       poly_int64 size = tree_to_poly_int64 (len);
    1632              :       /* If the bases are 2 different decls,
    1633              :          then there can be no overlapping.  */
    1634        32379 :       if (base1 && base2
    1635        31448 :           && DECL_P (base1) && DECL_P (base2)
    1636         1874 :           && base1 != base2)
    1637              :         ;
    1638              :       /* If we can't figure out the base or the bases are
    1639              :          not equal then fall back to an alignment check.  */
    1640        30734 :       else if (!base1
    1641        30734 :                || !base2
    1642        30734 :                || !operand_equal_p (base1, base2))
    1643              :         {
    1644        30362 :           unsigned int align1 = get_object_alignment (src);
    1645        30362 :           unsigned int align2 = get_object_alignment (dest2);
    1646        30362 :           align1 /= BITS_PER_UNIT;
    1647        30362 :           align2 /= BITS_PER_UNIT;
    1648              :           /* If the alignment of either object is less
    1649              :              than the size then there is a possibility
    1650              :              of overlapping.  */
    1651        30362 :           if (maybe_lt (align1, size)
    1652        30362 :               || maybe_lt (align2, size))
    1653        29001 :             return;
    1654              :         }
    1655              :       /* Make sure [offset1, offset1 + len - 1] does
    1656              :          not overlap with [offset2, offset2 + len - 1],
    1657              :          it is ok if they are at the same location though.  */
    1658          372 :       else if (ranges_maybe_overlap_p (offset1, size, offset2, size)
    1659          372 :           && !known_eq (offset2, offset1))
    1660              :         return;
    1661              :     }
    1662              : 
    1663       415544 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1664              :     {
    1665           11 :       fprintf (dump_file, "Simplified\n  ");
    1666           11 :       print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
    1667           11 :       fprintf (dump_file, "after previous\n  ");
    1668           11 :       print_gimple_stmt (dump_file, stmt, 0, dump_flags);
    1669              :     }
    1670       415544 :   gimple *orig_stmt = use_stmt;
    1671       415544 :   gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
    1672       415544 :   gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (src));
    1673       415544 :   update_stmt (use_stmt);
    1674              : 
    1675       415544 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1676              :     {
    1677           11 :       fprintf (dump_file, "into\n  ");
    1678           11 :       print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
    1679              :     }
    1680       415544 :   if (maybe_clean_or_replace_eh_stmt (orig_stmt, use_stmt))
    1681            0 :     bitmap_set_bit (to_purge, gimple_bb (use_stmt)->index);
    1682       415544 :   statistics_counter_event (cfun, "copy prop for aggregate", 1);
    1683              : }
    1684              : 
    1685              : /* Helper function for optimize_agr_copyprop_1, propagate aggregates
    1686              :    into the arguments of USE_STMT if the argument matches with DEST;
    1687              :    replacing it with SRC.  */
    1688              : static void
    1689       715805 : optimize_agr_copyprop_arg (gimple *defstmt, gcall *call,
    1690              :                            tree dest, tree src)
    1691              : {
    1692       715805 :   bool changed = false;
    1693      2398056 :   for (unsigned arg = 0; arg < gimple_call_num_args (call); arg++)
    1694              :     {
    1695      1682251 :       tree *argptr = gimple_call_arg_ptr (call, arg);
    1696      3170989 :       if (TREE_CODE (*argptr) == SSA_NAME
    1697       960295 :           || is_gimple_min_invariant (*argptr)
    1698      1875764 :           || TYPE_VOLATILE (TREE_TYPE (*argptr)))
    1699      1488738 :         continue;
    1700       193513 :       tree newsrc = new_src_based_on_copy (*argptr, dest, src);
    1701       193513 :       if (!newsrc)
    1702       116953 :         continue;
    1703              : 
    1704        76560 :       if (dump_file && (dump_flags & TDF_DETAILS))
    1705              :         {
    1706            9 :           fprintf (dump_file, "Simplified\n  ");
    1707            9 :           print_gimple_stmt (dump_file, call, 0, dump_flags);
    1708            9 :           fprintf (dump_file, "after previous\n  ");
    1709            9 :           print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
    1710              :         }
    1711        76560 :       *argptr = unshare_expr (newsrc);
    1712        76560 :       changed = true;
    1713        76560 :       if (dump_file && (dump_flags & TDF_DETAILS))
    1714              :         {
    1715            9 :           fprintf (dump_file, "into\n  ");
    1716            9 :           print_gimple_stmt (dump_file, call, 0, dump_flags);
    1717              :         }
    1718              :     }
    1719       715805 :   if (changed)
    1720        76382 :     update_stmt (call);
    1721       715805 : }
    1722              : 
    1723              : /* Helper function for optimize_agr_copyprop, propagate aggregates
    1724              :    into the return stmt USE if the operand of the return matches DEST;
    1725              :    replacing it with SRC.  */
    1726              : static void
    1727       129353 : optimize_agr_copyprop_return (gimple *defstmt, greturn *use,
    1728              :                               tree dest, tree src)
    1729              : {
    1730       129353 :   tree rvalue = gimple_return_retval (use);
    1731       129353 :   if (!rvalue
    1732        83523 :       || TREE_CODE (rvalue) == SSA_NAME
    1733        74577 :       || is_gimple_min_invariant (rvalue)
    1734       203507 :       || TYPE_VOLATILE (TREE_TYPE (rvalue)))
    1735              :     return;
    1736              : 
    1737              :   /* `return <retval>;` is already the best it could be.
    1738              :      Likewise `return *<retval>_N(D)`.  */
    1739        74153 :   if (TREE_CODE (rvalue) == RESULT_DECL
    1740        74153 :       || (TREE_CODE (rvalue) == MEM_REF
    1741            0 :           && TREE_CODE (TREE_OPERAND (rvalue, 0)) == SSA_NAME
    1742            0 :           && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (rvalue, 0)))
    1743              :                == RESULT_DECL))
    1744              :     return;
    1745        52611 :   tree newsrc = new_src_based_on_copy (rvalue, dest, src);
    1746        52611 :   if (!newsrc)
    1747              :     return;
    1748              :   /* Currently only support non-global vars.
    1749              :      See PR 124099 on enumtls not supporting expanding for GIMPLE_RETURN.
    1750              :      FIXME: could support VCEs too?  */
    1751        52485 :   if (!VAR_P (newsrc) || is_global_var (newsrc))
    1752              :     return;
    1753        26685 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1754              :     {
    1755            1 :       fprintf (dump_file, "Simplified\n  ");
    1756            1 :       print_gimple_stmt (dump_file, use, 0, dump_flags);
    1757            1 :       fprintf (dump_file, "after previous\n  ");
    1758            1 :       print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
    1759              :     }
    1760        26685 :   gimple_return_set_retval (use, newsrc);
    1761        26685 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1762              :     {
    1763            1 :       fprintf (dump_file, "into\n  ");
    1764            1 :       print_gimple_stmt (dump_file, use, 0, dump_flags);
    1765              :     }
    1766        26685 :   update_stmt (use);
    1767              : }
    1768              : 
    1769              : /* Optimizes
    1770              :    DEST = SRC;
    1771              :    DEST2 = DEST; # DEST2 = SRC2;
    1772              :    into
    1773              :    DEST = SRC;
    1774              :    DEST2 = SRC;
    1775              :    STMT is the first statement and SRC is the common
    1776              :    between the statements.
    1777              : 
    1778              :    Also optimizes:
    1779              :    DEST = SRC;
    1780              :    call_func(..., DEST, ...);
    1781              :    into:
    1782              :    DEST = SRC;
    1783              :    call_func(..., SRC, ...);
    1784              : 
    1785              : */
    1786              : static void
    1787      3925834 : optimize_agr_copyprop (gimple *stmt)
    1788              : {
    1789      7851668 :   if (gimple_has_volatile_ops (stmt))
    1790       411780 :     return;
    1791              : 
    1792              :   /* Can't prop if the statement could throw.  */
    1793      3924623 :   if (stmt_could_throw_p (cfun, stmt))
    1794              :     return;
    1795              : 
    1796      3516646 :   tree dest = gimple_assign_lhs (stmt);
    1797      3516646 :   tree src = gimple_assign_rhs1 (stmt);
    1798              :   /* If the statement is `src = src;` then ignore it. */
    1799      3516646 :   if (same_for_assignment (dest, src))
    1800              :     return;
    1801              : 
    1802      3514054 :   tree vdef = gimple_vdef (stmt);
    1803      3514054 :   imm_use_iterator iter;
    1804      3514054 :   gimple *use_stmt;
    1805     10133364 :   FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
    1806              :     {
    1807      6619310 :       if (gimple_assign_load_p (use_stmt)
    1808      6619310 :           && gimple_store_p (use_stmt))
    1809      1036597 :         optimize_agr_copyprop_1 (stmt, use_stmt, dest, src);
    1810      5582713 :       else if (is_gimple_call (use_stmt))
    1811       715805 :         optimize_agr_copyprop_arg (stmt, as_a<gcall*>(use_stmt), dest, src);
    1812      4866908 :       else if (is_a<greturn*> (use_stmt))
    1813       129353 :         optimize_agr_copyprop_return (stmt, as_a<greturn*>(use_stmt), dest, src);
    1814      3514054 :     }
    1815              : }
    1816              : 
    1817              : /* Simple DSE of the lhs from a clobber STMT.
    1818              :    This is used mostly to clean up from optimize_agr_copyprop and
    1819              :    to remove (exactly one) extra copy that might later on confuse SRA.
    1820              :    An example is:
    1821              :    ;; write to a and such.
    1822              :    b = a; // This statement is to be removed
    1823              :    b = {CLOBBER};
    1824              :    SRA will totally scalarize b (which means also a) here for the extra copy
    1825              :    which is not something welcomed. So removing the copy will
    1826              :    allow SRA to move the scalarization of a further down or not at all.
    1827              :    */
    1828              : static void
    1829      7511610 : do_simple_agr_dse (gassign *stmt, bool full_walk)
    1830              : {
    1831              :   /* Don't do this while in -Og as we want to keep around the copy
    1832              :      for debuggability.  */
    1833      7511610 :   if (optimize_debug)
    1834      5193770 :     return;
    1835      7508065 :   ao_ref read;
    1836      7508065 :   basic_block bb = gimple_bb (stmt);
    1837      7508065 :   tree lhs = gimple_assign_lhs (stmt);
    1838              :   /* Only handle clobbers of a full decl.  */
    1839      7508065 :   if (!DECL_P (lhs))
    1840              :     return;
    1841      6781565 :   ao_ref_init (&read, lhs);
    1842      6781565 :   tree vuse = gimple_vuse (stmt);
    1843      6781565 :   unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 4;
    1844     17517203 :   while (limit)
    1845              :     {
    1846     17503942 :       gimple *ostmt = SSA_NAME_DEF_STMT (vuse);
    1847              :       /* Don't handle phis, just declare to be done. */
    1848     17503942 :       if (is_a<gphi*>(ostmt) || gimple_nop_p (ostmt))
    1849              :         break;
    1850     15199363 :       basic_block obb = gimple_bb (ostmt);
    1851              :       /* If the clobber is not fully dominating the statement define,
    1852              :          then it is not "simple" to detect if the define is fully clobbered.  */
    1853     15199363 :       if (obb != bb && !dominated_by_p (CDI_DOMINATORS, bb, obb))
    1854      4463725 :         return;
    1855     15199363 :       gimple *use_stmt;
    1856     15199363 :       imm_use_iterator iter;
    1857     46414128 :       FOR_EACH_IMM_USE_STMT (use_stmt, iter, gimple_vdef (ostmt))
    1858              :         {
    1859     18020450 :           basic_block ubb = gimple_bb (use_stmt);
    1860     18020450 :           if (stmt == use_stmt)
    1861      5217095 :             continue;
    1862              :           /* If the use is a clobber for lhs,
    1863              :              then it can be safely skipped; this happens with eh
    1864              :              and sometimes jump threading.  */
    1865     12803355 :           if (gimple_clobber_p (use_stmt)
    1866     12803355 :               && lhs == gimple_assign_lhs (use_stmt))
    1867       176480 :             continue;
    1868              :           /* If the use is a phi and it is single use then check if that single use
    1869              :              is a clobber and lhs is the same.  */
    1870     12626875 :           if (gphi *use_phi = dyn_cast<gphi*>(use_stmt))
    1871              :             {
    1872       378785 :               use_operand_p ou;
    1873       378785 :               gimple *ostmt;
    1874       378785 :               if (single_imm_use (gimple_phi_result (use_phi), &ou, &ostmt)
    1875       316627 :                   && gimple_clobber_p (ostmt)
    1876       631759 :                   && lhs == gimple_assign_lhs (ostmt))
    1877        69906 :                 continue;
    1878              :               /* A phi node will never be dominating the clobber.  */
    1879       308879 :               return;
    1880              :             }
    1881              :           /* The use needs to be dominating the clobber. */
    1882      1502122 :           if ((ubb != bb && !dominated_by_p (CDI_DOMINATORS, bb, ubb))
    1883     13003021 :               || ref_maybe_used_by_stmt_p (use_stmt, &read, false))
    1884              :             return;
    1885              :           /* Count the above alias lookup towards the limit. */
    1886     11024826 :           limit--;
    1887     11024826 :           if (limit == 0)
    1888              :             return;
    1889      2005048 :         }
    1890     13194315 :       vuse = gimple_vuse (ostmt);
    1891              :       /* This is a call with an assignment to the clobber decl,
    1892              :          remove the lhs or the whole stmt if it was pure/const. */
    1893     13194315 :       if (is_a <gcall*>(ostmt)
    1894     13194315 :           && lhs == gimple_call_lhs (ostmt))
    1895              :         {
    1896              :           /* Don't remove stores/statements that are needed for non-call
    1897              :               eh to work.  */
    1898         4050 :           if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
    1899              :             return;
    1900              :           /* If we delete a stmt that could throw, mark the block
    1901              :              in to_purge to cleanup afterwards.  */
    1902         4044 :           if (stmt_could_throw_p (cfun, ostmt))
    1903         1035 :             bitmap_set_bit (to_purge, obb->index);
    1904         4044 :           int flags = gimple_call_flags (ostmt);
    1905         4044 :           if ((flags & (ECF_PURE|ECF_CONST|ECF_NOVOPS))
    1906          205 :               && !(flags & (ECF_LOOPING_CONST_OR_PURE)))
    1907              :             {
    1908          121 :                gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
    1909          121 :                if (dump_file && (dump_flags & TDF_DETAILS))
    1910              :                 {
    1911           14 :                   fprintf (dump_file, "Removing dead call store stmt ");
    1912           14 :                   print_gimple_stmt (dump_file, ostmt, 0);
    1913           14 :                   fprintf (dump_file, "\n");
    1914              :                 }
    1915          121 :               unlink_stmt_vdef (ostmt);
    1916          121 :               release_defs (ostmt);
    1917          121 :               gsi_remove (&gsi, true);
    1918          121 :               statistics_counter_event (cfun, "delete call dead store", 1);
    1919              :               /* Only remove the first store previous statement. */
    1920          121 :               return;
    1921              :             }
    1922              :           /* Make sure we do not remove a return slot we cannot reconstruct
    1923              :              later.  */
    1924         3923 :           if (gimple_call_return_slot_opt_p (as_a <gcall *>(ostmt))
    1925         3923 :               && (TREE_ADDRESSABLE (TREE_TYPE (gimple_call_fntype (ostmt)))
    1926          553 :                   || !poly_int_tree_p
    1927          553 :                       (TYPE_SIZE (TREE_TYPE (gimple_call_fntype (ostmt))))))
    1928              :             return;
    1929          690 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1930              :             {
    1931            6 :               fprintf (dump_file, "Removing lhs of call stmt ");
    1932            6 :               print_gimple_stmt (dump_file, ostmt, 0);
    1933            6 :               fprintf (dump_file, "\n");
    1934              :             }
    1935          690 :           gimple_call_set_lhs (ostmt, NULL_TREE);
    1936          690 :           update_stmt (ostmt);
    1937          690 :           statistics_counter_event (cfun, "removed lhs call", 1);
    1938          690 :           return;
    1939              :         }
    1940              :       /* This an assignment store to the clobbered decl,
    1941              :          then maybe remove it. */
    1942     13190265 :       if (is_a <gassign*>(ostmt)
    1943     11091550 :           && gimple_store_p (ostmt)
    1944     11091550 :           && !gimple_clobber_p (ostmt)
    1945     16494261 :           && lhs == gimple_assign_lhs (ostmt))
    1946              :         {
    1947              :           /* Don't remove stores/statements that are needed for non-call
    1948              :               eh to work.  */
    1949       169220 :           if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
    1950              :             return;
    1951              :           /* If we delete a stmt that could throw, mark the block
    1952              :              in to_purge to cleanup afterwards.  */
    1953       164138 :           if (stmt_could_throw_p (cfun, ostmt))
    1954            0 :             bitmap_set_bit (to_purge, obb->index);
    1955       164138 :           gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
    1956       164138 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1957              :             {
    1958           12 :               fprintf (dump_file, "Removing dead store stmt ");
    1959           12 :               print_gimple_stmt (dump_file, ostmt, 0);
    1960           12 :               fprintf (dump_file, "\n");
    1961              :             }
    1962       164138 :           unlink_stmt_vdef (ostmt);
    1963       164138 :           release_defs (ostmt);
    1964       164138 :           gsi_remove (&gsi, true);
    1965       164138 :           statistics_counter_event (cfun, "delete dead store", 1);
    1966              :           /* Only remove the first store previous statement. */
    1967       164138 :           return;
    1968              :         }
    1969              :       /* If the statement uses or maybe writes to the decl,
    1970              :          then nothing is to be removed. Don't know if the write
    1971              :          to the decl is partial write or a full one so the need
    1972              :          to stop.
    1973              :          e.g.
    1974              :          b.c = a;
    1975              :            Easier to stop here rather than do a full partial
    1976              :            dse of this statement.
    1977              :          b = {CLOBBER}; */
    1978     13021045 :       if (stmt_may_clobber_ref_p_1 (ostmt, &read, false)
    1979     13021045 :           || ref_maybe_used_by_stmt_p (ostmt, &read, false))
    1980              :         return;
    1981     10735638 :       limit--;
    1982              :     }
    1983              : }
    1984              : 
    1985              : /* Optimizes builtin memcmps for small constant sizes.
    1986              :    GSI_P is the GSI for the call. STMT is the call itself.
    1987              :    */
    1988              : 
    1989              : static bool
    1990       467804 : simplify_builtin_memcmp (gimple_stmt_iterator *gsi_p, gcall *stmt)
    1991              : {
    1992              :   /* Make sure memcmp arguments are the correct type.  */
    1993       467804 :   if (gimple_call_num_args (stmt) != 3)
    1994              :     return false;
    1995       467804 :   tree arg1 = gimple_call_arg (stmt, 0);
    1996       467804 :   tree arg2 = gimple_call_arg (stmt, 1);
    1997       467804 :   tree len = gimple_call_arg (stmt, 2);
    1998              : 
    1999       467804 :   if (!POINTER_TYPE_P (TREE_TYPE (arg1)))
    2000              :     return false;
    2001       467804 :   if (!POINTER_TYPE_P (TREE_TYPE (arg2)))
    2002              :     return false;
    2003       467804 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (len)))
    2004              :     return false;
    2005              : 
    2006              :   /* The return value of the memcmp has to be used
    2007              :      equality comparison to zero. */
    2008       467804 :   tree res = gimple_call_lhs (stmt);
    2009              : 
    2010       467804 :   if (!res || !use_in_zero_equality (res))
    2011              :     return false;
    2012              : 
    2013       452008 :   unsigned HOST_WIDE_INT leni;
    2014              : 
    2015       452008 :   if (tree_fits_uhwi_p (len)
    2016       629774 :       && (leni = tree_to_uhwi (len)) <= GET_MODE_SIZE (word_mode)
    2017       532375 :       && pow2p_hwi (leni))
    2018              :     {
    2019        19100 :       leni *= CHAR_TYPE_SIZE;
    2020        19100 :       unsigned align1 = get_pointer_alignment (arg1);
    2021        19100 :       unsigned align2 = get_pointer_alignment (arg2);
    2022        19100 :       unsigned align = MIN (align1, align2);
    2023        19100 :       scalar_int_mode mode;
    2024        38200 :       if (int_mode_for_size (leni, 1).exists (&mode)
    2025        19100 :           && (align >= leni || !targetm.slow_unaligned_access (mode, align)))
    2026              :         {
    2027        19100 :           location_t loc = gimple_location (stmt);
    2028        19100 :           tree type, off;
    2029        19100 :           type = build_nonstandard_integer_type (leni, 1);
    2030        38200 :           gcc_assert (known_eq (GET_MODE_BITSIZE (TYPE_MODE (type)), leni));
    2031        19100 :           tree ptrtype = build_pointer_type_for_mode (char_type_node,
    2032              :                                                       ptr_mode, true);
    2033        19100 :           off = build_int_cst (ptrtype, 0);
    2034              : 
    2035              :           /* Create unaligned types if needed. */
    2036        19100 :           tree type1 = type, type2 = type;
    2037        19100 :           if (TYPE_ALIGN (type1) > align1)
    2038         7784 :             type1 = build_aligned_type (type1, align1);
    2039        19100 :           if (TYPE_ALIGN (type2) > align2)
    2040         8279 :             type2 = build_aligned_type (type2, align2);
    2041              : 
    2042        19100 :           arg1 = build2_loc (loc, MEM_REF, type1, arg1, off);
    2043        19100 :           arg2 = build2_loc (loc, MEM_REF, type2, arg2, off);
    2044        19100 :           tree tem1 = fold_const_aggregate_ref (arg1);
    2045        19100 :           if (tem1)
    2046          242 :             arg1 = tem1;
    2047        19100 :           tree tem2 = fold_const_aggregate_ref (arg2);
    2048        19100 :           if (tem2)
    2049         7408 :             arg2 = tem2;
    2050        19100 :           res = fold_convert_loc (loc, TREE_TYPE (res),
    2051              :                                   fold_build2_loc (loc, NE_EXPR,
    2052              :                                                    boolean_type_node,
    2053              :                                                    arg1, arg2));
    2054        19100 :           gimplify_and_update_call_from_tree (gsi_p, res);
    2055        19100 :           return true;
    2056              :         }
    2057              :     }
    2058              : 
    2059              :   /* Replace memcmp with memcmp_eq if the above fails. */
    2060       432908 :   if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) == BUILT_IN_MEMCMP_EQ)
    2061              :     return false;
    2062       342927 :   if (!fold_before_rtl_expansion_p ())
    2063              :     return false;
    2064        89981 :   gimple_call_set_fndecl (stmt, builtin_decl_explicit (BUILT_IN_MEMCMP_EQ));
    2065        89981 :   update_stmt (stmt);
    2066        89981 :   return true;
    2067              : }
    2068              : 
    2069              : /* Optimizes builtin memchrs for small constant sizes with a const string.
    2070              :    GSI_P is the GSI for the call. STMT is the call itself.
    2071              :    */
    2072              : 
    2073              : static bool
    2074        11536 : simplify_builtin_memchr (gimple_stmt_iterator *gsi_p, gcall *stmt)
    2075              : {
    2076        11536 :   if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
    2077              :     return false;
    2078              : 
    2079        11536 :   if (gimple_call_num_args (stmt) != 3)
    2080              :     return false;
    2081              : 
    2082        11536 :   tree res = gimple_call_lhs (stmt);
    2083        11536 :   if (!res || !use_in_zero_equality (res))
    2084              :     return false;
    2085              : 
    2086         1405 :   tree ptr = gimple_call_arg (stmt, 0);
    2087         1405 :   if (TREE_CODE (ptr) != ADDR_EXPR
    2088         1405 :       || TREE_CODE (TREE_OPERAND (ptr, 0)) != STRING_CST)
    2089              :     return false;
    2090              : 
    2091          398 :   unsigned HOST_WIDE_INT slen
    2092          398 :     = TREE_STRING_LENGTH (TREE_OPERAND (ptr, 0));
    2093              :   /* It must be a non-empty string constant.  */
    2094          398 :   if (slen < 2)
    2095              :     return false;
    2096              : 
    2097              :   /* For -Os, only simplify strings with a single character.  */
    2098          394 :   if (!optimize_bb_for_speed_p (gimple_bb (stmt))
    2099          394 :       && slen > 2)
    2100              :     return false;
    2101              : 
    2102          378 :   tree size = gimple_call_arg (stmt, 2);
    2103              :   /* Size must be a constant which is <= UNITS_PER_WORD and
    2104              :      <= the string length.  */
    2105          378 :   if (!tree_fits_uhwi_p (size))
    2106              :     return false;
    2107              : 
    2108          378 :   unsigned HOST_WIDE_INT sz = tree_to_uhwi (size);
    2109          379 :   if (sz == 0 || sz > UNITS_PER_WORD || sz >= slen)
    2110              :     return false;
    2111              : 
    2112          326 :   tree ch = gimple_call_arg (stmt, 1);
    2113          326 :   location_t loc = gimple_location (stmt);
    2114          326 :   if (!useless_type_conversion_p (char_type_node,
    2115          326 :                                   TREE_TYPE (ch)))
    2116          326 :     ch = fold_convert_loc (loc, char_type_node, ch);
    2117          326 :   const char *p = TREE_STRING_POINTER (TREE_OPERAND (ptr, 0));
    2118          326 :   unsigned int isize = sz;
    2119          326 :   tree *op = XALLOCAVEC (tree, isize);
    2120         1153 :   for (unsigned int i = 0; i < isize; i++)
    2121              :     {
    2122          827 :       op[i] = build_int_cst (char_type_node, p[i]);
    2123          827 :       op[i] = fold_build2_loc (loc, EQ_EXPR, boolean_type_node,
    2124              :                                op[i], ch);
    2125              :     }
    2126          827 :   for (unsigned int i = isize - 1; i >= 1; i--)
    2127          501 :     op[i - 1] = fold_convert_loc (loc, boolean_type_node,
    2128              :                                   fold_build2_loc (loc,
    2129              :                                                    BIT_IOR_EXPR,
    2130              :                                                    boolean_type_node,
    2131          501 :                                                    op[i - 1],
    2132          501 :                                                    op[i]));
    2133          326 :   res = fold_convert_loc (loc, TREE_TYPE (res), op[0]);
    2134          326 :   gimplify_and_update_call_from_tree (gsi_p, res);
    2135          326 :   return true;
    2136              : }
    2137              : 
    2138              : /* *GSI_P is a GIMPLE_CALL to a builtin function.
    2139              :    Optimize
    2140              :    memcpy (p, "abcd", 4); // STMT1
    2141              :    memset (p + 4, ' ', 3); // STMT2
    2142              :    into
    2143              :    memcpy (p, "abcd   ", 7);
    2144              :    call if the latter can be stored by pieces during expansion.
    2145              : */
    2146              : 
    2147              : static bool
    2148       113405 : simplify_builtin_memcpy_memset (gimple_stmt_iterator *gsi_p, gcall *stmt2)
    2149              : {
    2150       113405 :   if (gimple_call_num_args (stmt2) != 3
    2151       113405 :       || gimple_call_lhs (stmt2)
    2152              :       || CHAR_BIT != 8
    2153       113405 :       || BITS_PER_UNIT != 8)
    2154              :     return false;
    2155              : 
    2156       107228 :   tree vuse = gimple_vuse (stmt2);
    2157       107228 :   if (vuse == NULL)
    2158              :     return false;
    2159       107212 :   gimple *stmt1 = SSA_NAME_DEF_STMT (vuse);
    2160              : 
    2161       107212 :   tree callee1;
    2162       107212 :   tree ptr1, src1, str1, off1, len1, lhs1;
    2163       107212 :   tree ptr2 = gimple_call_arg (stmt2, 0);
    2164       107212 :   tree val2 = gimple_call_arg (stmt2, 1);
    2165       107212 :   tree len2 = gimple_call_arg (stmt2, 2);
    2166       107212 :   tree diff, vdef, new_str_cst;
    2167       107212 :   gimple *use_stmt;
    2168       107212 :   unsigned int ptr1_align;
    2169       107212 :   unsigned HOST_WIDE_INT src_len;
    2170       107212 :   char *src_buf;
    2171       107212 :   use_operand_p use_p;
    2172              : 
    2173       107212 :   if (!tree_fits_shwi_p (val2)
    2174       102725 :       || !tree_fits_uhwi_p (len2)
    2175       172414 :       || compare_tree_int (len2, 1024) == 1)
    2176              :     return false;
    2177              : 
    2178        59623 :   if (is_gimple_call (stmt1))
    2179              :     {
    2180              :       /* If first stmt is a call, it needs to be memcpy
    2181              :          or mempcpy, with string literal as second argument and
    2182              :          constant length.  */
    2183        31015 :       callee1 = gimple_call_fndecl (stmt1);
    2184        31015 :       if (callee1 == NULL_TREE
    2185        30899 :           || !fndecl_built_in_p (callee1, BUILT_IN_NORMAL)
    2186        58109 :           || gimple_call_num_args (stmt1) != 3)
    2187              :         return false;
    2188        25743 :       if (DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMCPY
    2189        25743 :           && DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMPCPY)
    2190              :         return false;
    2191        11196 :       ptr1 = gimple_call_arg (stmt1, 0);
    2192        11196 :       src1 = gimple_call_arg (stmt1, 1);
    2193        11196 :       len1 = gimple_call_arg (stmt1, 2);
    2194        11196 :       lhs1 = gimple_call_lhs (stmt1);
    2195        11196 :       if (!tree_fits_uhwi_p (len1))
    2196              :         return false;
    2197        11109 :       str1 = string_constant (src1, &off1, NULL, NULL);
    2198        11109 :       if (str1 == NULL_TREE)
    2199              :         return false;
    2200         5222 :       if (!tree_fits_uhwi_p (off1)
    2201         5222 :           || compare_tree_int (off1, TREE_STRING_LENGTH (str1) - 1) > 0
    2202         5222 :           || compare_tree_int (len1, TREE_STRING_LENGTH (str1)
    2203         5222 :                                      - tree_to_uhwi (off1)) > 0
    2204         5155 :           || TREE_CODE (TREE_TYPE (str1)) != ARRAY_TYPE
    2205        15532 :           || TYPE_MODE (TREE_TYPE (TREE_TYPE (str1)))
    2206         5155 :              != TYPE_MODE (char_type_node))
    2207              :         return false;
    2208              :     }
    2209        28608 :   else if (gimple_assign_single_p (stmt1))
    2210              :     {
    2211              :       /* Otherwise look for length 1 memcpy optimized into
    2212              :          assignment.  */
    2213        17434 :       ptr1 = gimple_assign_lhs (stmt1);
    2214        17434 :       src1 = gimple_assign_rhs1 (stmt1);
    2215        17434 :       if (TREE_CODE (ptr1) != MEM_REF
    2216         3484 :           || TYPE_MODE (TREE_TYPE (ptr1)) != TYPE_MODE (char_type_node)
    2217        18423 :           || !tree_fits_shwi_p (src1))
    2218              :         return false;
    2219          343 :       ptr1 = build_fold_addr_expr (ptr1);
    2220          343 :       STRIP_USELESS_TYPE_CONVERSION (ptr1);
    2221          343 :       callee1 = NULL_TREE;
    2222          343 :       len1 = size_one_node;
    2223          343 :       lhs1 = NULL_TREE;
    2224          343 :       off1 = size_zero_node;
    2225          343 :       str1 = NULL_TREE;
    2226              :     }
    2227              :   else
    2228              :     return false;
    2229              : 
    2230         5498 :   diff = constant_pointer_difference (ptr1, ptr2);
    2231         5498 :   if (diff == NULL && lhs1 != NULL)
    2232              :     {
    2233            7 :       diff = constant_pointer_difference (lhs1, ptr2);
    2234            7 :       if (DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
    2235            7 :           && diff != NULL)
    2236            7 :         diff = size_binop (PLUS_EXPR, diff,
    2237              :                            fold_convert (sizetype, len1));
    2238              :     }
    2239              :   /* If the difference between the second and first destination pointer
    2240              :      is not constant, or is bigger than memcpy length, bail out.  */
    2241         5498 :   if (diff == NULL
    2242         4620 :       || !tree_fits_uhwi_p (diff)
    2243         4620 :       || tree_int_cst_lt (len1, diff)
    2244         9862 :       || compare_tree_int (diff, 1024) == 1)
    2245              :     return false;
    2246              : 
    2247              :   /* Use maximum of difference plus memset length and memcpy length
    2248              :      as the new memcpy length, if it is too big, bail out.  */
    2249         4364 :   src_len = tree_to_uhwi (diff);
    2250         4364 :   src_len += tree_to_uhwi (len2);
    2251         4364 :   if (src_len < tree_to_uhwi (len1))
    2252              :     src_len = tree_to_uhwi (len1);
    2253         4364 :   if (src_len > 1024)
    2254              :     return false;
    2255              : 
    2256              :   /* If mempcpy value is used elsewhere, bail out, as mempcpy
    2257              :      with bigger length will return different result.  */
    2258         4364 :   if (lhs1 != NULL_TREE
    2259           64 :       && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
    2260         4371 :       && (TREE_CODE (lhs1) != SSA_NAME
    2261            7 :           || !single_imm_use (lhs1, &use_p, &use_stmt)
    2262            7 :           || use_stmt != stmt2))
    2263              :     return false;
    2264              : 
    2265              :   /* If anything reads memory in between memcpy and memset
    2266              :      call, the modified memcpy call might change it.  */
    2267         4364 :   vdef = gimple_vdef (stmt1);
    2268         4364 :   if (vdef != NULL
    2269         4364 :       && (!single_imm_use (vdef, &use_p, &use_stmt)
    2270         3645 :           || use_stmt != stmt2))
    2271              :     return false;
    2272              : 
    2273         3645 :   ptr1_align = get_pointer_alignment (ptr1);
    2274              :   /* Construct the new source string literal.  */
    2275         3645 :   src_buf = XALLOCAVEC (char, src_len + 1);
    2276         3645 :   if (callee1)
    2277         3479 :     memcpy (src_buf,
    2278         3479 :             TREE_STRING_POINTER (str1) + tree_to_uhwi (off1),
    2279              :             tree_to_uhwi (len1));
    2280              :   else
    2281          166 :     src_buf[0] = tree_to_shwi (src1);
    2282         3645 :   memset (src_buf + tree_to_uhwi (diff),
    2283         3645 :           tree_to_shwi (val2), tree_to_uhwi (len2));
    2284         3645 :   src_buf[src_len] = '\0';
    2285              :   /* Neither builtin_strncpy_read_str nor builtin_memcpy_read_str
    2286              :      handle embedded '\0's.  */
    2287         3645 :   if (strlen (src_buf) != src_len)
    2288              :     return false;
    2289         3551 :   rtl_profile_for_bb (gimple_bb (stmt2));
    2290              :   /* If the new memcpy wouldn't be emitted by storing the literal
    2291              :      by pieces, this optimization might enlarge .rodata too much,
    2292              :      as commonly used string literals couldn't be shared any
    2293              :      longer.  */
    2294         3551 :   if (!can_store_by_pieces (src_len,
    2295              :                             builtin_strncpy_read_str,
    2296              :                             src_buf, ptr1_align, false))
    2297              :     return false;
    2298              : 
    2299         2669 :   new_str_cst = build_string_literal (src_len, src_buf);
    2300         2669 :   if (callee1)
    2301              :     {
    2302              :       /* If STMT1 is a mem{,p}cpy call, adjust it and remove
    2303              :          memset call.  */
    2304         2541 :       if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
    2305            7 :         gimple_call_set_lhs (stmt1, NULL_TREE);
    2306         2541 :       gimple_call_set_arg (stmt1, 1, new_str_cst);
    2307         2541 :       gimple_call_set_arg (stmt1, 2,
    2308         2541 :                            build_int_cst (TREE_TYPE (len1), src_len));
    2309         2541 :       update_stmt (stmt1);
    2310         2541 :       unlink_stmt_vdef (stmt2);
    2311         2541 :       gsi_replace (gsi_p, gimple_build_nop (), false);
    2312         2541 :       fwprop_invalidate_lattice (gimple_get_lhs (stmt2));
    2313         2541 :       release_defs (stmt2);
    2314         2541 :       if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
    2315              :         {
    2316            7 :           fwprop_invalidate_lattice (lhs1);
    2317            7 :           release_ssa_name (lhs1);
    2318              :         }
    2319              :       return true;
    2320              :     }
    2321              :   else
    2322              :     {
    2323              :       /* Otherwise, if STMT1 is length 1 memcpy optimized into
    2324              :          assignment, remove STMT1 and change memset call into
    2325              :          memcpy call.  */
    2326          128 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt1);
    2327              : 
    2328          128 :       if (!is_gimple_val (ptr1))
    2329           12 :         ptr1 = force_gimple_operand_gsi (gsi_p, ptr1, true, NULL_TREE,
    2330              :                                          true, GSI_SAME_STMT);
    2331          128 :       tree fndecl = builtin_decl_explicit (BUILT_IN_MEMCPY);
    2332          128 :       gimple_call_set_fndecl (stmt2, fndecl);
    2333          128 :       gimple_call_set_fntype (stmt2,
    2334          128 :                               TREE_TYPE (fndecl));
    2335          128 :       gimple_call_set_arg (stmt2, 0, ptr1);
    2336          128 :       gimple_call_set_arg (stmt2, 1, new_str_cst);
    2337          128 :       gimple_call_set_arg (stmt2, 2,
    2338          128 :                            build_int_cst (TREE_TYPE (len2), src_len));
    2339          128 :       unlink_stmt_vdef (stmt1);
    2340          128 :       gsi_remove (&gsi, true);
    2341          128 :       fwprop_invalidate_lattice (gimple_get_lhs (stmt1));
    2342          128 :       release_defs (stmt1);
    2343          128 :       update_stmt (stmt2);
    2344          128 :       return false;
    2345              :     }
    2346              : }
    2347              : 
    2348              : 
    2349              : /* Try to optimize out __builtin_stack_restore.  Optimize it out
    2350              :    if there is another __builtin_stack_restore in the same basic
    2351              :    block and no calls or ASM_EXPRs are in between, or if this block's
    2352              :    only outgoing edge is to EXIT_BLOCK and there are no calls or
    2353              :    ASM_EXPRs after this __builtin_stack_restore.
    2354              :    Note restore right before a noreturn function is not needed.
    2355              :    And skip some cheap calls that will most likely become an instruction.
    2356              :    Restoring the stack before a call is important to be able to keep
    2357              :    stack usage down so that call does not run out of stack.  */
    2358              : 
    2359              : 
    2360              : static bool
    2361        10629 : optimize_stack_restore (gimple_stmt_iterator *gsi, gimple *call)
    2362              : {
    2363        10629 :   if (!fold_before_rtl_expansion_p ())
    2364              :     return false;
    2365         2592 :   tree callee;
    2366         2592 :   gimple *stmt;
    2367              : 
    2368         2592 :   basic_block bb = gsi_bb (*gsi);
    2369              : 
    2370         2592 :   if (gimple_call_num_args (call) != 1
    2371         2592 :       || TREE_CODE (gimple_call_arg (call, 0)) != SSA_NAME
    2372         5184 :       || !POINTER_TYPE_P (TREE_TYPE (gimple_call_arg (call, 0))))
    2373              :     return false;
    2374              : 
    2375         2592 :   gimple_stmt_iterator i = *gsi;
    2376         6492 :   for (gsi_next (&i); !gsi_end_p (i); gsi_next (&i))
    2377              :     {
    2378         4369 :       stmt = gsi_stmt (i);
    2379         4369 :       if (is_a<gasm*> (stmt))
    2380              :         return false;
    2381         4368 :       gcall *call = dyn_cast<gcall*>(stmt);
    2382         4368 :       if (!call)
    2383         3685 :         continue;
    2384              : 
    2385              :       /* We can remove the restore in front of noreturn
    2386              :          calls.  Since the restore will happen either
    2387              :          via an unwind/longjmp or not at all. */
    2388          683 :       if (gimple_call_noreturn_p (call))
    2389              :         break;
    2390              : 
    2391              :       /* Internal calls are ok, to bypass
    2392              :          check first since fndecl will be null. */
    2393          667 :       if (gimple_call_internal_p (call))
    2394            1 :         continue;
    2395              : 
    2396          666 :       callee = gimple_call_fndecl (call);
    2397              :       /* Non-builtin calls are not ok. */
    2398          666 :       if (!callee
    2399          666 :           || !fndecl_built_in_p (callee))
    2400              :         return false;
    2401              : 
    2402              :       /* Do not remove stack updates before strub leave.  */
    2403          586 :       if (fndecl_built_in_p (callee, BUILT_IN___STRUB_LEAVE)
    2404              :           /* Alloca calls are not ok either. */
    2405          586 :           || fndecl_builtin_alloc_p (callee))
    2406              :         return false;
    2407              : 
    2408          371 :       if (fndecl_built_in_p (callee, BUILT_IN_STACK_RESTORE))
    2409           57 :         goto second_stack_restore;
    2410              : 
    2411              :       /* If not a simple or inexpensive builtin, then it is not ok either. */
    2412          314 :       if (!is_simple_builtin (callee)
    2413          314 :           && !is_inexpensive_builtin (callee))
    2414              :         return false;
    2415              :     }
    2416              : 
    2417              :   /* Allow one successor of the exit block, or zero successors.  */
    2418         2139 :   switch (EDGE_COUNT (bb->succs))
    2419              :     {
    2420              :     case 0:
    2421              :       break;
    2422         2049 :     case 1:
    2423         2049 :       if (single_succ_edge (bb)->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
    2424              :         return false;
    2425              :       break;
    2426              :     default:
    2427              :       return false;
    2428              :     }
    2429         1779 :  second_stack_restore:
    2430              : 
    2431              :   /* If there's exactly one use, then zap the call to __builtin_stack_save.
    2432              :      If there are multiple uses, then the last one should remove the call.
    2433              :      In any case, whether the call to __builtin_stack_save can be removed
    2434              :      or not is irrelevant to removing the call to __builtin_stack_restore.  */
    2435         1779 :   if (has_single_use (gimple_call_arg (call, 0)))
    2436              :     {
    2437         1607 :       gimple *stack_save = SSA_NAME_DEF_STMT (gimple_call_arg (call, 0));
    2438         1607 :       if (is_gimple_call (stack_save))
    2439              :         {
    2440         1605 :           callee = gimple_call_fndecl (stack_save);
    2441         1605 :           if (callee && fndecl_built_in_p (callee, BUILT_IN_STACK_SAVE))
    2442              :             {
    2443         1605 :               gimple_stmt_iterator stack_save_gsi;
    2444         1605 :               tree rhs;
    2445              : 
    2446         1605 :               stack_save_gsi = gsi_for_stmt (stack_save);
    2447         1605 :               rhs = build_int_cst (TREE_TYPE (gimple_call_arg (call, 0)), 0);
    2448         1605 :               replace_call_with_value (&stack_save_gsi, rhs);
    2449              :             }
    2450              :         }
    2451              :     }
    2452              : 
    2453              :   /* No effect, so the statement will be deleted.  */
    2454         1779 :   replace_call_with_value (gsi, NULL_TREE);
    2455         1779 :   return true;
    2456              : }
    2457              : 
    2458              : /* Optimizes strlen (s) ==/!= 0 to *s ==/!= 0. */
    2459              : static bool
    2460        61082 : optimize_strlen_comp (gimple_stmt_iterator *gsi, gimple *call)
    2461              : {
    2462        61082 :   if (!fold_before_rtl_expansion_p ())
    2463              :     return false;
    2464              : 
    2465        13145 :   tree lhs = gimple_call_lhs (call);
    2466        13145 :   if (lhs == NULL_TREE || use_in_zero_equality (lhs, true) == NULL)
    2467              :     return false;
    2468              : 
    2469              :   /* The string passed to strlen. */
    2470          108 :   tree ptr = gimple_call_arg (call, 0);
    2471              : 
    2472              :   /* Dereference the string. */
    2473          108 :   tree deref = fold_build2 (MEM_REF, char_type_node, ptr,
    2474              :                             build_zero_cst (ptr_type_node));
    2475              : 
    2476              :   /* Perform a type conversion. */
    2477          108 :   deref = fold_convert_loc (gimple_location (call),
    2478          108 :                             TREE_TYPE (lhs),
    2479              :                             deref);
    2480              : 
    2481              :   /* Replace the original call to strlen with the dereference we just built. */
    2482          108 :   gimplify_and_update_call_from_tree (gsi, deref);
    2483              : 
    2484          108 :   return true;
    2485              : }
    2486              : 
    2487              : /* *GSI_P is a GIMPLE_CALL to a builtin function.
    2488              :    Optimize
    2489              :    memcpy (p, "abcd", 4);
    2490              :    memset (p + 4, ' ', 3);
    2491              :    into
    2492              :    memcpy (p, "abcd   ", 7);
    2493              :    call if the latter can be stored by pieces during expansion.
    2494              : 
    2495              :    Optimize
    2496              :    memchr ("abcd", a, 4) == 0;
    2497              :    or
    2498              :    memchr ("abcd", a, 4) != 0;
    2499              :    to
    2500              :    (a == 'a' || a == 'b' || a == 'c' || a == 'd') == 0
    2501              :    or
    2502              :    (a == 'a' || a == 'b' || a == 'c' || a == 'd') != 0
    2503              : 
    2504              :    Also canonicalize __atomic_fetch_op (p, x, y) op x
    2505              :    to __atomic_op_fetch (p, x, y) or
    2506              :    __atomic_op_fetch (p, x, y) iop x
    2507              :    to __atomic_fetch_op (p, x, y) when possible (also __sync).  */
    2508              : 
    2509              : static bool
    2510      6319213 : simplify_builtin_call (gimple_stmt_iterator *gsi_p, tree callee2, bool full_walk)
    2511              : {
    2512      6319213 :   gimple *stmt2 = gsi_stmt (*gsi_p);
    2513      6319213 :   enum built_in_function other_atomic = END_BUILTINS;
    2514      6319213 :   enum tree_code atomic_op = ERROR_MARK;
    2515              : 
    2516      6319213 :   switch (DECL_FUNCTION_CODE (callee2))
    2517              :     {
    2518        61082 :     case BUILT_IN_STRLEN:
    2519        61082 :       return optimize_strlen_comp (gsi_p, as_a<gcall*>(stmt2));
    2520        10629 :     case BUILT_IN_STACK_RESTORE:
    2521        10629 :       return optimize_stack_restore (gsi_p, as_a<gcall*>(stmt2));
    2522       467804 :     case BUILT_IN_MEMCMP:
    2523       467804 :     case BUILT_IN_MEMCMP_EQ:
    2524       467804 :       return simplify_builtin_memcmp (gsi_p, as_a<gcall*>(stmt2));
    2525        11536 :     case BUILT_IN_MEMCHR:
    2526        11536 :       return simplify_builtin_memchr (gsi_p, as_a<gcall*>(stmt2));
    2527              : 
    2528       113405 :     case BUILT_IN_MEMSET:
    2529       113405 :       if (gimple_call_num_args (stmt2) == 3)
    2530              :         {
    2531              :           /* Try to prop the zeroing/value of the memset to memcpy
    2532              :              if the dest is an address and the value is a constant. */
    2533       113405 :           optimize_aggr_zeroprop (stmt2, full_walk);
    2534              :         }
    2535       113405 :       return simplify_builtin_memcpy_memset (gsi_p, as_a<gcall*>(stmt2));
    2536              : 
    2537              :  #define CASE_ATOMIC(NAME, OTHER, OP) \
    2538              :     case BUILT_IN_##NAME##_1:                                           \
    2539              :     case BUILT_IN_##NAME##_2:                                           \
    2540              :     case BUILT_IN_##NAME##_4:                                           \
    2541              :     case BUILT_IN_##NAME##_8:                                           \
    2542              :     case BUILT_IN_##NAME##_16:                                          \
    2543              :       atomic_op = OP;                                                   \
    2544              :       other_atomic                                                      \
    2545              :         = (enum built_in_function) (BUILT_IN_##OTHER##_1                \
    2546              :                                     + (DECL_FUNCTION_CODE (callee2)     \
    2547              :                                        - BUILT_IN_##NAME##_1));         \
    2548              :       goto handle_atomic_fetch_op;
    2549              : 
    2550        49277 :     CASE_ATOMIC (ATOMIC_FETCH_ADD, ATOMIC_ADD_FETCH, PLUS_EXPR)
    2551         7261 :     CASE_ATOMIC (ATOMIC_FETCH_SUB, ATOMIC_SUB_FETCH, MINUS_EXPR)
    2552         3084 :     CASE_ATOMIC (ATOMIC_FETCH_AND, ATOMIC_AND_FETCH, BIT_AND_EXPR)
    2553         3063 :     CASE_ATOMIC (ATOMIC_FETCH_XOR, ATOMIC_XOR_FETCH, BIT_XOR_EXPR)
    2554         4202 :     CASE_ATOMIC (ATOMIC_FETCH_OR, ATOMIC_OR_FETCH, BIT_IOR_EXPR)
    2555              : 
    2556         2373 :     CASE_ATOMIC (SYNC_FETCH_AND_ADD, SYNC_ADD_AND_FETCH, PLUS_EXPR)
    2557         2012 :     CASE_ATOMIC (SYNC_FETCH_AND_SUB, SYNC_SUB_AND_FETCH, MINUS_EXPR)
    2558         1876 :     CASE_ATOMIC (SYNC_FETCH_AND_AND, SYNC_AND_AND_FETCH, BIT_AND_EXPR)
    2559         2144 :     CASE_ATOMIC (SYNC_FETCH_AND_XOR, SYNC_XOR_AND_FETCH, BIT_XOR_EXPR)
    2560         1987 :     CASE_ATOMIC (SYNC_FETCH_AND_OR, SYNC_OR_AND_FETCH, BIT_IOR_EXPR)
    2561              : 
    2562        14461 :     CASE_ATOMIC (ATOMIC_ADD_FETCH, ATOMIC_FETCH_ADD, MINUS_EXPR)
    2563         8580 :     CASE_ATOMIC (ATOMIC_SUB_FETCH, ATOMIC_FETCH_SUB, PLUS_EXPR)
    2564         2428 :     CASE_ATOMIC (ATOMIC_XOR_FETCH, ATOMIC_FETCH_XOR, BIT_XOR_EXPR)
    2565              : 
    2566          854 :     CASE_ATOMIC (SYNC_ADD_AND_FETCH, SYNC_FETCH_AND_ADD, MINUS_EXPR)
    2567          740 :     CASE_ATOMIC (SYNC_SUB_AND_FETCH, SYNC_FETCH_AND_SUB, PLUS_EXPR)
    2568          800 :     CASE_ATOMIC (SYNC_XOR_AND_FETCH, SYNC_FETCH_AND_XOR, BIT_XOR_EXPR)
    2569              : 
    2570              : #undef CASE_ATOMIC
    2571              : 
    2572       105142 :     handle_atomic_fetch_op:
    2573       105142 :       if (gimple_call_num_args (stmt2) >= 2 && gimple_call_lhs (stmt2))
    2574              :         {
    2575        61074 :           tree lhs2 = gimple_call_lhs (stmt2), lhsc = lhs2;
    2576        61074 :           tree arg = gimple_call_arg (stmt2, 1);
    2577        61074 :           gimple *use_stmt, *cast_stmt = NULL;
    2578        61074 :           use_operand_p use_p;
    2579        61074 :           tree ndecl = builtin_decl_explicit (other_atomic);
    2580              : 
    2581        61074 :           if (ndecl == NULL_TREE || !single_imm_use (lhs2, &use_p, &use_stmt))
    2582              :             break;
    2583              : 
    2584        59957 :           if (gimple_assign_cast_p (use_stmt))
    2585              :             {
    2586        31573 :               cast_stmt = use_stmt;
    2587        31573 :               lhsc = gimple_assign_lhs (cast_stmt);
    2588        31573 :               if (lhsc == NULL_TREE
    2589        31573 :                   || !INTEGRAL_TYPE_P (TREE_TYPE (lhsc))
    2590        31022 :                   || (TYPE_PRECISION (TREE_TYPE (lhsc))
    2591        31022 :                       != TYPE_PRECISION (TREE_TYPE (lhs2)))
    2592        60977 :                   || !single_imm_use (lhsc, &use_p, &use_stmt))
    2593              :                 {
    2594         2697 :                   use_stmt = cast_stmt;
    2595         2697 :                   cast_stmt = NULL;
    2596         2697 :                   lhsc = lhs2;
    2597              :                 }
    2598              :             }
    2599              : 
    2600        59957 :           bool ok = false;
    2601        59957 :           tree oarg = NULL_TREE;
    2602        59957 :           enum tree_code ccode = ERROR_MARK;
    2603        59957 :           tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
    2604        59957 :           if (is_gimple_assign (use_stmt)
    2605        59957 :               && gimple_assign_rhs_code (use_stmt) == atomic_op)
    2606              :             {
    2607         1416 :               if (gimple_assign_rhs1 (use_stmt) == lhsc)
    2608         1016 :                 oarg = gimple_assign_rhs2 (use_stmt);
    2609          400 :               else if (atomic_op != MINUS_EXPR)
    2610              :                 oarg = gimple_assign_rhs1 (use_stmt);
    2611              :             }
    2612        58541 :           else if (atomic_op == MINUS_EXPR
    2613        13423 :                    && is_gimple_assign (use_stmt)
    2614         3744 :                    && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR
    2615          199 :                    && TREE_CODE (arg) == INTEGER_CST
    2616        58740 :                    && (TREE_CODE (gimple_assign_rhs2 (use_stmt))
    2617              :                        == INTEGER_CST))
    2618              :             {
    2619          183 :               tree a = fold_convert (TREE_TYPE (lhs2), arg);
    2620          183 :               tree o = fold_convert (TREE_TYPE (lhs2),
    2621              :                                      gimple_assign_rhs2 (use_stmt));
    2622          183 :               if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
    2623              :                 ok = true;
    2624              :             }
    2625        58358 :           else if (atomic_op == BIT_AND_EXPR || atomic_op == BIT_IOR_EXPR)
    2626              :             ;
    2627        52708 :           else if (gimple_code (use_stmt) == GIMPLE_COND)
    2628              :             {
    2629        19730 :               ccode = gimple_cond_code (use_stmt);
    2630        19730 :               crhs1 = gimple_cond_lhs (use_stmt);
    2631        19730 :               crhs2 = gimple_cond_rhs (use_stmt);
    2632              :             }
    2633        32978 :           else if (is_gimple_assign (use_stmt))
    2634              :             {
    2635         9853 :               if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
    2636              :                 {
    2637         4185 :                   ccode = gimple_assign_rhs_code (use_stmt);
    2638         4185 :                   crhs1 = gimple_assign_rhs1 (use_stmt);
    2639         4185 :                   crhs2 = gimple_assign_rhs2 (use_stmt);
    2640              :                 }
    2641              :             }
    2642        24931 :           if (ccode == EQ_EXPR || ccode == NE_EXPR)
    2643              :             {
    2644              :               /* Deal with x - y == 0 or x ^ y == 0
    2645              :                  being optimized into x == y and x + cst == 0
    2646              :                  into x == -cst.  */
    2647        22717 :               tree o = NULL_TREE;
    2648        22717 :               if (crhs1 == lhsc)
    2649              :                 o = crhs2;
    2650          133 :               else if (crhs2 == lhsc)
    2651          133 :                 o = crhs1;
    2652        22717 :               if (o && atomic_op != PLUS_EXPR)
    2653              :                 oarg = o;
    2654        10271 :               else if (o
    2655        10271 :                        && TREE_CODE (o) == INTEGER_CST
    2656        10271 :                        && TREE_CODE (arg) == INTEGER_CST)
    2657              :                 {
    2658         9561 :                   tree a = fold_convert (TREE_TYPE (lhs2), arg);
    2659         9561 :                   o = fold_convert (TREE_TYPE (lhs2), o);
    2660         9561 :                   if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
    2661        59957 :                     ok = true;
    2662              :                 }
    2663              :             }
    2664        59957 :           if (oarg && !ok)
    2665              :             {
    2666        13862 :               if (operand_equal_p (arg, oarg, 0))
    2667              :                 ok = true;
    2668        12521 :               else if (TREE_CODE (arg) == SSA_NAME
    2669         2215 :                        && TREE_CODE (oarg) == SSA_NAME)
    2670              :                 {
    2671          745 :                   tree oarg2 = oarg;
    2672          745 :                   if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (oarg)))
    2673              :                     {
    2674          104 :                       gimple *g = SSA_NAME_DEF_STMT (oarg);
    2675          104 :                       oarg2 = gimple_assign_rhs1 (g);
    2676          104 :                       if (TREE_CODE (oarg2) != SSA_NAME
    2677          104 :                           || !INTEGRAL_TYPE_P (TREE_TYPE (oarg2))
    2678          208 :                           || (TYPE_PRECISION (TREE_TYPE (oarg2))
    2679          104 :                               != TYPE_PRECISION (TREE_TYPE (oarg))))
    2680              :                         oarg2 = oarg;
    2681              :                     }
    2682          745 :                   if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (arg)))
    2683              :                     {
    2684          544 :                       gimple *g = SSA_NAME_DEF_STMT (arg);
    2685          544 :                       tree rhs1 = gimple_assign_rhs1 (g);
    2686              :                       /* Handle e.g.
    2687              :                          x.0_1 = (long unsigned int) x_4(D);
    2688              :                          _2 = __atomic_fetch_add_8 (&vlong, x.0_1, 0);
    2689              :                          _3 = (long int) _2;
    2690              :                          _7 = x_4(D) + _3;  */
    2691          544 :                       if (rhs1 == oarg || rhs1 == oarg2)
    2692              :                         ok = true;
    2693              :                       /* Handle e.g.
    2694              :                          x.18_1 = (short unsigned int) x_5(D);
    2695              :                          _2 = (int) x.18_1;
    2696              :                          _3 = __atomic_fetch_xor_2 (&vshort, _2, 0);
    2697              :                          _4 = (short int) _3;
    2698              :                          _8 = x_5(D) ^ _4;
    2699              :                          This happens only for char/short.  */
    2700          160 :                       else if (TREE_CODE (rhs1) == SSA_NAME
    2701          160 :                                && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    2702          320 :                                && (TYPE_PRECISION (TREE_TYPE (rhs1))
    2703          160 :                                    == TYPE_PRECISION (TREE_TYPE (lhs2))))
    2704              :                         {
    2705          160 :                           g = SSA_NAME_DEF_STMT (rhs1);
    2706          160 :                           if (gimple_assign_cast_p (g)
    2707          160 :                               && (gimple_assign_rhs1 (g) == oarg
    2708            0 :                                   || gimple_assign_rhs1 (g) == oarg2))
    2709              :                             ok = true;
    2710              :                         }
    2711              :                     }
    2712          745 :                   if (!ok && arg == oarg2)
    2713              :                     /* Handle e.g.
    2714              :                        _1 = __sync_fetch_and_add_4 (&v, x_5(D));
    2715              :                        _2 = (int) _1;
    2716              :                        x.0_3 = (int) x_5(D);
    2717              :                        _7 = _2 + x.0_3;  */
    2718              :                     ok = true;
    2719              :                 }
    2720              :             }
    2721              : 
    2722        58616 :           if (ok)
    2723              :             {
    2724         2622 :               tree new_lhs = make_ssa_name (TREE_TYPE (lhs2));
    2725         2622 :               gimple_call_set_lhs (stmt2, new_lhs);
    2726         2622 :               gimple_call_set_fndecl (stmt2, ndecl);
    2727         2622 :               gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
    2728         2622 :               if (ccode == ERROR_MARK)
    2729         2000 :                 gimple_assign_set_rhs_with_ops (&gsi, cast_stmt
    2730              :                                                 ? NOP_EXPR : SSA_NAME,
    2731              :                                                 new_lhs);
    2732              :               else
    2733              :                 {
    2734         1399 :                   crhs1 = new_lhs;
    2735         1399 :                   crhs2 = build_zero_cst (TREE_TYPE (lhs2));
    2736         1399 :                   if (gimple_code (use_stmt) == GIMPLE_COND)
    2737              :                     {
    2738         1054 :                       gcond *cond_stmt = as_a <gcond *> (use_stmt);
    2739         1054 :                       gimple_cond_set_lhs (cond_stmt, crhs1);
    2740         1054 :                       gimple_cond_set_rhs (cond_stmt, crhs2);
    2741              :                     }
    2742          345 :                   else if (gimple_assign_rhs_class (use_stmt)
    2743              :                            == GIMPLE_BINARY_RHS)
    2744              :                     {
    2745          345 :                       gimple_assign_set_rhs1 (use_stmt, crhs1);
    2746          345 :                       gimple_assign_set_rhs2 (use_stmt, crhs2);
    2747              :                     }
    2748              :                 }
    2749         2622 :               update_stmt (use_stmt);
    2750         2622 :               if (atomic_op != BIT_AND_EXPR
    2751         2622 :                   && atomic_op != BIT_IOR_EXPR
    2752         2622 :                   && !stmt_ends_bb_p (stmt2))
    2753              :                 {
    2754              :                   /* For the benefit of debug stmts, emit stmt(s) to set
    2755              :                      lhs2 to the value it had from the new builtin.
    2756              :                      E.g. if it was previously:
    2757              :                      lhs2 = __atomic_fetch_add_8 (ptr, arg, 0);
    2758              :                      emit:
    2759              :                      new_lhs = __atomic_add_fetch_8 (ptr, arg, 0);
    2760              :                      lhs2 = new_lhs - arg;
    2761              :                      We also keep cast_stmt if any in the IL for
    2762              :                      the same reasons.
    2763              :                      These stmts will be DCEd later and proper debug info
    2764              :                      will be emitted.
    2765              :                      This is only possible for reversible operations
    2766              :                      (+/-/^) and without -fnon-call-exceptions.  */
    2767         2281 :                   gsi = gsi_for_stmt (stmt2);
    2768         2281 :                   tree type = TREE_TYPE (lhs2);
    2769         2281 :                   if (TREE_CODE (arg) == INTEGER_CST)
    2770         1699 :                     arg = fold_convert (type, arg);
    2771          582 :                   else if (!useless_type_conversion_p (type, TREE_TYPE (arg)))
    2772              :                     {
    2773            0 :                       tree narg = make_ssa_name (type);
    2774            0 :                       gimple *g = gimple_build_assign (narg, NOP_EXPR, arg);
    2775            0 :                       gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    2776            0 :                       arg = narg;
    2777              :                     }
    2778         2281 :                   enum tree_code rcode;
    2779         2281 :                   switch (atomic_op)
    2780              :                     {
    2781              :                     case PLUS_EXPR: rcode = MINUS_EXPR; break;
    2782          733 :                     case MINUS_EXPR: rcode = PLUS_EXPR; break;
    2783          498 :                     case BIT_XOR_EXPR: rcode = atomic_op; break;
    2784              :                     default: gcc_unreachable ();
    2785              :                     }
    2786         2281 :                   gimple *g = gimple_build_assign (lhs2, rcode, new_lhs, arg);
    2787         2281 :                   gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    2788         2281 :                   update_stmt (stmt2);
    2789              :                 }
    2790              :               else
    2791              :                 {
    2792              :                   /* For e.g.
    2793              :                      lhs2 = __atomic_fetch_or_8 (ptr, arg, 0);
    2794              :                      after we change it to
    2795              :                      new_lhs = __atomic_or_fetch_8 (ptr, arg, 0);
    2796              :                      there is no way to find out the lhs2 value (i.e.
    2797              :                      what the atomic memory contained before the operation),
    2798              :                      values of some bits are lost.  We have checked earlier
    2799              :                      that we don't have any non-debug users except for what
    2800              :                      we are already changing, so we need to reset the
    2801              :                      debug stmts and remove the cast_stmt if any.  */
    2802          341 :                   imm_use_iterator iter;
    2803          676 :                   FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs2)
    2804          335 :                     if (use_stmt != cast_stmt)
    2805              :                       {
    2806          168 :                         gcc_assert (is_gimple_debug (use_stmt));
    2807          168 :                         gimple_debug_bind_reset_value (use_stmt);
    2808          168 :                         update_stmt (use_stmt);
    2809          341 :                       }
    2810          341 :                   if (cast_stmt)
    2811              :                     {
    2812          167 :                       gsi = gsi_for_stmt (cast_stmt);
    2813          167 :                       gsi_remove (&gsi, true);
    2814              :                     }
    2815          341 :                   update_stmt (stmt2);
    2816          341 :                   release_ssa_name (lhs2);
    2817              :                 }
    2818              :             }
    2819              :         }
    2820              :       break;
    2821              : 
    2822              :     default:
    2823              :       break;
    2824              :     }
    2825              :   return false;
    2826              : }
    2827              : 
    2828              : /* Given a ssa_name in NAME see if it was defined by an assignment and
    2829              :    set CODE to be the code and ARG1 to the first operand on the rhs and ARG2
    2830              :    to the second operand on the rhs. */
    2831              : 
    2832              : static inline void
    2833     17727550 : defcodefor_name (tree name, enum tree_code *code, tree *arg1, tree *arg2)
    2834              : {
    2835     17727550 :   gimple *def;
    2836     17727550 :   enum tree_code code1;
    2837     17727550 :   tree arg11;
    2838     17727550 :   tree arg21;
    2839     17727550 :   tree arg31;
    2840     17727550 :   enum gimple_rhs_class grhs_class;
    2841              : 
    2842     17727550 :   code1 = TREE_CODE (name);
    2843     17727550 :   arg11 = name;
    2844     17727550 :   arg21 = NULL_TREE;
    2845     17727550 :   arg31 = NULL_TREE;
    2846     17727550 :   grhs_class = get_gimple_rhs_class (code1);
    2847              : 
    2848     17727550 :   if (code1 == SSA_NAME)
    2849              :     {
    2850     11786005 :       def = SSA_NAME_DEF_STMT (name);
    2851              : 
    2852     11786005 :       if (def && is_gimple_assign (def)
    2853     19088784 :           && can_propagate_from (def))
    2854              :         {
    2855      5010454 :           code1 = gimple_assign_rhs_code (def);
    2856      5010454 :           arg11 = gimple_assign_rhs1 (def);
    2857      5010454 :           arg21 = gimple_assign_rhs2 (def);
    2858      5010454 :           arg31 = gimple_assign_rhs3 (def);
    2859              :         }
    2860              :     }
    2861      5941545 :   else if (grhs_class != GIMPLE_SINGLE_RHS)
    2862            0 :     code1 = ERROR_MARK;
    2863              : 
    2864     17727550 :   *code = code1;
    2865     17727550 :   *arg1 = arg11;
    2866     17727550 :   if (arg2)
    2867     17710291 :     *arg2 = arg21;
    2868     17727550 :   if (arg31)
    2869         2715 :     *code = ERROR_MARK;
    2870     17727550 : }
    2871              : 
    2872              : 
    2873              : /* Recognize rotation patterns.  Return true if a transformation
    2874              :    applied, otherwise return false.
    2875              : 
    2876              :    We are looking for X with unsigned type T with bitsize B, OP being
    2877              :    +, | or ^, some type T2 wider than T.  For:
    2878              :    (X << CNT1) OP (X >> CNT2)                               iff CNT1 + CNT2 == B
    2879              :    ((T) ((T2) X << CNT1)) OP ((T) ((T2) X >> CNT2)) iff CNT1 + CNT2 == B
    2880              : 
    2881              :    transform these into:
    2882              :    X r<< CNT1
    2883              : 
    2884              :    Or for:
    2885              :    (X << Y) OP (X >> (B - Y))
    2886              :    (X << (int) Y) OP (X >> (int) (B - Y))
    2887              :    ((T) ((T2) X << Y)) OP ((T) ((T2) X >> (B - Y)))
    2888              :    ((T) ((T2) X << (int) Y)) OP ((T) ((T2) X >> (int) (B - Y)))
    2889              :    (X << Y) | (X >> ((-Y) & (B - 1)))
    2890              :    (X << (int) Y) | (X >> (int) ((-Y) & (B - 1)))
    2891              :    ((T) ((T2) X << Y)) | ((T) ((T2) X >> ((-Y) & (B - 1))))
    2892              :    ((T) ((T2) X << (int) Y)) | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
    2893              : 
    2894              :    transform these into (last 2 only if ranger can prove Y < B
    2895              :    or Y = N * B):
    2896              :    X r<< Y
    2897              :    or
    2898              :    X r<< (& & (B - 1))
    2899              :    The latter for the forms with T2 wider than T if ranger can't prove Y < B.
    2900              : 
    2901              :    Or for:
    2902              :    (X << (Y & (B - 1))) | (X >> ((-Y) & (B - 1)))
    2903              :    (X << (int) (Y & (B - 1))) | (X >> (int) ((-Y) & (B - 1)))
    2904              :    ((T) ((T2) X << (Y & (B - 1)))) | ((T) ((T2) X >> ((-Y) & (B - 1))))
    2905              :    ((T) ((T2) X << (int) (Y & (B - 1)))) \
    2906              :      | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
    2907              : 
    2908              :    transform these into:
    2909              :    X r<< (Y & (B - 1))
    2910              : 
    2911              :    Note, in the patterns with T2 type, the type of OP operands
    2912              :    might be even a signed type, but should have precision B.
    2913              :    Expressions with & (B - 1) should be recognized only if B is
    2914              :    a power of 2.  */
    2915              : 
    2916              : static bool
    2917     10383032 : simplify_rotate (gimple_stmt_iterator *gsi)
    2918              : {
    2919     10383032 :   gimple *stmt = gsi_stmt (*gsi);
    2920     10383032 :   tree arg[2], rtype, rotcnt = NULL_TREE;
    2921     10383032 :   tree def_arg1[2], def_arg2[2];
    2922     10383032 :   enum tree_code def_code[2];
    2923     10383032 :   tree lhs;
    2924     10383032 :   int i;
    2925     10383032 :   bool swapped_p = false;
    2926     10383032 :   gimple *g;
    2927     10383032 :   gimple *def_arg_stmt[2] = { NULL, NULL };
    2928     10383032 :   int wider_prec = 0;
    2929     10383032 :   bool add_masking = false;
    2930              : 
    2931     10383032 :   arg[0] = gimple_assign_rhs1 (stmt);
    2932     10383032 :   arg[1] = gimple_assign_rhs2 (stmt);
    2933     10383032 :   rtype = TREE_TYPE (arg[0]);
    2934              : 
    2935              :   /* Only create rotates in complete modes.  Other cases are not
    2936              :      expanded properly.  */
    2937     10383032 :   if (!INTEGRAL_TYPE_P (rtype)
    2938     10383032 :       || !type_has_mode_precision_p (rtype))
    2939              :     return false;
    2940              : 
    2941     26446941 :   for (i = 0; i < 2; i++)
    2942              :     {
    2943     17631294 :       defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
    2944     17631294 :       if (TREE_CODE (arg[i]) == SSA_NAME)
    2945     11689749 :         def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
    2946              :     }
    2947              : 
    2948              :   /* Look through narrowing (or same precision) conversions.  */
    2949      7851182 :   if (CONVERT_EXPR_CODE_P (def_code[0])
    2950       964465 :       && CONVERT_EXPR_CODE_P (def_code[1])
    2951       141932 :       && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[0]))
    2952       117639 :       && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[1]))
    2953       109670 :       && TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
    2954       109670 :          == TYPE_PRECISION (TREE_TYPE (def_arg1[1]))
    2955        63478 :       && TYPE_PRECISION (TREE_TYPE (def_arg1[0])) >= TYPE_PRECISION (rtype)
    2956        44404 :       && has_single_use (arg[0])
    2957      8847607 :       && has_single_use (arg[1]))
    2958              :     {
    2959        26806 :       wider_prec = TYPE_PRECISION (TREE_TYPE (def_arg1[0]));
    2960        80418 :       for (i = 0; i < 2; i++)
    2961              :         {
    2962        53612 :           arg[i] = def_arg1[i];
    2963        53612 :           defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
    2964        53612 :           if (TREE_CODE (arg[i]) == SSA_NAME)
    2965        53612 :             def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
    2966              :         }
    2967              :     }
    2968              :   else
    2969              :     {
    2970              :       /* Handle signed rotate; the RSHIFT_EXPR has to be done
    2971              :          in unsigned type but LSHIFT_EXPR could be signed.  */
    2972      8788841 :       i = (def_code[0] == LSHIFT_EXPR || def_code[0] == RSHIFT_EXPR);
    2973      7833427 :       if (CONVERT_EXPR_CODE_P (def_code[i])
    2974       955414 :           && (def_code[1 - i] == LSHIFT_EXPR || def_code[1 - i] == RSHIFT_EXPR)
    2975        32375 :           && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[i]))
    2976        31109 :           && TYPE_PRECISION (rtype) == TYPE_PRECISION (TREE_TYPE (def_arg1[i]))
    2977      8793820 :           && has_single_use (arg[i]))
    2978              :         {
    2979         2120 :           arg[i] = def_arg1[i];
    2980         2120 :           defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
    2981         2120 :           if (TREE_CODE (arg[i]) == SSA_NAME)
    2982         2120 :             def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
    2983              :         }
    2984              :     }
    2985              : 
    2986              :   /* One operand has to be LSHIFT_EXPR and one RSHIFT_EXPR.  */
    2987      9005437 :   for (i = 0; i < 2; i++)
    2988      8980105 :     if (def_code[i] != LSHIFT_EXPR && def_code[i] != RSHIFT_EXPR)
    2989              :       return false;
    2990       231122 :     else if (!has_single_use (arg[i]))
    2991              :       return false;
    2992        25332 :   if (def_code[0] == def_code[1])
    2993              :     return false;
    2994              : 
    2995              :   /* If we've looked through narrowing conversions before, look through
    2996              :      widening conversions from unsigned type with the same precision
    2997              :      as rtype here.  */
    2998        20968 :   if (TYPE_PRECISION (TREE_TYPE (def_arg1[0])) != TYPE_PRECISION (rtype))
    2999        19345 :     for (i = 0; i < 2; i++)
    3000              :       {
    3001        12897 :         tree tem;
    3002        12897 :         enum tree_code code;
    3003        12897 :         defcodefor_name (def_arg1[i], &code, &tem, NULL);
    3004            1 :         if (!CONVERT_EXPR_CODE_P (code)
    3005        12896 :             || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
    3006        25793 :             || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
    3007            1 :           return false;
    3008        12896 :         def_arg1[i] = tem;
    3009              :       }
    3010              :   /* Both shifts have to use the same first operand.  */
    3011        20967 :   if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
    3012        33263 :       || !types_compatible_p (TREE_TYPE (def_arg1[0]),
    3013        12296 :                               TREE_TYPE (def_arg1[1])))
    3014              :     {
    3015         8671 :       if ((TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
    3016         8671 :            != TYPE_PRECISION (TREE_TYPE (def_arg1[1])))
    3017         8671 :           || (TYPE_UNSIGNED (TREE_TYPE (def_arg1[0]))
    3018         8671 :               == TYPE_UNSIGNED (TREE_TYPE (def_arg1[1]))))
    3019         8647 :         return false;
    3020              : 
    3021              :       /* Handle signed rotate; the RSHIFT_EXPR has to be done
    3022              :          in unsigned type but LSHIFT_EXPR could be signed.  */
    3023          543 :       i = def_code[0] != RSHIFT_EXPR;
    3024          543 :       if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[i])))
    3025              :         return false;
    3026              : 
    3027          506 :       tree tem;
    3028          506 :       enum tree_code code;
    3029          506 :       defcodefor_name (def_arg1[i], &code, &tem, NULL);
    3030          303 :       if (!CONVERT_EXPR_CODE_P (code)
    3031          203 :           || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
    3032          709 :           || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
    3033              :         return false;
    3034          194 :       def_arg1[i] = tem;
    3035          194 :       if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
    3036          218 :           || !types_compatible_p (TREE_TYPE (def_arg1[0]),
    3037           24 :                                   TREE_TYPE (def_arg1[1])))
    3038              :         return false;
    3039              :     }
    3040        12296 :   else if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[0])))
    3041              :     return false;
    3042              : 
    3043              :   /* CNT1 + CNT2 == B case above.  */
    3044        11065 :   if (tree_fits_uhwi_p (def_arg2[0])
    3045         1568 :       && tree_fits_uhwi_p (def_arg2[1])
    3046        11065 :       && tree_to_uhwi (def_arg2[0])
    3047         1568 :          + tree_to_uhwi (def_arg2[1]) == TYPE_PRECISION (rtype))
    3048              :     rotcnt = def_arg2[0];
    3049        10142 :   else if (TREE_CODE (def_arg2[0]) != SSA_NAME
    3050         9497 :            || TREE_CODE (def_arg2[1]) != SSA_NAME)
    3051              :     return false;
    3052              :   else
    3053              :     {
    3054         9497 :       tree cdef_arg1[2], cdef_arg2[2], def_arg2_alt[2];
    3055         9497 :       enum tree_code cdef_code[2];
    3056         9497 :       gimple *def_arg_alt_stmt[2] = { NULL, NULL };
    3057         9497 :       int check_range = 0;
    3058         9497 :       gimple *check_range_stmt = NULL;
    3059              :       /* Look through conversion of the shift count argument.
    3060              :          The C/C++ FE cast any shift count argument to integer_type_node.
    3061              :          The only problem might be if the shift count type maximum value
    3062              :          is equal or smaller than number of bits in rtype.  */
    3063        28491 :       for (i = 0; i < 2; i++)
    3064              :         {
    3065        18994 :           def_arg2_alt[i] = def_arg2[i];
    3066        18994 :           defcodefor_name (def_arg2[i], &cdef_code[i],
    3067              :                            &cdef_arg1[i], &cdef_arg2[i]);
    3068        14723 :           if (CONVERT_EXPR_CODE_P (cdef_code[i])
    3069         4271 :               && INTEGRAL_TYPE_P (TREE_TYPE (cdef_arg1[i]))
    3070         4271 :               && TYPE_PRECISION (TREE_TYPE (cdef_arg1[i]))
    3071         4271 :                  > floor_log2 (TYPE_PRECISION (rtype))
    3072        23265 :               && type_has_mode_precision_p (TREE_TYPE (cdef_arg1[i])))
    3073              :             {
    3074         4271 :               def_arg2_alt[i] = cdef_arg1[i];
    3075         4271 :               if (TREE_CODE (def_arg2[i]) == SSA_NAME)
    3076         4271 :                 def_arg_alt_stmt[i] = SSA_NAME_DEF_STMT (def_arg2[i]);
    3077         4271 :               defcodefor_name (def_arg2_alt[i], &cdef_code[i],
    3078              :                                &cdef_arg1[i], &cdef_arg2[i]);
    3079              :             }
    3080              :           else
    3081        14723 :             def_arg_alt_stmt[i] = def_arg_stmt[i];
    3082              :         }
    3083        25816 :       for (i = 0; i < 2; i++)
    3084              :         /* Check for one shift count being Y and the other B - Y,
    3085              :            with optional casts.  */
    3086        18643 :         if (cdef_code[i] == MINUS_EXPR
    3087          862 :             && tree_fits_shwi_p (cdef_arg1[i])
    3088          862 :             && tree_to_shwi (cdef_arg1[i]) == TYPE_PRECISION (rtype)
    3089        19465 :             && TREE_CODE (cdef_arg2[i]) == SSA_NAME)
    3090              :           {
    3091          822 :             tree tem;
    3092          822 :             enum tree_code code;
    3093              : 
    3094          822 :             if (cdef_arg2[i] == def_arg2[1 - i]
    3095          472 :                 || cdef_arg2[i] == def_arg2_alt[1 - i])
    3096              :               {
    3097          350 :                 rotcnt = cdef_arg2[i];
    3098          350 :                 check_range = -1;
    3099          350 :                 if (cdef_arg2[i] == def_arg2[1 - i])
    3100          350 :                   check_range_stmt = def_arg_stmt[1 - i];
    3101              :                 else
    3102            0 :                   check_range_stmt = def_arg_alt_stmt[1 - i];
    3103          806 :                 break;
    3104              :               }
    3105          472 :             defcodefor_name (cdef_arg2[i], &code, &tem, NULL);
    3106           16 :             if (CONVERT_EXPR_CODE_P (code)
    3107          456 :                 && INTEGRAL_TYPE_P (TREE_TYPE (tem))
    3108          456 :                 && TYPE_PRECISION (TREE_TYPE (tem))
    3109          456 :                    > floor_log2 (TYPE_PRECISION (rtype))
    3110          456 :                 && type_has_mode_precision_p (TREE_TYPE (tem))
    3111          928 :                 && (tem == def_arg2[1 - i]
    3112          288 :                     || tem == def_arg2_alt[1 - i]))
    3113              :               {
    3114          456 :                 rotcnt = tem;
    3115          456 :                 check_range = -1;
    3116          456 :                 if (tem == def_arg2[1 - i])
    3117          168 :                   check_range_stmt = def_arg_stmt[1 - i];
    3118              :                 else
    3119          288 :                   check_range_stmt = def_arg_alt_stmt[1 - i];
    3120              :                 break;
    3121              :               }
    3122              :           }
    3123              :         /* The above sequence isn't safe for Y being 0,
    3124              :            because then one of the shifts triggers undefined behavior.
    3125              :            This alternative is safe even for rotation count of 0.
    3126              :            One shift count is Y and the other (-Y) & (B - 1).
    3127              :            Or one shift count is Y & (B - 1) and the other (-Y) & (B - 1).  */
    3128        17821 :         else if (cdef_code[i] == BIT_AND_EXPR
    3129        28752 :                  && pow2p_hwi (TYPE_PRECISION (rtype))
    3130        12433 :                  && tree_fits_shwi_p (cdef_arg2[i])
    3131        24866 :                  && tree_to_shwi (cdef_arg2[i])
    3132        12433 :                     == TYPE_PRECISION (rtype) - 1
    3133        12373 :                  && TREE_CODE (cdef_arg1[i]) == SSA_NAME
    3134        30194 :                  && gimple_assign_rhs_code (stmt) == BIT_IOR_EXPR)
    3135              :           {
    3136         2313 :             tree tem;
    3137         2313 :             enum tree_code code;
    3138              : 
    3139         2313 :             defcodefor_name (cdef_arg1[i], &code, &tem, NULL);
    3140         2114 :             if (CONVERT_EXPR_CODE_P (code)
    3141          199 :                 && INTEGRAL_TYPE_P (TREE_TYPE (tem))
    3142          199 :                 && TYPE_PRECISION (TREE_TYPE (tem))
    3143          199 :                    > floor_log2 (TYPE_PRECISION (rtype))
    3144         2512 :                 && type_has_mode_precision_p (TREE_TYPE (tem)))
    3145          199 :               defcodefor_name (tem, &code, &tem, NULL);
    3146              : 
    3147         2313 :             if (code == NEGATE_EXPR)
    3148              :               {
    3149         1532 :                 if (tem == def_arg2[1 - i] || tem == def_arg2_alt[1 - i])
    3150              :                   {
    3151          853 :                     rotcnt = tem;
    3152          853 :                     check_range = 1;
    3153          853 :                     if (tem == def_arg2[1 - i])
    3154          845 :                       check_range_stmt = def_arg_stmt[1 - i];
    3155              :                     else
    3156            8 :                       check_range_stmt = def_arg_alt_stmt[1 - i];
    3157         1518 :                     break;
    3158              :                   }
    3159          679 :                 tree tem2;
    3160          679 :                 defcodefor_name (tem, &code, &tem2, NULL);
    3161          237 :                 if (CONVERT_EXPR_CODE_P (code)
    3162          442 :                     && INTEGRAL_TYPE_P (TREE_TYPE (tem2))
    3163          442 :                     && TYPE_PRECISION (TREE_TYPE (tem2))
    3164          442 :                        > floor_log2 (TYPE_PRECISION (rtype))
    3165         1121 :                     && type_has_mode_precision_p (TREE_TYPE (tem2)))
    3166              :                   {
    3167          442 :                     if (tem2 == def_arg2[1 - i]
    3168          442 :                         || tem2 == def_arg2_alt[1 - i])
    3169              :                       {
    3170          228 :                         rotcnt = tem2;
    3171          228 :                         check_range = 1;
    3172          228 :                         if (tem2 == def_arg2[1 - i])
    3173            0 :                           check_range_stmt = def_arg_stmt[1 - i];
    3174              :                         else
    3175          228 :                           check_range_stmt = def_arg_alt_stmt[1 - i];
    3176              :                         break;
    3177              :                       }
    3178              :                   }
    3179              :                 else
    3180          237 :                   tem2 = NULL_TREE;
    3181              : 
    3182          451 :                 if (cdef_code[1 - i] == BIT_AND_EXPR
    3183          438 :                     && tree_fits_shwi_p (cdef_arg2[1 - i])
    3184          876 :                     && tree_to_shwi (cdef_arg2[1 - i])
    3185          438 :                        == TYPE_PRECISION (rtype) - 1
    3186          889 :                     && TREE_CODE (cdef_arg1[1 - i]) == SSA_NAME)
    3187              :                   {
    3188          438 :                     if (tem == cdef_arg1[1 - i]
    3189          213 :                         || tem2 == cdef_arg1[1 - i])
    3190              :                       {
    3191              :                         rotcnt = def_arg2[1 - i];
    3192          437 :                         break;
    3193              :                       }
    3194          193 :                     tree tem3;
    3195          193 :                     defcodefor_name (cdef_arg1[1 - i], &code, &tem3, NULL);
    3196            0 :                     if (CONVERT_EXPR_CODE_P (code)
    3197          193 :                         && INTEGRAL_TYPE_P (TREE_TYPE (tem3))
    3198          193 :                         && TYPE_PRECISION (TREE_TYPE (tem3))
    3199          193 :                            > floor_log2 (TYPE_PRECISION (rtype))
    3200          386 :                         && type_has_mode_precision_p (TREE_TYPE (tem3)))
    3201              :                       {
    3202          193 :                         if (tem == tem3 || tem2 == tem3)
    3203              :                           {
    3204              :                             rotcnt = def_arg2[1 - i];
    3205              :                             break;
    3206              :                           }
    3207              :                       }
    3208              :                   }
    3209              :               }
    3210              :           }
    3211         2324 :       if (check_range && wider_prec > TYPE_PRECISION (rtype))
    3212              :         {
    3213         1533 :           if (TREE_CODE (rotcnt) != SSA_NAME)
    3214          573 :             return false;
    3215         1533 :           int_range_max r;
    3216         1533 :           range_query *q = get_range_query (cfun);
    3217         1533 :           if (q == get_global_range_query ())
    3218         1522 :             q = enable_ranger (cfun);
    3219         1533 :           if (!q->range_of_expr (r, rotcnt, check_range_stmt))
    3220              :             {
    3221            0 :               if (check_range > 0)
    3222              :                 return false;
    3223            0 :               r.set_varying (TREE_TYPE (rotcnt));
    3224              :             }
    3225         1533 :           int prec = TYPE_PRECISION (TREE_TYPE (rotcnt));
    3226         1533 :           signop sign = TYPE_SIGN (TREE_TYPE (rotcnt));
    3227         1533 :           wide_int min = wide_int::from (TYPE_PRECISION (rtype), prec, sign);
    3228         1533 :           wide_int max = wide_int::from (wider_prec - 1, prec, sign);
    3229         1533 :           if (check_range < 0)
    3230          616 :             max = min;
    3231         1533 :           int_range<1> r2 (TREE_TYPE (rotcnt), min, max);
    3232         1533 :           r.intersect (r2);
    3233         1533 :           if (!r.undefined_p ())
    3234              :             {
    3235         1181 :               if (check_range > 0)
    3236              :                 {
    3237          589 :                   int_range_max r3;
    3238         1844 :                   for (int i = TYPE_PRECISION (rtype) + 1; i < wider_prec;
    3239         1255 :                        i += TYPE_PRECISION (rtype))
    3240              :                     {
    3241         1255 :                       int j = i + TYPE_PRECISION (rtype) - 2;
    3242         1255 :                       min = wide_int::from (i, prec, sign);
    3243         1255 :                       max = wide_int::from (MIN (j, wider_prec - 1),
    3244         1255 :                                             prec, sign);
    3245         1255 :                       int_range<1> r4 (TREE_TYPE (rotcnt), min, max);
    3246         1255 :                       r3.union_ (r4);
    3247         1255 :                     }
    3248          589 :                   r.intersect (r3);
    3249          589 :                   if (!r.undefined_p ())
    3250          573 :                     return false;
    3251          589 :                 }
    3252              :               add_masking = true;
    3253              :             }
    3254         1533 :         }
    3255         8924 :       if (rotcnt == NULL_TREE)
    3256              :         return false;
    3257         1751 :       swapped_p = i != 1;
    3258              :     }
    3259              : 
    3260         2674 :   if (!useless_type_conversion_p (TREE_TYPE (def_arg2[0]),
    3261         2674 :                                   TREE_TYPE (rotcnt)))
    3262              :     {
    3263          496 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (def_arg2[0])),
    3264              :                                NOP_EXPR, rotcnt);
    3265          496 :       gsi_insert_before (gsi, g, GSI_SAME_STMT);
    3266          496 :       rotcnt = gimple_assign_lhs (g);
    3267              :     }
    3268         2674 :   if (add_masking)
    3269              :     {
    3270          608 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (rotcnt)),
    3271              :                                BIT_AND_EXPR, rotcnt,
    3272          608 :                                build_int_cst (TREE_TYPE (rotcnt),
    3273          608 :                                               TYPE_PRECISION (rtype) - 1));
    3274          608 :       gsi_insert_before (gsi, g, GSI_SAME_STMT);
    3275          608 :       rotcnt = gimple_assign_lhs (g);
    3276              :     }
    3277         2674 :   lhs = gimple_assign_lhs (stmt);
    3278         2674 :   if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
    3279         1010 :     lhs = make_ssa_name (TREE_TYPE (def_arg1[0]));
    3280         2674 :   g = gimple_build_assign (lhs,
    3281         2674 :                            ((def_code[0] == LSHIFT_EXPR) ^ swapped_p)
    3282              :                            ? LROTATE_EXPR : RROTATE_EXPR, def_arg1[0], rotcnt);
    3283         2674 :   if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
    3284              :     {
    3285         1010 :       gsi_insert_before (gsi, g, GSI_SAME_STMT);
    3286         1010 :       g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, lhs);
    3287              :     }
    3288         2674 :   gsi_replace (gsi, g, false);
    3289         2674 :   return true;
    3290              : }
    3291              : 
    3292              : 
    3293              : /* Check whether an array contains a valid table according to VALIDATE_FN.  */
    3294              : template<typename ValidateFn>
    3295              : static bool
    3296           55 : check_table_array (tree ctor, HOST_WIDE_INT &zero_val, unsigned bits,
    3297              :                   ValidateFn validate_fn)
    3298              : {
    3299              :   tree elt, idx;
    3300           55 :   unsigned HOST_WIDE_INT i, raw_idx = 0;
    3301           55 :   unsigned matched = 0;
    3302              : 
    3303           55 :   zero_val = 0;
    3304              : 
    3305         2608 :   FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), i, idx, elt)
    3306              :     {
    3307         2608 :       if (!tree_fits_shwi_p (idx))
    3308              :         return false;
    3309         2608 :       if (!tree_fits_shwi_p (elt) && TREE_CODE (elt) != RAW_DATA_CST)
    3310              :         return false;
    3311              : 
    3312         2608 :       unsigned HOST_WIDE_INT index = tree_to_shwi (idx);
    3313              :       HOST_WIDE_INT val;
    3314              : 
    3315         2608 :       if (TREE_CODE (elt) == INTEGER_CST)
    3316         2544 :         val = tree_to_shwi (elt);
    3317              :       else
    3318              :         {
    3319           64 :           if (raw_idx == (unsigned) RAW_DATA_LENGTH (elt))
    3320              :             {
    3321            0 :               raw_idx = 0;
    3322            0 :               continue;
    3323              :             }
    3324           64 :           if (TYPE_UNSIGNED (TREE_TYPE (elt)))
    3325            0 :             val = RAW_DATA_UCHAR_ELT (elt, raw_idx);
    3326              :           else
    3327           64 :             val = RAW_DATA_SCHAR_ELT (elt, raw_idx);
    3328           64 :           index += raw_idx;
    3329           64 :           raw_idx++;
    3330           64 :           i--;
    3331              :         }
    3332              : 
    3333         2608 :       if (index > bits * 2)
    3334              :         return false;
    3335              : 
    3336         2608 :       if (index == 0)
    3337              :         {
    3338           55 :           zero_val = val;
    3339           55 :           matched++;
    3340              :         }
    3341              : 
    3342         2608 :       if (val >= 0 && val < bits && validate_fn (val, index))
    3343         1888 :         matched++;
    3344              : 
    3345         2608 :       if (matched > bits)
    3346              :         return true;
    3347              :     }
    3348              : 
    3349              :   return false;
    3350              : }
    3351              : 
    3352              : /* Check whether a string contains a valid table according to VALIDATE_FN.  */
    3353              : template<typename ValidateFn>
    3354              : static bool
    3355           39 : check_table_string (tree string, HOST_WIDE_INT &zero_val,unsigned bits,
    3356              :                     ValidateFn validate_fn)
    3357              : {
    3358           39 :   unsigned HOST_WIDE_INT len = TREE_STRING_LENGTH (string);
    3359           39 :   unsigned matched = 0;
    3360           39 :   const unsigned char *p = (const unsigned char *) TREE_STRING_POINTER (string);
    3361              : 
    3362           39 :   if (len < bits || len > bits * 2)
    3363              :     return false;
    3364              : 
    3365           39 :   zero_val = p[0];
    3366              : 
    3367         1703 :   for (unsigned i = 0; i < len; i++)
    3368         1664 :     if (p[i] < bits && validate_fn (p[i], i))
    3369         1664 :       matched++;
    3370              : 
    3371           39 :   return matched == bits;
    3372              : }
    3373              : 
    3374              : /* Check whether CTOR contains a valid table according to VALIDATE_FN.  */
    3375              : template<typename ValidateFn>
    3376              : static bool
    3377          102 : check_table (tree ctor, tree type, HOST_WIDE_INT &zero_val, unsigned bits,
    3378              :              ValidateFn validate_fn)
    3379              : {
    3380          102 :   if (TREE_CODE (ctor) == CONSTRUCTOR)
    3381           55 :     return check_table_array (ctor, zero_val, bits, validate_fn);
    3382              :   else if (TREE_CODE (ctor) == STRING_CST
    3383           47 :            && TYPE_PRECISION (type) == CHAR_TYPE_SIZE)
    3384           39 :     return check_table_string (ctor, zero_val, bits, validate_fn);
    3385              :   return false;
    3386              : }
    3387              : 
    3388              : /* Match.pd function to match the ctz expression.  */
    3389              : extern bool gimple_ctz_table_index (tree, tree *, tree (*)(tree));
    3390              : extern bool gimple_clz_table_index (tree, tree *, tree (*)(tree));
    3391              : extern bool gimple_clz_msb_iso_table_index (tree, tree *, tree (*)(tree));
    3392              : 
    3393              : /* Recognize count leading and trailing zeroes idioms.
    3394              :    The canonical form is array[((x & -x) * C) >> SHIFT] where C is a magic
    3395              :    constant which when multiplied by a power of 2 creates a unique value
    3396              :    in the top 5 or 6 bits.  This is then indexed into a table which maps it
    3397              :    to the number of trailing zeroes.  Array[0] is returned so the caller can
    3398              :    emit an appropriate sequence depending on whether ctz (0) is defined on
    3399              :    the target.  */
    3400              : 
    3401              : static bool
    3402      2020771 : simplify_count_zeroes (gimple_stmt_iterator *gsi)
    3403              : {
    3404      2020771 :   gimple *stmt = gsi_stmt (*gsi);
    3405      2020771 :   tree array_ref = gimple_assign_rhs1 (stmt);
    3406      2020771 :   tree res_ops[3];
    3407              : 
    3408      2020771 :   gcc_checking_assert (TREE_CODE (array_ref) == ARRAY_REF);
    3409              : 
    3410      2020771 :   internal_fn fn = IFN_LAST;
    3411              :   /* When true, the matched idiom is a CLZ using DeBruijn CTZ on the
    3412              :      isolated MSB -- see clz_msb_iso_table_index in match.pd.  The
    3413              :      table stores MSB positions and must satisfy the direct CTZ
    3414              :      DeBruijn property, so we validate it with the CTZ checkfn even
    3415              :      though we emit IFN_CLZ code.  */
    3416      2020771 :   bool clz_via_ctz = false;
    3417              :   /* For CTZ we recognize ((x & -x) * C) >> SHIFT where the array data
    3418              :      represents the number of trailing zeros.  */
    3419      2020771 :   if (gimple_ctz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0], NULL))
    3420              :     fn = IFN_CTZ;
    3421              :   /* For CLZ we recognize
    3422              :        x |= x >> 1;
    3423              :        x |= x >> 2;
    3424              :        x |= x >> 4;
    3425              :        x |= x >> 8;
    3426              :        x |= x >> 16;
    3427              :        (x * C) >> SHIFT
    3428              :      where 31 minus the array data represents the number of leading zeros.  */
    3429      2020705 :   else if (gimple_clz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0],
    3430              :                                    NULL))
    3431              :     fn = IFN_CLZ;
    3432              :   /* Variant CLZ idiom: after the OR-cascade sets all bits from 0 to
    3433              :      the original MSB, (value - (value >> 1)) isolates the MSB as a
    3434              :      power of two (2^k), and the subsequent DeBruijn multiply-and-shift
    3435              :      is a CTZ-style lookup on 2^k.  The table stores MSB positions
    3436              :      directly.  */
    3437      2020665 :   else if (gimple_clz_msb_iso_table_index (TREE_OPERAND (array_ref, 1),
    3438              :                                            &res_ops[0], NULL))
    3439              :     {
    3440              :       fn = IFN_CLZ;
    3441              :       clz_via_ctz = true;
    3442              :     }
    3443              :   else
    3444              :     return false;
    3445              : 
    3446          107 :   HOST_WIDE_INT zero_val;
    3447          107 :   tree type = TREE_TYPE (array_ref);
    3448          107 :   tree array = TREE_OPERAND (array_ref, 0);
    3449          107 :   tree input_type = TREE_TYPE (res_ops[0]);
    3450          107 :   unsigned input_bits = tree_to_shwi (TYPE_SIZE (input_type));
    3451              : 
    3452              :   /* Check the array element type is integral and not wider than 64 bits,
    3453              :      and the input is an unsigned 32-bit or 64-bit type.  The table values
    3454              :      are bit positions in [0, input_bits - 1], so any integer element type
    3455              :      with at least 6 bits of precision suffices; the cap is just to keep
    3456              :      the transformation simple.  */
    3457          107 :   if (!INTEGRAL_TYPE_P (type) || TYPE_PRECISION (type) > 64
    3458          214 :       || !TYPE_UNSIGNED (input_type))
    3459              :     return false;
    3460          103 :   if (input_bits != 32 && input_bits != 64)
    3461              :     return false;
    3462              : 
    3463          103 :   if (!direct_internal_fn_supported_p (fn, input_type, OPTIMIZE_FOR_BOTH))
    3464              :     return false;
    3465              : 
    3466              :   /* Check the lower bound of the array is zero.  */
    3467          103 :   tree low = array_ref_low_bound (array_ref);
    3468          103 :   if (!low || !integer_zerop (low))
    3469              :     return false;
    3470              : 
    3471              :   /* Check the shift extracts the top 5..7 bits.  */
    3472          103 :   unsigned shiftval = tree_to_shwi (res_ops[2]);
    3473          103 :   if (shiftval < input_bits - 7 || shiftval > input_bits - 5)
    3474              :     return false;
    3475              : 
    3476          102 :   tree ctor = ctor_for_folding (array);
    3477          102 :   if (!ctor)
    3478              :     return false;
    3479          102 :   unsigned HOST_WIDE_INT mulval = tree_to_uhwi (res_ops[1]);
    3480              :   /* CTZ and the MSB-isolation CLZ variant both use the direct CTZ
    3481              :      DeBruijn check (table[(magic << data) >> shift] == data).  */
    3482          102 :   if (fn == IFN_CTZ || clz_via_ctz)
    3483              :     {
    3484         2782 :       auto checkfn = [&](unsigned data, unsigned i) -> bool
    3485              :         {
    3486         2720 :           unsigned HOST_WIDE_INT mask
    3487         2720 :             = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
    3488         2720 :           return (((mulval << data) & mask) >> shiftval) == i;
    3489           62 :         };
    3490           62 :       if (!check_table (ctor, type, zero_val, input_bits, checkfn))
    3491            8 :         return false;
    3492              :     }
    3493           40 :   else if (fn == IFN_CLZ)
    3494              :     {
    3495         1569 :       auto checkfn = [&](unsigned data, unsigned i) -> bool
    3496              :         {
    3497         1529 :           unsigned HOST_WIDE_INT mask
    3498         1529 :             = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
    3499              :           /* The OR-cascade produces a value with all bits from 0 to the
    3500              :              original MSB set.  Compute (1 << (data + 1)) - 1 to simulate
    3501              :              that value.  When data + 1 equals HOST_BITS_PER_WIDE_INT
    3502              :              (i.e. data is the MSB position of a 64-bit input) the shift
    3503              :              is undefined behavior, so handle that case explicitly using
    3504              :              all-ones.  Without this, any well-formed 64-bit DeBruijn CLZ
    3505              :              table is rejected because its entry for the all-ones input
    3506              :              correctly maps to the MSB (e.g. table[...] == 63).
    3507              :              PR tree-optimization/122569.  */
    3508         3057 :           unsigned HOST_WIDE_INT all_bits_below
    3509              :             = (data + 1 == HOST_BITS_PER_WIDE_INT)
    3510         1529 :               ? HOST_WIDE_INT_M1U
    3511         1528 :               : ((HOST_WIDE_INT_1U << (data + 1)) - 1);
    3512         1529 :           return (((all_bits_below * mulval) & mask) >> shiftval) == i;
    3513           40 :         };
    3514           40 :     if (!check_table (ctor, type, zero_val, input_bits, checkfn))
    3515            0 :       return false;
    3516              :     }
    3517              : 
    3518              :   /* The sequence below computes the result in int, for a CLZ as
    3519              :      INPUT_BITS - 1 minus the count.  If the table entry for zero does not
    3520              :      survive that, select it in TYPE after the conversion instead.  */
    3521           94 :   HOST_WIDE_INT table_zero = zero_val;
    3522           94 :   bool late_select
    3523           94 :     = (!wi::fits_to_tree_p (zero_val, integer_type_node)
    3524           94 :        || (fn == IFN_CLZ
    3525           34 :            && !wi::fits_to_tree_p (input_bits - 1 - zero_val,
    3526           34 :                                  integer_type_node)));
    3527              : 
    3528           94 :   HOST_WIDE_INT ctz_val = -1;
    3529           94 :   bool zero_ok;
    3530           94 :   if (fn == IFN_CTZ)
    3531              :     {
    3532           53 :       ctz_val = 0;
    3533          106 :       zero_ok = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
    3534              :                                            ctz_val) == 2;
    3535              :     }
    3536           41 :   else if (fn == IFN_CLZ)
    3537              :     {
    3538           41 :       ctz_val = 32;
    3539           41 :       zero_ok = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
    3540              :                                            ctz_val) == 2;
    3541           41 :       if (!late_select)
    3542           34 :         zero_val = input_bits - 1 - zero_val;
    3543              :     }
    3544              :   /* Pass the value at zero only if the target defines it; otherwise the
    3545              :      COND_EXPR below supplies the result for zero.  */
    3546           94 :   int nargs = zero_ok ? 2 : 1;
    3547              : 
    3548              :   /* If the input value can't be zero, don't special case ctz (0).  */
    3549           94 :   range_query *q = get_range_query (cfun);
    3550           94 :   if (q == get_global_range_query ())
    3551           94 :     q = enable_ranger (cfun);
    3552           94 :   int_range_max vr;
    3553           94 :   if (q->range_of_expr (vr, res_ops[0], stmt)
    3554           94 :       && !range_includes_zero_p (vr))
    3555              :     {
    3556           24 :       zero_ok = true;
    3557           24 :       zero_val = 0;
    3558           24 :       ctz_val = 0;
    3559           24 :       nargs = 1;
    3560           24 :       late_select = false;
    3561              :     }
    3562              : 
    3563           94 :   gimple_seq seq = NULL;
    3564           94 :   gimple *g;
    3565          102 :   gcall *call = gimple_build_call_internal (fn, nargs, res_ops[0],
    3566              :                                             nargs == 1 ? NULL_TREE
    3567          102 :                                             : build_int_cst (integer_type_node,
    3568            8 :                                                              ctz_val));
    3569           94 :   gimple_set_location (call, gimple_location (stmt));
    3570           94 :   gimple_set_lhs (call, make_ssa_name (integer_type_node));
    3571           94 :   gimple_seq_add_stmt (&seq, call);
    3572              : 
    3573           94 :   tree prev_lhs = gimple_call_lhs (call);
    3574              : 
    3575           94 :   if (late_select || (zero_ok && zero_val == ctz_val))
    3576              :     ;
    3577              :   /* Emit ctz (x) & 31 if ctz (0) is 32 but we need to return 0.  */
    3578            6 :   else if (zero_ok && zero_val == 0 && ctz_val == input_bits)
    3579              :     {
    3580            5 :       g = gimple_build_assign (make_ssa_name (integer_type_node),
    3581              :                                BIT_AND_EXPR, prev_lhs,
    3582              :                                build_int_cst (integer_type_node,
    3583            5 :                                               input_bits - 1));
    3584            5 :       gimple_set_location (g, gimple_location (stmt));
    3585            5 :       gimple_seq_add_stmt (&seq, g);
    3586            5 :       prev_lhs = gimple_assign_lhs (g);
    3587              :     }
    3588              :   /* As fallback emit a conditional move.  */
    3589              :   else
    3590              :     {
    3591           42 :       g = gimple_build_assign (make_ssa_name (boolean_type_node), EQ_EXPR,
    3592              :                                res_ops[0], build_zero_cst (input_type));
    3593           42 :       gimple_set_location (g, gimple_location (stmt));
    3594           42 :       gimple_seq_add_stmt (&seq, g);
    3595           42 :       tree cond = gimple_assign_lhs (g);
    3596           42 :       g = gimple_build_assign (make_ssa_name (integer_type_node),
    3597              :                                COND_EXPR, cond,
    3598           42 :                                build_int_cst (integer_type_node, zero_val),
    3599              :                                prev_lhs);
    3600           42 :       gimple_set_location (g, gimple_location (stmt));
    3601           42 :       gimple_seq_add_stmt (&seq, g);
    3602           42 :       prev_lhs = gimple_assign_lhs (g);
    3603              :     }
    3604              : 
    3605           94 :   if (fn == IFN_CLZ)
    3606              :     {
    3607           41 :       g = gimple_build_assign (make_ssa_name (integer_type_node),
    3608              :                                MINUS_EXPR,
    3609              :                                build_int_cst (integer_type_node,
    3610           41 :                                               input_bits - 1),
    3611              :                                prev_lhs);
    3612           41 :       gimple_set_location (g, gimple_location (stmt));
    3613           41 :       gimple_seq_add_stmt (&seq, g);
    3614           41 :       prev_lhs = gimple_assign_lhs (g);
    3615              :     }
    3616              : 
    3617           94 :   if (late_select)
    3618              :     {
    3619           21 :       g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, prev_lhs);
    3620           21 :       gimple_set_location (g, gimple_location (stmt));
    3621           21 :       gimple_seq_add_stmt (&seq, g);
    3622           21 :       prev_lhs = gimple_assign_lhs (g);
    3623           21 :       g = gimple_build_assign (make_ssa_name (boolean_type_node), EQ_EXPR,
    3624              :                                res_ops[0], build_zero_cst (input_type));
    3625           21 :       gimple_set_location (g, gimple_location (stmt));
    3626           21 :       gimple_seq_add_stmt (&seq, g);
    3627           21 :       g = gimple_build_assign (gimple_assign_lhs (stmt), COND_EXPR,
    3628              :                                gimple_assign_lhs (g),
    3629           21 :                                build_int_cst (type, table_zero), prev_lhs);
    3630              :     }
    3631              :   else
    3632           73 :     g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, prev_lhs);
    3633           94 :   gimple_seq_add_stmt (&seq, g);
    3634           94 :   gsi_replace_with_seq (gsi, seq, true);
    3635           94 :   return true;
    3636           94 : }
    3637              : 
    3638              : /* Long-multiply fold framework.
    3639              : 
    3640              :    Walks the outer addition or bit_ior chain on a candidate statement,
    3641              :    classifies each summand against the atom match patterns from
    3642              :    match.pd, and looks the resulting multiset of (kind, extract) tuples
    3643              :    up in a table.  On a hit, three cross-summand consistency checks
    3644              :    decide whether the wide multiply is emitted.  */
    3645              : 
    3646              : /* Match.pd recognizers for the conditional carry-add pattern. */
    3647              : 
    3648              : extern bool gimple_cond_carry_add (tree, tree *, tree (*)(tree));
    3649              : 
    3650              : /* Match.pd functions to match long multiplication.  */
    3651              : 
    3652              : extern bool gimple_mul_hi (tree, tree *, tree (*)(tree));
    3653              : extern bool gimple_mul_lo (tree, tree *, tree (*)(tree));
    3654              : extern bool gimple_mul_hilo (tree, tree *, tree (*)(tree));
    3655              : extern bool gimple_mul_lolo (tree, tree *, tree (*)(tree));
    3656              : extern bool gimple_mul_hihi (tree, tree *, tree (*)(tree));
    3657              : extern bool gimple_mul_cross_sum (tree, tree *, tree (*)(tree));
    3658              : extern bool gimple_mul_low_sum (tree, tree *, tree (*)(tree));
    3659              : extern bool gimple_mul_low_accum (tree, tree *, tree (*)(tree));
    3660              : extern bool gimple_mul_carry_cross_sum (tree, tree *, tree (*)(tree));
    3661              : extern bool gimple_mul_carry_low_sum (tree, tree *, tree (*)(tree));
    3662              : extern bool gimple_mul_carry_low (tree, tree *, tree (*)(tree));
    3663              : extern bool gimple_mul_ladder_sum1 (tree, tree *, tree (*)(tree));
    3664              : extern bool gimple_mul_ladder_sum2 (tree, tree *, tree (*)(tree));
    3665              : extern bool gimple_mul_ladder_sum3 (tree, tree *, tree (*)(tree));
    3666              : extern bool gimple_mul_ladder_part_sum (tree, tree *, tree (*)(tree));
    3667              : 
    3668              : /* Append to SEQ statements assigning DEST the high-part multiply of
    3669              :    OP1 and OP2, emitted as
    3670              :      (N)(((2N) op1 * (2N) op2) >> N).
    3671              :    pass_optimize_widening_mul's convert_mult_to_widen and
    3672              :    convert_mult_to_highpart later rewrite this to a single
    3673              :    WIDEN_MULT_EXPR or MULT_HIGHPART_EXPR when the target supports it,
    3674              :    otherwise the 2N multiply expands directly.  Emitting the canonical
    3675              :    widening shape keeps target-capability decisions in the layer that
    3676              :    already owns them.  */
    3677              : 
    3678              : static void
    3679         1926 : build_mul_high_seq (tree op1, tree op2, tree dest, location_t loc,
    3680              :                     gimple_seq *seq)
    3681              : {
    3682         1926 :   tree op_type = TREE_TYPE (op1);
    3683         1926 :   unsigned int width = TYPE_PRECISION (op_type);
    3684         1926 :   tree wide_type = build_nonstandard_integer_type (width * 2, 1);
    3685              : 
    3686         1926 :   tree wide_a = gimple_convert (seq, loc, wide_type, op1);
    3687         1926 :   tree wide_b = gimple_convert (seq, loc, wide_type, op2);
    3688         1926 :   tree wide_prod = gimple_build (seq, loc, MULT_EXPR, wide_type,
    3689              :                                  wide_a, wide_b);
    3690         1926 :   tree hi = gimple_build (seq, loc, RSHIFT_EXPR, wide_type, wide_prod,
    3691         1926 :                           build_int_cst (integer_type_node, width));
    3692              : 
    3693         1926 :   gimple *prod = gimple_build_assign (dest, NOP_EXPR, hi);
    3694         1926 :   gimple_set_location (prod, loc);
    3695         1926 :   gimple_seq_add_stmt (seq, prod);
    3696         1926 : }
    3697              : 
    3698              : /* Append to SEQ statements combining ACC with each of EXTRAS under
    3699              :    OUTER, the last one assigning to STMT's lhs.  EXTRAS are leaves of
    3700              :    STMT's own chain, so any combining order is valid.  */
    3701              : 
    3702              : static void
    3703          361 : long_mul_apply_extras (tree acc, const vec<tree> &extras, tree_code outer,
    3704              :                        gassign *stmt, gimple_seq *seq)
    3705              : {
    3706          361 :   location_t loc = gimple_location (stmt);
    3707          361 :   tree lhs = gimple_assign_lhs (stmt);
    3708          388 :   for (unsigned i = 0; i + 1 < extras.length (); i++)
    3709           27 :     acc = gimple_build (seq, loc, outer, TREE_TYPE (lhs), acc, extras[i]);
    3710          361 :   gimple *last = gimple_build_assign (lhs, outer, acc, extras.last ());
    3711          361 :   gimple_set_location (last, loc);
    3712          361 :   gimple_seq_add_stmt (seq, last);
    3713          361 : }
    3714              : 
    3715              : /* Replace STMT with a high-part multiply of OP1 and OP2, combining any
    3716              :    EXTRAS back on top under OUTER.  */
    3717              : 
    3718              : static void
    3719         1912 : create_mul_high_seq (tree op1, tree op2, gassign *stmt,
    3720              :                      const vec<tree> &extras, tree_code outer)
    3721              : {
    3722         1912 :   gimple_seq seq = NULL;
    3723         1912 :   tree lhs = gimple_assign_lhs (stmt);
    3724         2273 :   tree dest = extras.is_empty () ? lhs : make_ssa_name (TREE_TYPE (lhs));
    3725         1912 :   build_mul_high_seq (op1, op2, dest, gimple_location (stmt), &seq);
    3726         1912 :   if (!extras.is_empty ())
    3727          361 :     long_mul_apply_extras (dest, extras, outer, stmt, &seq);
    3728         1912 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    3729         1912 :   gsi_replace_with_seq (&gsi, seq, true);
    3730         1912 : }
    3731              : 
    3732              : /* Replace STMT with a low-part multiply of OP1 and OP2, combining any
    3733              :    EXTRAS back on top under OUTER.  */
    3734              : 
    3735              : static void
    3736           34 : create_mul_low_seq (tree op1, tree op2, gassign *stmt,
    3737              :                     const vec<tree> &extras, tree_code outer)
    3738              : {
    3739           34 :   gimple_seq seq = NULL;
    3740           34 :   tree lhs = gimple_assign_lhs (stmt);
    3741           34 :   tree dest = extras.is_empty () ? lhs : make_ssa_name (TREE_TYPE (lhs));
    3742           34 :   gimple *prod = gimple_build_assign (dest, MULT_EXPR, op1, op2);
    3743           34 :   gimple_set_location (prod, gimple_location (stmt));
    3744           34 :   gimple_seq_add_stmt (&seq, prod);
    3745           34 :   if (!extras.is_empty ())
    3746            0 :     long_mul_apply_extras (dest, extras, outer, stmt, &seq);
    3747           34 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    3748           34 :   gsi_replace_with_seq (&gsi, seq, true);
    3749           34 : }
    3750              : 
    3751              : /* Widest match.pd atom (mul_carry_low_sum) takes 7 captures; round up
    3752              :    to 8 for the scratch buffers below.  */
    3753              : static constexpr unsigned LONG_MUL_MAX_CAPTURES = 8;
    3754              : 
    3755              : /* Longest variant in long_mul_table has 4 summands.  */
    3756              : static constexpr unsigned LONG_MUL_MAX_SUMMANDS = 4;
    3757              : 
    3758              : /* Cap on the leaves set aside as not part of the idiom, so an
    3759              :    arbitrarily long unrelated chain still bails early.  */
    3760              : static constexpr unsigned LONG_MUL_MAX_EXTRAS = 4;
    3761              : 
    3762              : namespace {
    3763              : 
    3764              : enum long_mul_kind : unsigned char {
    3765              :   LMK_INVALID,
    3766              :   LMK_MUL_HIHI,
    3767              :   LMK_MUL_LOLO,
    3768              :   LMK_MUL_HILO,
    3769              :   LMK_CROSS_SUM,
    3770              :   LMK_LOW_ACCUM,
    3771              :   LMK_LOW_SUM,
    3772              :   LMK_LADDER_SUM1,
    3773              :   LMK_LADDER_SUM2,
    3774              :   LMK_LADDER_SUM3,
    3775              :   LMK_LADDER_PART_SUM,
    3776              :   LMK_CARRY_LOW,
    3777              :   LMK_CARRY_CROSS_SUM,
    3778              :   LMK_CARRY_LOW_SUM,
    3779              : };
    3780              : 
    3781              : /* How the leaf wraps its inner kind.  Carry kinds use LMX_NONE: their
    3782              :    match.pd pattern bakes the lshift in, so the leaf is already the
    3783              :    complete carry expression.  */
    3784              : 
    3785              : enum long_mul_extract : unsigned char  {
    3786              :   LMX_NONE,
    3787              :   LMX_HI,
    3788              :   LMX_LO,
    3789              :   LMX_SHL_N,
    3790              : };
    3791              : 
    3792              : struct long_mul_summand {
    3793      6990872 :   long_mul_summand () {}
    3794              :   long_mul_summand (tree);
    3795              :   long_mul_summand (const long_mul_summand &) = default;
    3796              :   long_mul_kind kind = LMK_INVALID;
    3797              :   long_mul_extract extract = LMX_NONE;
    3798              :   tree op0 = NULL_TREE, op1 = NULL_TREE;
    3799              :   tree hilo[3] = { NULL_TREE, NULL_TREE, NULL_TREE };
    3800              :   tree carry_a = NULL_TREE, carry_b = NULL_TREE;
    3801              :   unsigned HOST_WIDE_INT shift = 0;
    3802      6971639 :   operator bool () { return kind != LMK_INVALID; }
    3803              : private:
    3804              :   bool classify_carry (tree leaf);
    3805              :   bool classify_hi_extract (tree, unsigned HOST_WIDE_INT);
    3806              :   bool classify_lo_extract (tree);
    3807              :   bool classify_shl_extract (tree, unsigned HOST_WIDE_INT);
    3808              :   bool classify_plus_kinds (tree);
    3809              :   bool classify_bare (tree);
    3810              :   void set (long_mul_kind kind, const tree *res_ops);
    3811              : };
    3812              : 
    3813              : }
    3814              : 
    3815              : /* Walk the OUTER addition or BIT_IOR chain rooted at STMT and collect
    3816              :    the leaf operands into LEAVES.  Descends through single-use
    3817              :    intermediate stmts of the same code.  Returns false once the leaf
    3818              :    count exceeds LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS, so an
    3819              :    overlong chain bails mid-walk instead of after a full traversal.
    3820              : 
    3821              :    If SHARED_DEF_OUT is non-NULL, record there the first inner stmt that
    3822              :    shares the outer code but has more than one use -- descending into it
    3823              :    would change semantics, so it stays as a leaf.  Such a leaf often
    3824              :    classifies as something no row matches, silently disabling the fold;
    3825              :    the caller surfaces this as a dump-file hint.  */
    3826              : 
    3827              : static bool
    3828      3405795 : long_mul_linearize_chain (gimple *stmt, tree_code outer, vec<tree> &leaves,
    3829              :                           gimple **shared_def_out = NULL)
    3830              : {
    3831      3405795 :   auto_vec<tree, 8> stack;
    3832      3405795 :   stack.safe_push (gimple_assign_rhs2 (stmt));
    3833      3405795 :   stack.safe_push (gimple_assign_rhs1 (stmt));
    3834              : 
    3835     10566324 :   while (!stack.is_empty ())
    3836              :     {
    3837      7161324 :       tree t = stack.pop ();
    3838      7161324 :       if (TREE_CODE (t) == SSA_NAME)
    3839              :         {
    3840      4704415 :           gimple *def = SSA_NAME_DEF_STMT (t);
    3841      4704415 :           if (def
    3842      4704415 :               && is_gimple_assign (def)
    3843      7946130 :               && gimple_assign_rhs_code (def) == outer)
    3844              :             {
    3845       340181 :               if (has_single_use (t))
    3846              :                 {
    3847       177212 :                   stack.safe_push (gimple_assign_rhs2 (def));
    3848       177212 :                   stack.safe_push (gimple_assign_rhs1 (def));
    3849       177212 :                   continue;
    3850              :                 }
    3851       162969 :               if (shared_def_out && !*shared_def_out)
    3852       147894 :                 *shared_def_out = def;
    3853              :             }
    3854              :         }
    3855      6984112 :       leaves.safe_push (t);
    3856      6984112 :       if (leaves.length () > LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS)
    3857          795 :         return false;
    3858              :     }
    3859      6810000 :   return !leaves.is_empty ();
    3860      3405795 : }
    3861              : 
    3862              : /* If EXPR is defined by LSHIFT_EXPR with a uhwi-valued amount, return
    3863              :    the shifted input via *INNER_OUT and the amount via *SHIFT_OUT.  */
    3864              : 
    3865              : static bool
    3866      6911593 : long_mul_is_lshift_def (tree expr, tree *inner_out,
    3867              :                         unsigned HOST_WIDE_INT *shift_out)
    3868              : {
    3869      6911593 :   if (TREE_CODE (expr) != SSA_NAME)
    3870              :     return false;
    3871      4455170 :   gimple *def = SSA_NAME_DEF_STMT (expr);
    3872      4455170 :   if (!def || !is_gimple_assign (def)
    3873      7448401 :       || gimple_assign_rhs_code (def) != LSHIFT_EXPR)
    3874              :     return false;
    3875        85503 :   tree amount = gimple_assign_rhs2 (def);
    3876        85503 :   if (!tree_fits_uhwi_p (amount))
    3877              :     return false;
    3878        59208 :   *inner_out = gimple_assign_rhs1 (def);
    3879        59208 :   *shift_out = tree_to_uhwi (amount);
    3880        59208 :   return true;
    3881              : }
    3882              : 
    3883              : /* Fill THIS's kind plus the captures from RES_OPS that the kind requires.
    3884              :    The kind itself determines how many (op0, op1) and hilo captures to
    3885              :    pick up from RES_OPS, and whether a baked-in shift is present.  */
    3886              : 
    3887              : void
    3888        55611 : long_mul_summand::set (long_mul_kind kind, const tree *res_ops)
    3889              : {
    3890        55611 :   this->kind = kind;
    3891        55611 :   unsigned n_ops = 0;
    3892        55611 :   unsigned n_hilos = 0;
    3893        55611 :   int shift_idx = -1;
    3894        53164 :   switch (kind)
    3895              :     {
    3896            0 :     default:
    3897            0 :       gcc_unreachable ();
    3898              :     case LMK_MUL_HIHI:
    3899              :     case LMK_MUL_LOLO:
    3900              :     case LMK_MUL_HILO:
    3901              :       n_ops = 2;
    3902              :       break;
    3903         5750 :     case LMK_CROSS_SUM:
    3904         5750 :       n_hilos = 2;
    3905         5750 :       break;
    3906        18606 :     case LMK_LOW_ACCUM:
    3907        18606 :     case LMK_LOW_SUM:
    3908        18606 :     case LMK_LADDER_SUM1:
    3909        18606 :     case LMK_LADDER_SUM2:
    3910        18606 :     case LMK_LADDER_SUM3:
    3911        18606 :       n_ops = 2;
    3912        18606 :       n_hilos = 2;
    3913        18606 :       break;
    3914          148 :     case LMK_LADDER_PART_SUM:
    3915          148 :       n_ops = 2;
    3916          148 :       n_hilos = 1;
    3917          148 :       break;
    3918              :     case LMK_CARRY_CROSS_SUM:
    3919              :       n_hilos = 3;
    3920              :       shift_idx = 3;
    3921              :       break;
    3922          106 :     case LMK_CARRY_LOW_SUM:
    3923          106 :       n_ops = 2;
    3924          106 :       n_hilos = 3;
    3925          106 :       shift_idx = 5;
    3926          106 :       break;
    3927         2447 :     case LMK_CARRY_LOW:
    3928         2447 :       this->carry_a = res_ops[0];
    3929         2447 :       this->carry_b = res_ops[1];
    3930            0 :       return;
    3931              :     }
    3932        24610 :   if (n_ops >= 1)
    3933        47240 :     this->op0 = res_ops[0];
    3934        47240 :   if (n_ops >= 2)
    3935        47240 :     this->op1 = res_ops[1];
    3936        53164 :   gcc_checking_assert (n_hilos <= 3);
    3937       102864 :   for (unsigned i = 0; i < n_hilos; i++)
    3938        49700 :     this->hilo[i] = res_ops[n_ops + i];
    3939        53164 :   if (shift_idx >= 0)
    3940              :     /* The carry atoms (mul_carry_cross_sum, mul_carry_low_sum) capture the
    3941              :        shift since it is always less than TYPE_PRECISION, using tree_to_uhwi is safe.  */
    3942          280 :     this->shift = tree_to_uhwi (res_ops[shift_idx]);
    3943              : }
    3944              : 
    3945              : /* Classify LEAF as a carry-kind summand.  The lshift amount is baked
    3946              :    into mul_carry_cross_sum / mul_carry_low_sum, so they're tried before
    3947              :    any branch that looks for a generic (X >> N) or (X << N) wrapper.  */
    3948              : 
    3949              : bool
    3950      6971639 : long_mul_summand::classify_carry (tree leaf)
    3951              : {
    3952      6971639 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    3953              :   /* mul_carry_low_sum's inner is constrained to mul_low_sum (cross_sum
    3954              :      + mul_hi(mul_lolo)); mul_carry_cross_sum's inner is just
    3955              :      mul_cross_sum (any plus); mul_carry_low matches gt:c (@0, plus(@0,
    3956              :      @1)) without a baked-in shift.  Most specific first, so the
    3957              :      less-constrained pattern doesn't shadow the more-constrained one.  */
    3958      6971639 :   if (gimple_mul_carry_low_sum (leaf, res_ops, NULL))
    3959              :     {
    3960          106 :       set (LMK_CARRY_LOW_SUM, res_ops);
    3961          106 :       return true;
    3962              :     }
    3963      6971533 :   if (gimple_mul_carry_cross_sum (leaf, res_ops, NULL))
    3964              :     {
    3965          174 :       set (LMK_CARRY_CROSS_SUM, res_ops);
    3966          174 :       return true;
    3967              :     }
    3968      6971359 :   if (gimple_mul_carry_low (leaf, res_ops, NULL))
    3969              :     {
    3970         2447 :       set (LMK_CARRY_LOW, res_ops);
    3971         2447 :       return true;
    3972              :     }
    3973              :   return false;
    3974              : }
    3975              : 
    3976              : /* Plus-based summand kinds shared by the (X >> SHIFT) and (X << SHIFT)
    3977              :    classifiers.  Order is by specificity: mul_low_sum's first arm is any
    3978              :    plus, so mul_ladder_sum1/3 (which constrain that arm to a plus
    3979              :    containing a mul_lo) and mul_low_accum (which constrains both arms)
    3980              :    shadow it and must come first.  */
    3981              : 
    3982              : bool
    3983        85550 : long_mul_summand::classify_plus_kinds (tree inner)
    3984              : {
    3985        85550 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    3986        85550 :   if (gimple_mul_low_accum (inner, res_ops, NULL))
    3987              :     {
    3988           28 :       set (LMK_LOW_ACCUM, res_ops);
    3989           28 :       return true;
    3990              :     }
    3991        85522 :   if (gimple_mul_ladder_sum3 (inner, res_ops, NULL))
    3992              :     {
    3993           15 :       set (LMK_LADDER_SUM3, res_ops);
    3994           15 :       return true;
    3995              :     }
    3996        85507 :   if (gimple_mul_ladder_sum1 (inner, res_ops, NULL))
    3997              :     {
    3998        18230 :       set (LMK_LADDER_SUM1, res_ops);
    3999        18230 :       return true;
    4000              :     }
    4001        67277 :   if (gimple_mul_low_sum (inner, res_ops, NULL))
    4002              :     {
    4003          230 :       set (LMK_LOW_SUM, res_ops);
    4004          230 :       return true;
    4005              :     }
    4006        67047 :   if (gimple_mul_ladder_sum2 (inner, res_ops, NULL))
    4007              :     {
    4008          103 :       set (LMK_LADDER_SUM2, res_ops);
    4009          103 :       return true;
    4010              :     }
    4011              :   return false;
    4012              : }
    4013              : 
    4014              : /* Classify INNER -- already unwrapped from an outer (X >> SHIFT) -- as
    4015              :    a high-half-extracted summand.  mul_hilo (mult-shape) is orthogonal
    4016              :    to the plus-based kinds and is tried first; ladder_part_sum (one arm
    4017              :    unconstrained) and mul_cross_sum (any plus) are the fallbacks after
    4018              :    the shared plus-based ladder.  */
    4019              : 
    4020              : bool
    4021        35612 : long_mul_summand::classify_hi_extract (tree inner, unsigned HOST_WIDE_INT shift)
    4022              : {
    4023        35612 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    4024        35612 :   this->extract = LMX_HI;
    4025        35612 :   this->shift = shift;
    4026        35612 :   if (gimple_mul_hilo (inner, res_ops, NULL))
    4027              :     {
    4028         9258 :       set (LMK_MUL_HILO, res_ops);
    4029         9258 :       return true;
    4030              :     }
    4031        26354 :   if (classify_plus_kinds (inner))
    4032              :     return true;
    4033        16919 :   if (gimple_mul_ladder_part_sum (inner, res_ops, NULL))
    4034              :     {
    4035          148 :       set (LMK_LADDER_PART_SUM, res_ops);
    4036          148 :       return true;
    4037              :     }
    4038        16771 :   if (gimple_mul_cross_sum (inner, res_ops, NULL))
    4039              :     {
    4040         2821 :       set (LMK_CROSS_SUM, res_ops);
    4041         2821 :       return true;
    4042              :     }
    4043              :   return false;
    4044              : }
    4045              : 
    4046              : /* Classify INNER -- already unwrapped from an outer (X & MASK) -- as
    4047              :    a low-half-masked summand.  */
    4048              : 
    4049              : bool
    4050        21721 : long_mul_summand::classify_lo_extract (tree inner)
    4051              : {
    4052        21721 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    4053        21721 :   this->extract = LMX_LO;
    4054        21721 :   if (gimple_mul_lolo (inner, res_ops, NULL))
    4055              :     {
    4056         9331 :       set (LMK_MUL_LOLO, res_ops);
    4057         9331 :       return true;
    4058              :     }
    4059              :   return false;
    4060              : }
    4061              : 
    4062              : /* Classify INNER -- already unwrapped from an outer (X << SHIFT) -- as
    4063              :    a left-shifted summand.  No mul_hilo / ladder_part_sum here -- those
    4064              :    shapes appear only under (X >> SHIFT).  */
    4065              : 
    4066              : bool
    4067        59196 : long_mul_summand::classify_shl_extract (tree inner,
    4068              :                                         unsigned HOST_WIDE_INT shift)
    4069              : {
    4070        59196 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    4071        59196 :   this->extract = LMX_SHL_N;
    4072        59196 :   this->shift = shift;
    4073        59196 :   if (classify_plus_kinds (inner))
    4074              :     return true;
    4075        50025 :   if (gimple_mul_cross_sum (inner, res_ops, NULL))
    4076              :     {
    4077         2929 :       set (LMK_CROSS_SUM, res_ops);
    4078         2929 :       return true;
    4079              :     }
    4080              :   return false;
    4081              : }
    4082              : 
    4083              : /* Classify LEAF as one of the bare-kind summands (no extraction
    4084              :    wrapper): mul_hihi or mul_lolo standing on their own.  */
    4085              : 
    4086              : bool
    4087      6852383 : long_mul_summand::classify_bare (tree leaf)
    4088              : {
    4089      6852383 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    4090      6852383 :   if (gimple_mul_hihi (leaf, res_ops, NULL))
    4091              :     {
    4092         9728 :       set (LMK_MUL_HIHI, res_ops);
    4093         9728 :       return true;
    4094              :     }
    4095      6842655 :   if (gimple_mul_lolo (leaf, res_ops, NULL))
    4096              :     {
    4097           63 :       set (LMK_MUL_LOLO, res_ops);
    4098           63 :       return true;
    4099              :     }
    4100              :   return false;
    4101              : }
    4102              : 
    4103              : /* Classify LEAF as one of the long-multiply summand shapes.  On success,
    4104              :    fill *INFO with the kind, extract, captured operands and shift.
    4105              :    Dispatches to per-extract helpers; the order matters because the
    4106              :    carry kinds bake an lshift into the pattern and would otherwise be
    4107              :    misread by the (X << N) branch.  */
    4108              : 
    4109      6971639 : long_mul_summand::long_mul_summand (tree leaf)
    4110      6971639 :  : long_mul_summand()
    4111              : {
    4112      6971639 :   tree res_ops[LONG_MUL_MAX_CAPTURES];
    4113              : 
    4114      6971639 :   if (classify_carry (leaf))
    4115       119256 :     return;
    4116              : 
    4117      6968912 :   if (gimple_mul_hi (leaf, res_ops, NULL))
    4118              :     {
    4119        35612 :       classify_hi_extract (res_ops[0], tree_to_uhwi (res_ops[1]));
    4120        35612 :       return;
    4121              :     }
    4122              : 
    4123      6933300 :   if (gimple_mul_lo (leaf, res_ops, NULL))
    4124              :     {
    4125        21721 :       classify_lo_extract (res_ops[0]);
    4126        21721 :       return;
    4127              :     }
    4128              : 
    4129      6911579 :   tree inner;
    4130      6911579 :   unsigned HOST_WIDE_INT shift;
    4131      6911579 :   if (long_mul_is_lshift_def (leaf, &inner, &shift))
    4132              :     {
    4133        59196 :       classify_shl_extract (inner, shift);
    4134        59196 :       return;
    4135              :     }
    4136              : 
    4137      6852383 :   classify_bare (leaf);
    4138              : }
    4139              : 
    4140              : /* qsort comparator: sort summands by (kind, extract) to put a multiset
    4141              :    into canonical order for table lookup.  Unstable sort within a tie is
    4142              :    harmless: no row in long_mul_table pairs distinct subterms under the
    4143              :    same (kind, extract), and long_mul_check_consistency cross-validates
    4144              :    that matching summands share one canonical (op0, op1).  */
    4145              : 
    4146              : static int
    4147       140020 : long_mul_summand_compare (const void *a, const void *b)
    4148              : {
    4149       140020 :   const long_mul_summand *sa = (const long_mul_summand *) a;
    4150       140020 :   const long_mul_summand *sb = (const long_mul_summand *) b;
    4151       140020 :   gcc_checking_assert (sa->kind != LMK_INVALID);
    4152       140020 :   gcc_checking_assert (sb->kind != LMK_INVALID);
    4153       140020 :   if (sa->kind != sb->kind)
    4154       139157 :     return (int) sa->kind - (int) sb->kind;
    4155          863 :   return (int) sa->extract - (int) sb->extract;
    4156              : }
    4157              : 
    4158              : /* One row of the long-multiply variant table.  COUNT is how many entries
    4159              :    of SIG carry the row's signature (2 to LONG_MUL_MAX_SUMMANDS); a row
    4160              :    with fewer summands leaves the remaining SIG entries zero-initialized.
    4161              :    Those zeros are not a terminator -- {LMK_MUL_HIHI, LMX_NONE} is itself a
    4162              :    valid signature -- so long_mul_signature_matches is bounded by COUNT,
    4163              :    never by a sentinel entry.  */
    4164              : 
    4165              : struct long_mul_row {
    4166              :   enum long_mul_row_part { HIGH_PART, LOW_PART } part;
    4167              :   tree_code outer;
    4168              :   unsigned char count;
    4169              :   struct {
    4170              :     long_mul_kind kind;
    4171              :     long_mul_extract extract;
    4172              :   } sig[LONG_MUL_MAX_SUMMANDS];
    4173              :   bool (*extra_check) (const vec<long_mul_summand> &, gimple *);
    4174              : };
    4175              : 
    4176              : /* True if (A, B) is the same pair as (OP0, OP1) in either order.  */
    4177              : 
    4178              : static inline bool
    4179        23337 : long_mul_same_ops (tree a, tree b, tree op0, tree op1)
    4180              : {
    4181        32740 :   return (a == op0 && b == op1) || (a == op1 && b == op0);
    4182              : }
    4183              : 
    4184              : /* True if H is a cross-half product of (OP0, OP1) -- gimple_mul_hilo
    4185              :    recognizes it and its captured operands match the pair.  */
    4186              : 
    4187              : static bool
    4188         4473 : long_mul_is_cross_half (tree h, tree op0, tree op1)
    4189              : {
    4190         4473 :   tree scratch[LONG_MUL_MAX_CAPTURES];
    4191         4473 :   return gimple_mul_hilo (h, scratch, NULL)
    4192         4473 :          && long_mul_same_ops (scratch[0], scratch[1], op0, op1);
    4193              : }
    4194              : 
    4195              : /* Orientation of the mul_hilo capture H relative to (OP0, OP1):
    4196              :    returns 0 for high(OP0)*low(OP1), 1 for high(OP1)*low(OP0), or -1
    4197              :    if H does not decompose that way.  A cross-sum of two mul_hilos must
    4198              :    see one of each orientation -- otherwise a doubled factor would fold
    4199              :    to the wrong value.  (In a self-multiply the two orientations
    4200              :    coincide; see the OP0 == OP1 bypass in long_mul_check_consistency.)  */
    4201              : 
    4202              : static int
    4203         6085 : long_mul_hilo_orientation (tree h, tree op0, tree op1)
    4204              : {
    4205         6085 :   tree scratch[LONG_MUL_MAX_CAPTURES];
    4206         6085 :   if (!gimple_mul_hilo (h, scratch, NULL))
    4207              :     return -1;
    4208         6085 :   if (scratch[0] == op0 && scratch[1] == op1)
    4209              :     return 0;
    4210         2135 :   if (scratch[0] == op1 && scratch[1] == op0)
    4211         2135 :     return 1;
    4212              :   return -1;
    4213              : }
    4214              : 
    4215              : /* Find the first summand that carries operand captures, and return its
    4216              :    (op0, op1) pair in *OP0_OUT / *OP1_OUT.  Returns false if no summand
    4217              :    provides them.  */
    4218              : 
    4219              : static bool
    4220         8136 : long_mul_canonical_ops (const vec<long_mul_summand> &summands,
    4221              :                         tree *op0_out, tree *op1_out)
    4222              : {
    4223        24408 :   for (const long_mul_summand &s : summands)
    4224         8136 :     if (s.op0)
    4225              :       {
    4226         8136 :         *op0_out = s.op0;
    4227         8136 :         *op1_out = s.op1;
    4228         8136 :         return true;
    4229              :       }
    4230              :   return false;
    4231              : }
    4232              : 
    4233              : /* Return the first summand in SUMMANDS whose kind matches KIND, or NULL.  */
    4234              : 
    4235              : static const long_mul_summand *
    4236         3738 : long_mul_find_summand (const vec<long_mul_summand> &summands,
    4237              :                        long_mul_kind kind)
    4238              : {
    4239         3738 :   gcc_checking_assert (kind != LMK_INVALID);
    4240        16827 :   for (const long_mul_summand &s : summands)
    4241         9351 :     if (s.kind == kind)
    4242              :       return &s;
    4243              :   return NULL;
    4244              : }
    4245              : 
    4246              : /* Run the cross-summand validation invariants and return the canonical
    4247              :    (op0, op1).  Returns false unless all summands that carry operands use
    4248              :    the same (op0, op1) pair (in either order), every LMX_HI/LMX_SHL_N shift
    4249              :    equals halfwidth, every captured hilo is a true cross-half product of
    4250              :    (op0, op1), and every cross-half pair (both those inside a single
    4251              :    mul_cross_sum-bearing summand and those spread across separate
    4252              :    LMK_MUL_HILO summands) contains one of each orientation.  */
    4253              : 
    4254              : static bool
    4255         8124 : long_mul_check_consistency (const vec<long_mul_summand> &summands,
    4256              :                    tree *op0_out, tree *op1_out)
    4257              : {
    4258         8124 :   tree op0, op1;
    4259         8124 :   if (!long_mul_canonical_ops (summands, &op0, &op1))
    4260              :     return false;
    4261              : 
    4262         8124 :   tree op_type = TREE_TYPE (op0);
    4263         8124 :   if (!INTEGRAL_TYPE_P (op_type)
    4264         8124 :       || TYPE_PRECISION (op_type) % 2 != 0)
    4265              :     return false;
    4266         8124 :   unsigned int halfwidth = TYPE_PRECISION (op_type) / 2;
    4267              : 
    4268              :   /* Self-multiply (x*x) collapses the two cross-halves onto one value,
    4269              :      so the complementarity constraint is a trivial no-op there.  */
    4270         8124 :   bool need_orient = op0 != op1;
    4271         8124 :   int mul_hilo_orient[2] = { 0, 0 };
    4272              : 
    4273        37249 :   for (const long_mul_summand &s : summands)
    4274              :     {
    4275        18981 :       if (s.op0 && !long_mul_same_ops (s.op0, s.op1, op0, op1))
    4276              :         return false;
    4277        12903 :       if ((s.extract == LMX_HI || s.extract == LMX_SHL_N)
    4278         4690 :           && s.shift != halfwidth)
    4279              :         return false;
    4280        51558 :       for (tree h : s.hilo)
    4281        38681 :         if (h && !long_mul_is_cross_half (h, op0, op1))
    4282              :           return false;
    4283              : 
    4284        12877 :       if (!need_orient)
    4285            8 :         continue;
    4286              : 
    4287              :       /* The two cross-sum operands are the last two non-null hilos:
    4288              :          (hilo[1], hilo[2]) for the CARRY_*_SUM kinds, (hilo[0], hilo[1]) for
    4289              :          the CROSS_SUM / SUM / ACCUM / LADDER_SUM kinds, none for the
    4290              :          rest.  */
    4291        12869 :       tree a = NULL_TREE;
    4292        12869 :       tree b = NULL_TREE;
    4293        12869 :       if (s.hilo[2])
    4294              :         {
    4295           77 :           a = s.hilo[1];
    4296           77 :           b = s.hilo[2];
    4297              :         }
    4298        12792 :       else if (s.hilo[1])
    4299              :         {
    4300         2034 :           a = s.hilo[0];
    4301         2034 :           b = s.hilo[1];
    4302              :         }
    4303        12869 :       if (a && b
    4304        14980 :           && (long_mul_hilo_orientation (a, op0, op1)
    4305         2111 :               == long_mul_hilo_orientation (b, op0, op1)))
    4306              :         return false;
    4307              : 
    4308              :       /* Two LMK_MUL_HILO summands (the two-hilos ladder form) stand for
    4309              :          the two cross-halves separately; count orientations and require
    4310              :          the pair to be complementary.  s.op0/op1 is already validated to
    4311              :          match (op0, op1) in some order above.  */
    4312        12869 :       if (s.kind == LMK_MUL_HILO && s.op0)
    4313         2532 :         mul_hilo_orient[s.op0 == op1]++;
    4314              :     }
    4315              : 
    4316         2020 :   if (mul_hilo_orient[0] + mul_hilo_orient[1] >= 2
    4317           10 :       && (mul_hilo_orient[0] == 0 || mul_hilo_orient[1] == 0))
    4318              :     return false;
    4319              : 
    4320         2020 :   *op0_out = op0;
    4321         2020 :   *op1_out = op1;
    4322         2020 :   return true;
    4323              : }
    4324              : 
    4325              : /* Compare the (already-sorted) SUMMANDS multiset against ROW.sig.  */
    4326              : 
    4327              : static bool
    4328       124705 : long_mul_signature_matches (const vec<long_mul_summand> &summands,
    4329              :                    const long_mul_row &row)
    4330              : {
    4331       249410 :   if (row.count != summands.length ())
    4332              :     return false;
    4333        75726 :   for (unsigned i = 0; i < row.count; i++)
    4334        67602 :     if (summands[i].kind != row.sig[i].kind
    4335        67602 :         || summands[i].extract != row.sig[i].extract)
    4336              :       return false;
    4337              :   return true;
    4338              : }
    4339              : 
    4340              : /* Extra check for the two-carries high-part row: the LMK_CARRY_LOW summand's
    4341              :    two operands (carry_a, carry_b) must be a (cross_shifted, mul_lolo) pair
    4342              :    consistent with the multiset's canonical (op0, op1).  */
    4343              : 
    4344              : static bool
    4345           12 : long_mul_check_two_carries (const vec<long_mul_summand> &summands,
    4346              :                             gimple *)
    4347              : {
    4348           12 :   tree op0, op1;
    4349           12 :   if (!long_mul_canonical_ops (summands, &op0, &op1))
    4350              :     return false;
    4351           12 :   unsigned int halfwidth = TYPE_PRECISION (TREE_TYPE (op0)) / 2;
    4352              : 
    4353           12 :   const long_mul_summand *cl = long_mul_find_summand (summands, LMK_CARRY_LOW);
    4354           12 :   if (!cl)
    4355              :     return false;
    4356              : 
    4357              :   /* The two carry_low operands must be (cross_shifted, mul_lolo) in either
    4358              :      order.  cross_shifted = LSHIFT_EXPR (mul_cross_sum, halfwidth).  */
    4359           12 :   tree cs = cl->carry_a, lolo = cl->carry_b;
    4360           12 :   tree inner;
    4361           12 :   unsigned HOST_WIDE_INT shift;
    4362           12 :   if (!long_mul_is_lshift_def (cs, &inner, &shift))
    4363              :     {
    4364            2 :       std::swap (cs, lolo);
    4365            2 :       if (!long_mul_is_lshift_def (cs, &inner, &shift))
    4366              :         return false;
    4367              :     }
    4368           12 :   if (shift != halfwidth)
    4369              :     return false;
    4370              : 
    4371           12 :   tree scratch[LONG_MUL_MAX_CAPTURES];
    4372           12 :   if (!gimple_mul_cross_sum (inner, scratch, NULL))
    4373              :     return false;
    4374           36 :   for (int i = 0; i < 2; i++)
    4375           24 :     if (!long_mul_is_cross_half (scratch[i], op0, op1))
    4376              :       return false;
    4377           12 :   if (!gimple_mul_lolo (lolo, scratch, NULL)
    4378           12 :       || !long_mul_same_ops (scratch[0], scratch[1], op0, op1))
    4379              :     return false;
    4380              : 
    4381              :   return true;
    4382              : }
    4383              : 
    4384              : /* Extra check for xh*yh + (hilo >> N) + (ladder_sum1 >> N).  hilo must be
    4385              :    the product that ladder_sum1 masks, since the high half of the unmasked
    4386              :    one is already in the sum.  */
    4387              : 
    4388              : static bool
    4389         1769 : long_mul_check_ladder_sum1 (const vec<long_mul_summand> &summands,
    4390              :                             gimple *)
    4391              : {
    4392         1769 :   const long_mul_summand *hilo
    4393         1769 :     = long_mul_find_summand (summands, LMK_MUL_HILO);
    4394         1769 :   const long_mul_summand *sum
    4395         1769 :     = long_mul_find_summand (summands, LMK_LADDER_SUM1);
    4396         1769 :   return long_mul_hilo_orientation (sum->hilo[0], hilo->op0, hilo->op1) == 0;
    4397              : }
    4398              : 
    4399              : /* Likewise for xh*yh + (ladder_sum2 >> N) + (ladder_part_sum >> N):
    4400              :    ladder_part_sum must hold the product that ladder_sum2 masks.  */
    4401              : 
    4402              : static bool
    4403           94 : long_mul_check_ladder_sum2 (const vec<long_mul_summand> &summands,
    4404              :                             gimple *)
    4405              : {
    4406           94 :   const long_mul_summand *part
    4407           94 :     = long_mul_find_summand (summands, LMK_LADDER_PART_SUM);
    4408           94 :   const long_mul_summand *sum
    4409           94 :     = long_mul_find_summand (summands, LMK_LADDER_SUM2);
    4410           94 :   tree masked[LONG_MUL_MAX_CAPTURES];
    4411           94 :   return (gimple_mul_hilo (sum->hilo[0], masked, NULL)
    4412           94 :           && (long_mul_hilo_orientation (part->hilo[0], masked[0], masked[1])
    4413           94 :               == 0));
    4414              : }
    4415              : 
    4416              : /* The lolo + cross_shifted shape is also the low half of a two-carry
    4417              :    long-multiply, where an unsigned overflow compare against one of
    4418              :    the PLUS operands is the low-carry term consumed by the matching
    4419              :    high-part fold.  Folding to mul_lo here destroys cross_shifted,
    4420              :    which both the compare and the high-part match still need; defer
    4421              :    so the high-part fold runs first.  After it does, the compare is
    4422              :    dead and the surviving lolo + cross_shifted is picked up by this
    4423              :    row in the next forwprop instance.  Returns false to defer.  */
    4424              : 
    4425              : static bool
    4426           51 : long_mul_check_low_plus_defer (const vec<long_mul_summand> &, gimple *stmt)
    4427              : {
    4428              :   /* The PHI entry passes its gphi as the candidate but commits only to
    4429              :      HIGH_PART rows, so a LOW_PART row never folds from there.  Guard the
    4430              :      gimple_assign accessors regardless, so this stays correct if a future
    4431              :      PLUS-shaped row reachable from the PHI path uses it.  */
    4432           51 :   if (!is_gimple_assign (stmt))
    4433              :     return false;
    4434              : 
    4435           51 :   tree lhs = gimple_assign_lhs (stmt);
    4436           51 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    4437           51 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    4438              : 
    4439           51 :   imm_use_iterator iter;
    4440           51 :   gimple *use_stmt;
    4441           79 :   FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    4442              :     {
    4443           64 :       tree cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
    4444           64 :       enum tree_code use_code = ERROR_MARK;
    4445           64 :       if (is_gimple_assign (use_stmt))
    4446              :         {
    4447           42 :           use_code = gimple_assign_rhs_code (use_stmt);
    4448           42 :           cmp_op1 = gimple_assign_rhs1 (use_stmt);
    4449           42 :           cmp_op2 = gimple_assign_rhs2 (use_stmt);
    4450              :         }
    4451           22 :       else if (gcond *cond = dyn_cast<gcond *> (use_stmt))
    4452              :         {
    4453            9 :           use_code = gimple_cond_code (cond);
    4454            9 :           cmp_op1 = gimple_cond_lhs (cond);
    4455            9 :           cmp_op2 = gimple_cond_rhs (cond);
    4456              :         }
    4457           64 :       if (use_code == GT_EXPR || use_code == LT_EXPR
    4458           28 :           || use_code == GE_EXPR || use_code == LE_EXPR)
    4459              :         {
    4460           36 :           tree other = (cmp_op1 == lhs) ? cmp_op2
    4461           23 :                      : (cmp_op2 == lhs) ? cmp_op1 : NULL_TREE;
    4462           36 :           if (other && (other == rhs1 || other == rhs2))
    4463           36 :             return false;
    4464              :         }
    4465           36 :     }
    4466           15 :   return true;
    4467              : }
    4468              : 
    4469              : /* Long-multiply variant table.  Each row enumerates the multiset of
    4470              :    (kind, extract) summands that compose one long-multiply form.  Rows
    4471              :    are sorted by long_mul_summand_compare, matching the input summands'
    4472              :    sort order, so a plain element-wise compare suffices.  Rows describe
    4473              :    unsigned schoolbook expansions on an even-width 2N-bit type split at
    4474              :    half-width N; EXTRA_CHECK carries invariants the (kind, extract)
    4475              :    signature cannot express.
    4476              : 
    4477              :    The formula on each row uses xh, xl, yh, yl for the half-width pieces
    4478              :    of x and y, cross_sum for xh*yl + xl*yh, and hilo for either cross-half
    4479              :    product (consumers validate the operand shape).  */
    4480              : 
    4481              : static const long_mul_row long_mul_table[] = {
    4482              :   /* HIGH-PART folds.  */
    4483              :   /* xh*yh + (low_sum >> N) + ((hilo > low_sum) << N),
    4484              :      low_sum = cross_sum + (xl*yl >> N).  */
    4485              :   { long_mul_row::HIGH_PART, PLUS_EXPR, 3,
    4486              :     { { LMK_MUL_HIHI, LMX_NONE },
    4487              :       { LMK_LOW_SUM, LMX_HI },
    4488              :       { LMK_CARRY_LOW_SUM, LMX_NONE } },
    4489              :     NULL },
    4490              :   /* xh*yh + (low_accum >> N) + (cross_sum >> N) + ((hilo > cross_sum) << N),
    4491              :      low_accum = (xl*yl >> N) + (cross_sum & mask).  */
    4492              :   { long_mul_row::HIGH_PART, PLUS_EXPR, 4,
    4493              :     { { LMK_MUL_HIHI, LMX_NONE },
    4494              :       { LMK_CROSS_SUM, LMX_HI },
    4495              :       { LMK_LOW_ACCUM, LMX_HI },
    4496              :       { LMK_CARRY_CROSS_SUM, LMX_NONE } },
    4497              :     NULL },
    4498              :   /* xh*yh + (cross_sum >> N) + carry_low + ((hilo > cross_sum) << N),
    4499              :      carry_low = (xl*yl + (cross_sum << N)) < (cross_sum << N).  */
    4500              :   { long_mul_row::HIGH_PART, PLUS_EXPR, 4,
    4501              :     { { LMK_MUL_HIHI, LMX_NONE },
    4502              :       { LMK_CROSS_SUM, LMX_HI },
    4503              :       { LMK_CARRY_LOW, LMX_NONE },
    4504              :       { LMK_CARRY_CROSS_SUM, LMX_NONE } },
    4505              :     long_mul_check_two_carries },
    4506              :   /* xh*yh + (hilo >> N) + (ladder_sum1 >> N),
    4507              :      ladder_sum1 = (hilo & mask) + hilo' + (xl*yl >> N),
    4508              :      hilo, hilo' the two cross-half products.  */
    4509              :   { long_mul_row::HIGH_PART, PLUS_EXPR, 3,
    4510              :     { { LMK_MUL_HIHI, LMX_NONE },
    4511              :       { LMK_MUL_HILO, LMX_HI },
    4512              :       { LMK_LADDER_SUM1, LMX_HI } },
    4513              :     long_mul_check_ladder_sum1 },
    4514              :   /* xh*yh + (ladder_sum2 >> N) + (ladder_part_sum >> N),
    4515              :      ladder_part_sum = (xl*yl >> N) + hilo,
    4516              :      ladder_sum2 = (ladder_part_sum & mask) + hilo'.  */
    4517              :   { long_mul_row::HIGH_PART, PLUS_EXPR, 3,
    4518              :     { { LMK_MUL_HIHI, LMX_NONE },
    4519              :       { LMK_LADDER_SUM2, LMX_HI },
    4520              :       { LMK_LADDER_PART_SUM, LMX_HI } },
    4521              :     long_mul_check_ladder_sum2 },
    4522              :   /* xh*yh + (hilo >> N) + (hilo' >> N) + (ladder_sum3 >> N),
    4523              :      ladder_sum3 = (hilo & mask) + (hilo' & mask) + (xl*yl >> N).  */
    4524              :   { long_mul_row::HIGH_PART, PLUS_EXPR, 4,
    4525              :     { { LMK_MUL_HIHI, LMX_NONE },
    4526              :       { LMK_MUL_HILO, LMX_HI },
    4527              :       { LMK_MUL_HILO, LMX_HI },
    4528              :       { LMK_LADDER_SUM3, LMX_HI } },
    4529              :     NULL },
    4530              :   /* LOW-PART folds.  Recover the lower 2N bits from xl*yl plus a
    4531              :      shifted cross-half term.  */
    4532              :   /* xl*yl + (cross_sum << N).  */
    4533              :   { long_mul_row::LOW_PART, PLUS_EXPR, 2,
    4534              :     { { LMK_MUL_LOLO, LMX_NONE },
    4535              :       { LMK_CROSS_SUM, LMX_SHL_N } },
    4536              :     long_mul_check_low_plus_defer },
    4537              :   /* (xl*yl & mask) | (low_accum << N),
    4538              :      low_accum = (xl*yl >> N) + (cross_sum & mask).  */
    4539              :   { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
    4540              :     { { LMK_MUL_LOLO, LMX_LO },
    4541              :       { LMK_LOW_ACCUM, LMX_SHL_N } },
    4542              :     NULL },
    4543              :   /* (xl*yl & mask) | (low_sum << N),
    4544              :      low_sum = cross_sum + (xl*yl >> N).  */
    4545              :   { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
    4546              :     { { LMK_MUL_LOLO, LMX_LO },
    4547              :       { LMK_LOW_SUM, LMX_SHL_N } },
    4548              :     NULL },
    4549              :   /* (xl*yl & mask) | (ladder_sum1 << N),
    4550              :      ladder_sum1 as in the high ladder row above.  */
    4551              :   { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
    4552              :     { { LMK_MUL_LOLO, LMX_LO },
    4553              :       { LMK_LADDER_SUM1, LMX_SHL_N } },
    4554              :     NULL },
    4555              :   /* (xl*yl & mask) | (ladder_sum2 << N),
    4556              :      ladder_sum2 as in the high ladder row above.  */
    4557              :   { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
    4558              :     { { LMK_MUL_LOLO, LMX_LO },
    4559              :       { LMK_LADDER_SUM2, LMX_SHL_N } },
    4560              :     NULL },
    4561              :   /* (xl*yl & mask) | (ladder_sum3 << N),
    4562              :      ladder_sum3 as in the high ladder-long row above.  */
    4563              :   { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
    4564              :     { { LMK_MUL_LOLO, LMX_LO },
    4565              :       { LMK_LADDER_SUM3, LMX_SHL_N } },
    4566              :     NULL },
    4567              : };
    4568              : 
    4569              : /* If a multi-used inner addition (sharing the chain's outer code) blocked
    4570              :    linearization of a long-mul candidate, emit a dump-file hint pointing
    4571              :    at it.  */
    4572              : 
    4573              : static void
    4574        21370 : long_mul_hint_shared_intermediate (gimple *shared_def)
    4575              : {
    4576        21370 :   if (!shared_def || !dump_file || !(dump_flags & TDF_DETAILS))
    4577              :     return;
    4578            0 :   fprintf (dump_file, "long-mul fold rejected: shared intermediate at ");
    4579            0 :   print_gimple_stmt (dump_file, shared_def, 0, TDF_SLIM);
    4580              : }
    4581              : 
    4582              : /* Search long_mul_table for a row whose multiset matches SUMMANDS for
    4583              :    outer kind OUTER on a result of type LHS_TYPE.  CANDIDATE_STMT is
    4584              :    passed to per-row extra_check predicates.  On a hit, returns the
    4585              :    matching row and writes the half-width operands via OUT_OP0/OUT_OP1.
    4586              :    No IR mutation.  */
    4587              : 
    4588              : static const long_mul_row *
    4589        18763 : long_mul_classify_match (const vec<long_mul_summand> &summands,
    4590              :                          tree lhs_type, tree_code outer,
    4591              :                          gimple *candidate_stmt,
    4592              :                          tree *out_op0, tree *out_op1)
    4593              : {
    4594              :   /* HIGH_PART rows emit a 2N-bit multiply that pass_optimize_widening_mul
    4595              :      consumes -- either via WIDEN_MULT_EXPR / MULT_HIGHPART conversion when
    4596              :      the target has a native 2N multiply, or via lower_long_mul_high_chain
    4597              :      when it does not.  LOW_PART rows emit a plain MULT_EXPR.  Emission
    4598              :      needs a 2N mode to exist in the mode table AND the widening_mul pass
    4599              :      to be active: without the pass, the emit could reach RTL expand as an
    4600              :      unexpandable 2N multiply (e.g. OImode).  BITINT_TYPE is
    4601              :      refused -- the long_mul_high_chain atom excludes it.  */
    4602        18763 :   scalar_int_mode mode, wide_mode;
    4603        18763 :   bool can_emit_high
    4604        18763 :     = optimize_widening_mul_active_p ()
    4605        18672 :       && TREE_CODE (lhs_type) != BITINT_TYPE
    4606        37344 :       && is_a <scalar_int_mode> (TYPE_MODE (lhs_type), &mode)
    4607        37435 :       && GET_MODE_2XWIDER_MODE (mode).exists (&wide_mode);
    4608              : 
    4609       226350 :   for (const long_mul_row &row : long_mul_table)
    4610              :     {
    4611       410970 :       if (row.outer != outer
    4612       125251 :           || (row.part == long_mul_row::HIGH_PART && !can_emit_high)
    4613       334252 :           || !long_mul_signature_matches (summands, row))
    4614       207587 :         continue;
    4615              : 
    4616         8124 :       tree op0, op1;
    4617         8124 :       if (!long_mul_check_consistency (summands, &op0, &op1))
    4618         6104 :         continue;
    4619              : 
    4620              :       /* Do not emit the wide chain when an operand is subject to
    4621              :          abnormal coalescing: the widening_mul-side consumers refuse
    4622              :          such operands (see convert_mult_to_widen), which would leave
    4623              :          the chain without a consumer.  */
    4624         2020 :       if (row.part == long_mul_row::HIGH_PART
    4625         2020 :           && ((TREE_CODE (op0) == SSA_NAME
    4626         1950 :                && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op0))
    4627         1950 :               || (TREE_CODE (op1) == SSA_NAME
    4628         1950 :                   && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op1))))
    4629            0 :         continue;
    4630              : 
    4631         2020 :       if (row.extra_check && !row.extra_check (summands, candidate_stmt))
    4632           60 :         continue;
    4633              : 
    4634         1960 :       *out_op0 = op0;
    4635         1960 :       *out_op1 = op1;
    4636         1960 :       return &row;
    4637              :     }
    4638              :   return NULL;
    4639              : }
    4640              : 
    4641              : /* Walk STMT's outer chain (kind OUTER), classify each leaf as a
    4642              :    long-multiply summand, optionally add the already-classified EXTRA,
    4643              :    and look the multiset up in long_mul_table for a result of type
    4644              :    LHS_TYPE.  CANDIDATE is passed to per-row extra_check predicates.
    4645              : 
    4646              :    If EXTRAS_OUT is non-NULL, leaves matching no summand are set aside
    4647              :    there instead of failing the match, and the caller must re-apply
    4648              :    them on top of the folded multiply.  A leaf that does match is
    4649              :    always consumed: if that makes the signature miss every row the
    4650              :    match fails, rather than retrying with the leaf demoted to an extra
    4651              :    (subset search would be exponential).
    4652              : 
    4653              :    Returns the matched row and the half-width operands via
    4654              :    OUT_OP0/OUT_OP1, or NULL on a miss.  No IR mutation.  */
    4655              : 
    4656              : static const long_mul_row *
    4657      3405795 : long_mul_classify_chain (gimple *stmt, tree_code outer, tree lhs_type,
    4658              :                          gimple *candidate, const long_mul_summand *extra,
    4659              :                          vec<tree> *extras_out,
    4660              :                          tree *out_op0, tree *out_op1)
    4661              : {
    4662      3405795 :   auto_vec<tree, LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS> leaves;
    4663      3405795 :   gimple *shared_def = NULL;
    4664      3405795 :   if (!long_mul_linearize_chain (stmt, outer, leaves, &shared_def))
    4665              :     return NULL;
    4666              : 
    4667      3405000 :   auto_vec<long_mul_summand,
    4668      3405000 :            LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS + 1> summands;
    4669     17182072 :   for (tree leaf : leaves)
    4670              :     {
    4671      6971639 :       if (long_mul_summand s{leaf})
    4672              :         {
    4673        55611 :           gcc_checking_assert (s.kind != LMK_INVALID);
    4674        55611 :           summands.quick_push (s);
    4675              :         }
    4676     13831780 :       else if (extras_out && extras_out->length () < LONG_MUL_MAX_EXTRAS)
    4677      6911461 :         extras_out->safe_push (leaf);
    4678              :       else
    4679              :         {
    4680         4567 :           long_mul_hint_shared_intermediate (shared_def);
    4681         4567 :           return NULL;
    4682              :         }
    4683              :     }
    4684      3400433 :   if (extra)
    4685              :     {
    4686          354 :       gcc_checking_assert (extra->kind != LMK_INVALID);
    4687          354 :       summands.quick_push (*extra);
    4688              :     }
    4689      6805433 :   if (summands.length () < 2
    4690      3400433 :       || summands.length () > LONG_MUL_MAX_SUMMANDS)
    4691              :     return NULL;
    4692        18763 :   summands.qsort (long_mul_summand_compare);
    4693              : 
    4694        18763 :   const long_mul_row *row
    4695        18763 :     = long_mul_classify_match (summands, lhs_type, outer, candidate,
    4696              :                                out_op0, out_op1);
    4697        18763 :   if (!row)
    4698        16803 :     long_mul_hint_shared_intermediate (shared_def);
    4699              :   return row;
    4700      3405000 : }
    4701              : 
    4702              : /* Top-level entry for long-multiply folding.  Walks STMT's outer
    4703              :    addition or BIT_IOR chain, classifies the summands, and dispatches
    4704              :    to create_mul_high_seq / create_mul_low_seq if the multiset matches
    4705              :    a known long-multiply form.  Returns true on success.  */
    4706              : 
    4707              : static bool
    4708     10235596 : match_long_mul (gassign *stmt)
    4709              : {
    4710     10235596 :   tree_code outer = gimple_assign_rhs_code (stmt);
    4711     10235596 :   if (outer != PLUS_EXPR && outer != BIT_IOR_EXPR)
    4712              :     return false;
    4713              : 
    4714              :   /* Skip non-candidate adds (signed, pointer, odd-width) before walking the
    4715              :      chain.  No legitimate long-mul leaf has a type the atoms would reject.
    4716              :      This just avoids the linearize/classify work on every other PLUS/IOR.  */
    4717     10235596 :   tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
    4718     10235596 :   if (!INTEGRAL_TYPE_P (lhs_type)
    4719      8944543 :       || !TYPE_UNSIGNED (lhs_type)
    4720     14044510 :       || TYPE_PRECISION (lhs_type) % 2 != 0)
    4721              :     return false;
    4722              : 
    4723              :   /* Only start at the end of a chain: a consumer with the same code
    4724              :      linearizes through this statement anyway, so starting here is
    4725              :      redundant.  A consumer in another block does not count -- folding
    4726              :      at the later use could sink a loop-invariant multiply into a
    4727              :      loop.  */
    4728      3572924 :   use_operand_p use_p;
    4729      3572924 :   gimple *use_stmt;
    4730      3572924 :   if (single_imm_use (gimple_assign_lhs (stmt), &use_p, &use_stmt)
    4731      2525115 :       && is_gimple_assign (use_stmt)
    4732      1383742 :       && gimple_assign_rhs_code (use_stmt) == outer
    4733      3747553 :       && gimple_bb (use_stmt) == gimple_bb (stmt))
    4734              :     return false;
    4735              : 
    4736      3405165 :   auto_vec<tree, LONG_MUL_MAX_EXTRAS> extras;
    4737      3405165 :   tree op0, op1;
    4738      3405165 :   const long_mul_row *row
    4739      3405165 :     = long_mul_classify_chain (stmt, outer, lhs_type, stmt, NULL, &extras,
    4740              :                                &op0, &op1);
    4741      3405165 :   if (!row)
    4742              :     return false;
    4743              : 
    4744         1946 :   if (row->part == long_mul_row::HIGH_PART)
    4745              :     {
    4746         1912 :       create_mul_high_seq (op0, op1, stmt, extras, outer);
    4747         1912 :       if (dump_file && (dump_flags & TDF_DETAILS))
    4748           35 :         fprintf (dump_file, "Long multiplication high part folded.\n");
    4749              :       return true;
    4750              :     }
    4751           34 :   create_mul_low_seq (op0, op1, stmt, extras, outer);
    4752           34 :   if (dump_file && (dump_flags & TDF_DETAILS))
    4753           10 :     fprintf (dump_file, "Long multiplication low part folded.\n");
    4754              :   return true;
    4755      3405165 : }
    4756              : 
    4757              : /* PHI-driven entry for long-multiply folding.  When PHI's value
    4758              :    flattens to base + (carry << N), probe sum to classify the carry
    4759              :    kind, linearize base for the remaining high-part summands, and run
    4760              :    the long-multiply table.  On a hit, emit a 2N-bit multiply at the
    4761              :    top of the join block with PHI_RES as its LHS and remove the PHI.
    4762              :    Otherwise leave the IR untouched.  Only HIGH_PART rows are
    4763              :    reachable.  LOW_PART rows are BIT_IOR-shaped and never produce a
    4764              :    carry PHI.  */
    4765              : 
    4766              : static bool
    4767      8856254 : match_long_mul_phi (gphi *phi)
    4768              : {
    4769      8856254 :   tree phi_res = gimple_phi_result (phi);
    4770      8856254 :   tree lhs_type = TREE_TYPE (phi_res);
    4771      6266098 :   if (!INTEGRAL_TYPE_P (lhs_type) || !TYPE_UNSIGNED (lhs_type)
    4772     11633169 :       || TYPE_PRECISION (lhs_type) % 2 != 0)
    4773              :     return false;
    4774              : 
    4775      2046507 :   tree cca_ops[4];
    4776      2046507 :   if (!gimple_cond_carry_add (phi_res, cca_ops, NULL))
    4777              :     return false;
    4778        19233 :   tree cmp_lhs = cca_ops[0];
    4779        19233 :   tree sum = cca_ops[1];
    4780        19233 :   tree base = cca_ops[2];
    4781              : 
    4782              :   /* Classify sum and populate the carry summand directly.  Most
    4783              :      specific first, mirroring long_mul_classify_carry's order.  */
    4784        19233 :   long_mul_summand carry{};
    4785        19233 :   tree sum_ops[LONG_MUL_MAX_CAPTURES];
    4786        19233 :   unsigned HOST_WIDE_INT shift_amt
    4787        19233 :     = wi::exact_log2 (wi::to_wide (cca_ops[3]));
    4788        19233 :   unsigned HOST_WIDE_INT halfwidth = TYPE_PRECISION (lhs_type) / 2;
    4789        19233 :   carry.shift = shift_amt;
    4790              : 
    4791        19233 :   if (shift_amt == halfwidth
    4792        19233 :       && gimple_mul_low_sum (sum, sum_ops, NULL))
    4793              :     {
    4794              :       /* mul_carry_low_sum's flat form ties the outer lshift amount to
    4795              :          the inner mul_hi's INTEGER_CST@0 via match.pd capture re-use;
    4796              :          the PHI form has no such tie, so gate on shift_amt explicitly.  */
    4797           14 :       carry.kind = LMK_CARRY_LOW_SUM;
    4798           14 :       carry.op0 = sum_ops[0];
    4799           14 :       carry.op1 = sum_ops[1];
    4800           14 :       carry.hilo[0] = cmp_lhs;
    4801           14 :       carry.hilo[1] = sum_ops[2];
    4802           14 :       carry.hilo[2] = sum_ops[3];
    4803              :     }
    4804        19219 :   else if (shift_amt == halfwidth
    4805        19219 :            && gimple_mul_cross_sum (sum, sum_ops, NULL))
    4806              :     {
    4807              :       /* mul_cross_sum is just (plus:c @0 @1) with no half-width
    4808              :          constraint.  Gate here to mirror mul_carry_cross_sum;
    4809              :          a mismatch falls through to the LMK_CARRY_LOW branch.  */
    4810            1 :       carry.kind = LMK_CARRY_CROSS_SUM;
    4811            1 :       carry.hilo[0] = cmp_lhs;
    4812            1 :       carry.hilo[1] = sum_ops[0];
    4813            1 :       carry.hilo[2] = sum_ops[1];
    4814              :     }
    4815        19218 :   else if (shift_amt == 0 && TREE_CODE (sum) == SSA_NAME)
    4816              :     {
    4817        18070 :       gimple *def = SSA_NAME_DEF_STMT (sum);
    4818        18070 :       if (!is_gimple_assign (def)
    4819        18070 :           || gimple_assign_rhs_code (def) != PLUS_EXPR)
    4820              :         return false;
    4821         4744 :       tree p1 = gimple_assign_rhs1 (def);
    4822         4744 :       tree p2 = gimple_assign_rhs2 (def);
    4823         4744 :       if (p1 != cmp_lhs && p2 != cmp_lhs)
    4824              :         return false;
    4825         4009 :       carry.kind = LMK_CARRY_LOW;
    4826         4009 :       carry.carry_a = cmp_lhs;
    4827         4009 :       carry.carry_b = p1 == cmp_lhs ? p2 : p1;
    4828         4009 :     }
    4829              :   else
    4830              :     return false;
    4831              : 
    4832              :   /* Linearize base, the rest of the high-part chain.  */
    4833         4024 :   if (TREE_CODE (base) != SSA_NAME)
    4834              :     return false;
    4835         4024 :   gimple *base_def = SSA_NAME_DEF_STMT (base);
    4836         4024 :   if (!is_gimple_assign (base_def)
    4837         4024 :       || gimple_assign_rhs_code (base_def) != PLUS_EXPR)
    4838              :     return false;
    4839              : 
    4840          630 :   tree op0, op1;
    4841          630 :   const long_mul_row *row
    4842          630 :     = long_mul_classify_chain (base_def, PLUS_EXPR, lhs_type, phi, &carry,
    4843              :                                NULL, &op0, &op1);
    4844          630 :   if (!row || row->part != long_mul_row::HIGH_PART)
    4845              :     return false;
    4846              : 
    4847           14 :   gimple_seq seq = NULL;
    4848           14 :   build_mul_high_seq (op0, op1, phi_res, gimple_location (phi), &seq);
    4849           14 :   gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (phi));
    4850           14 :   gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
    4851           14 :   gimple_stmt_iterator psi = gsi_for_stmt (phi);
    4852           14 :   remove_phi_node (&psi, false);
    4853           14 :   if (dump_file && (dump_flags & TDF_DETAILS))
    4854            4 :     fprintf (dump_file,
    4855              :              "Long multiplication high part folded (carry PHI).\n");
    4856              :   return true;
    4857              : }
    4858              : 
    4859              : /* Verify if we have the following structure:
    4860              : 
    4861              :    iftmp1 = PHI <pow2a, pow2b, pow2c, ...>
    4862              :    _ssa1 = _ssa2 MOD|DIV iftmp1;
    4863              :    _ssa3 = _ssa1 EQ|NE 0;
    4864              : 
    4865              :    And, if the right conditions are met, change the PHI args
    4866              :    and "_ssa1" stmt to a cheaper alternative.
    4867              : 
    4868              :    - for MOD, if either "_ssa2" is known to be positive or
    4869              :    "_ssa1" is used just in zero comparisons:
    4870              : 
    4871              :    iftmp1 = PHI <(pow2a - 1), (pow2b - 1), (pow2c - 1), ...>
    4872              :    _ssa1 = _ssa2 & iftmp1;
    4873              : 
    4874              :    - for DIV, if "_ssa2" is known to be positive:
    4875              : 
    4876              :    iftmp1 = PHI <log2 (pow2a), log2 (pow2b), log2 (pow2c), ...>
    4877              :    _ssa1 = _ssa2 >> iftmp1;  */
    4878              : static bool
    4879       134686 : simplify_phi_result_movdiv (gimple *stmt, tree_code code)
    4880              : {
    4881       134686 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    4882       134686 :   tree_code new_code;
    4883       134686 :   bool needs_reset = false;
    4884              : 
    4885              :   /* Skip complex types (PR127163) */
    4886       134686 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
    4887              :     return false;
    4888              : 
    4889        68081 :   switch (code)
    4890              :     {
    4891          228 :       case TRUNC_MOD_EXPR:
    4892          228 :       case FLOOR_MOD_EXPR:
    4893          228 :         if (!tree_expr_nonnegative_p (rhs1))
    4894              :           {
    4895              :             /* For a negative X, the truncating X % 2^k and X & (2^k - 1)
    4896              :                differ (e.g. -3 % 4 is -3 but -3 & 3 is 1), though both are
    4897              :                zero for the same X.  That is enough when the result is only
    4898              :                compared with zero, but the range recorded for the MOD result
    4899              :                no longer holds for the AND, so reset it.  */
    4900          143 :             needs_reset = true;
    4901          143 :             if (!use_in_zero_equality (gimple_assign_lhs (stmt), true))
    4902              :               return false;
    4903              :           }
    4904              : 
    4905              :         new_code = BIT_AND_EXPR;
    4906              :         break;
    4907              : 
    4908          723 :       case TRUNC_DIV_EXPR:
    4909          723 :       case FLOOR_DIV_EXPR:
    4910          723 :       case EXACT_DIV_EXPR:
    4911          723 :         if (!tree_expr_nonnegative_p (rhs1))
    4912              :           return false;
    4913              : 
    4914              :         new_code = RSHIFT_EXPR;
    4915              :         break;
    4916              : 
    4917              :      default:
    4918              :         return false;
    4919              :     }
    4920              : 
    4921          572 :   gphi *phi = as_a<gphi *> (SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt)));
    4922              : 
    4923         1199 :   for (unsigned int i = 0; i < gimple_phi_num_args (phi); i++)
    4924          614 :     if (!integer_pow2p (gimple_phi_arg_def (phi, i))
    4925          614 :         || tree_int_cst_sgn (gimple_phi_arg_def (phi, i)) < 0)
    4926              :       return false;
    4927              : 
    4928           13 :   tree type = TREE_TYPE (gimple_phi_result (phi));
    4929           13 :   tree new_phires = make_ssa_name (type);
    4930           13 :   gphi *new_phi = create_phi_node (new_phires, phi->bb);
    4931              : 
    4932           54 :   for (unsigned int i = 0; i < gimple_phi_num_args (phi); i++)
    4933              :     {
    4934           28 :       tree phi_arg = gimple_phi_arg_def (phi, i);
    4935           28 :       edge e = gimple_phi_arg_edge (phi, i);
    4936           28 :       tree arg;
    4937              : 
    4938           28 :       if (new_code == RSHIFT_EXPR)
    4939            4 :         arg = wide_int_to_tree (type, wi::exact_log2 (wi::to_wide (phi_arg)));
    4940              :       else
    4941           24 :         arg = wide_int_to_tree (type, wi::to_wide (phi_arg) - 1);
    4942              : 
    4943           28 :       add_phi_arg (new_phi, arg, e,
    4944           28 :                    gimple_phi_arg_location (phi, e->dest_idx));
    4945              :     }
    4946              : 
    4947              :   /* Add a gimple_convert to integer_type_node for new_phires
    4948              :      since it might be a long long which we want to convert
    4949              :      into an integer or a bit_int that we want to convert into
    4950              :      an integer.  */
    4951           13 :   gimple_stmt_iterator gsi;
    4952           13 :   if (new_code == RSHIFT_EXPR)
    4953              :     {
    4954            2 :       gsi = gsi_for_stmt (stmt);
    4955            2 :       new_phires = gimple_convert (&gsi, true, GSI_SAME_STMT,
    4956              :                                    gimple_location (stmt),
    4957              :                                    integer_type_node, new_phires);
    4958              :     }
    4959              : 
    4960           13 :   gimple_assign_set_rhs2 (stmt, new_phires);
    4961           13 :   gimple_assign_set_rhs_code (stmt, new_code);
    4962           13 :   if (needs_reset)
    4963            8 :     reset_flow_sensitive_info (gimple_assign_lhs (stmt));
    4964           13 :   update_stmt (stmt);
    4965              : 
    4966           13 :   gsi = gsi_for_phi (phi);
    4967           13 :   remove_phi_node (&gsi, true);
    4968              : 
    4969           13 :   return true;
    4970              : }
    4971              : 
    4972              : /* Determine whether applying the 2 permutations (mask1 then mask2)
    4973              :    gives back one of the input.  */
    4974              : 
    4975              : static int
    4976           46 : is_combined_permutation_identity (tree mask1, tree mask2)
    4977              : {
    4978           46 :   tree mask;
    4979           46 :   unsigned HOST_WIDE_INT nelts, i, j;
    4980           46 :   bool maybe_identity1 = true;
    4981           46 :   bool maybe_identity2 = true;
    4982              : 
    4983           46 :   gcc_checking_assert (TREE_CODE (mask1) == VECTOR_CST
    4984              :                        && TREE_CODE (mask2) == VECTOR_CST);
    4985              : 
    4986              :   /* For VLA masks, check for the following pattern:
    4987              :      v1 = VEC_PERM_EXPR (v0, ..., mask1)
    4988              :      v2 = VEC_PERM_EXPR (v1, ..., mask2)
    4989              :      -->
    4990              :      v2 = v0
    4991              :      if mask1 == mask2 == {nelts - 1, nelts - 2, ...}.  */
    4992              : 
    4993           46 :   if (operand_equal_p (mask1, mask2, 0)
    4994           46 :       && !VECTOR_CST_NELTS (mask1).is_constant ())
    4995              :     {
    4996              :       vec_perm_builder builder;
    4997              :       if (tree_to_vec_perm_builder (&builder, mask1))
    4998              :         {
    4999              :           poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask1));
    5000              :           vec_perm_indices sel (builder, 1, nelts);
    5001              :           if (sel.series_p (0, 1, nelts - 1, -1))
    5002              :             return 1;
    5003              :         }
    5004              :     }
    5005              : 
    5006           46 :   mask = fold_ternary (VEC_PERM_EXPR, TREE_TYPE (mask1), mask1, mask1, mask2);
    5007           46 :   if (mask == NULL_TREE || TREE_CODE (mask) != VECTOR_CST)
    5008              :     return 0;
    5009              : 
    5010           46 :   if (!VECTOR_CST_NELTS (mask).is_constant (&nelts))
    5011              :     return 0;
    5012           80 :   for (i = 0; i < nelts; i++)
    5013              :     {
    5014           80 :       tree val = VECTOR_CST_ELT (mask, i);
    5015           80 :       gcc_assert (TREE_CODE (val) == INTEGER_CST);
    5016           80 :       j = TREE_INT_CST_LOW (val) & (2 * nelts - 1);
    5017           80 :       if (j == i)
    5018              :         maybe_identity2 = false;
    5019           63 :       else if (j == i + nelts)
    5020              :         maybe_identity1 = false;
    5021              :       else
    5022              :         return 0;
    5023              :     }
    5024            0 :   return maybe_identity1 ? 1 : maybe_identity2 ? 2 : 0;
    5025              : }
    5026              : 
    5027              : /* Combine a shuffle with its arguments.  Returns true if there were any
    5028              :    changes made.  */
    5029              : 
    5030              : static bool
    5031       193226 : simplify_permutation (gimple_stmt_iterator *gsi)
    5032              : {
    5033       193226 :   gimple *stmt = gsi_stmt (*gsi);
    5034       193226 :   gimple *def_stmt = NULL;
    5035       193226 :   tree op0, op1, op2, op3, arg0, arg1;
    5036       193226 :   enum tree_code code, code2 = ERROR_MARK;
    5037       193226 :   bool single_use_op0 = false;
    5038              : 
    5039       193226 :   gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
    5040              : 
    5041       193226 :   op0 = gimple_assign_rhs1 (stmt);
    5042       193226 :   op1 = gimple_assign_rhs2 (stmt);
    5043       193226 :   op2 = gimple_assign_rhs3 (stmt);
    5044              : 
    5045       193226 :   if (TREE_CODE (op2) != VECTOR_CST)
    5046              :     return false;
    5047              : 
    5048       190434 :   if (TREE_CODE (op0) == VECTOR_CST)
    5049              :     {
    5050              :       code = VECTOR_CST;
    5051              :       arg0 = op0;
    5052              :     }
    5053       188265 :   else if (TREE_CODE (op0) == SSA_NAME)
    5054              :     {
    5055       188265 :       def_stmt = get_prop_source_stmt (op0, false, &single_use_op0);
    5056       188265 :       if (!def_stmt)
    5057              :         return false;
    5058       179756 :       code = gimple_assign_rhs_code (def_stmt);
    5059       179756 :       if (code == VIEW_CONVERT_EXPR)
    5060              :         {
    5061         1636 :           tree rhs = gimple_assign_rhs1 (def_stmt);
    5062         1636 :           tree name = TREE_OPERAND (rhs, 0);
    5063         1636 :           if (TREE_CODE (name) != SSA_NAME)
    5064              :             return false;
    5065         1636 :           if (!has_single_use (name))
    5066          254 :             single_use_op0 = false;
    5067              :           /* Here we update the def_stmt through this VIEW_CONVERT_EXPR,
    5068              :              but still keep the code to indicate it comes from
    5069              :              VIEW_CONVERT_EXPR.  */
    5070         1636 :           def_stmt = SSA_NAME_DEF_STMT (name);
    5071         1636 :           if (!def_stmt || !is_gimple_assign (def_stmt))
    5072              :             return false;
    5073          849 :           if (gimple_assign_rhs_code (def_stmt) != CONSTRUCTOR)
    5074              :             return false;
    5075              :         }
    5076       178386 :       if (!can_propagate_from (def_stmt))
    5077              :         return false;
    5078        24663 :       arg0 = gimple_assign_rhs1 (def_stmt);
    5079              :     }
    5080              :   else
    5081              :     return false;
    5082              : 
    5083              :   /* Two consecutive shuffles.  */
    5084        24663 :   if (code == VEC_PERM_EXPR)
    5085              :     {
    5086         6757 :       tree orig;
    5087         6757 :       int ident;
    5088              : 
    5089         6757 :       if (op0 != op1)
    5090              :         return false;
    5091           46 :       op3 = gimple_assign_rhs3 (def_stmt);
    5092           46 :       if (TREE_CODE (op3) != VECTOR_CST)
    5093              :         return false;
    5094              :       /* is_combined_permutation_identity assumes that both permutes and
    5095              :          their inputs have the same number of elements.  Mixed widths are
    5096              :          left to the match.pd VEC_PERM merge patterns.  */
    5097           46 :       if (maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)),
    5098           46 :                     TYPE_VECTOR_SUBPARTS (TREE_TYPE (op3)))
    5099           92 :           || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op0)),
    5100              :                        TYPE_VECTOR_SUBPARTS
    5101           46 :                          (TREE_TYPE (gimple_assign_rhs1 (def_stmt)))))
    5102              :         return false;
    5103           46 :       ident = is_combined_permutation_identity (op3, op2);
    5104           46 :       if (!ident)
    5105              :         return false;
    5106            0 :       orig = (ident == 1) ? gimple_assign_rhs1 (def_stmt)
    5107            0 :                           : gimple_assign_rhs2 (def_stmt);
    5108            0 :       gimple_assign_set_rhs1 (stmt, unshare_expr (orig));
    5109            0 :       gimple_assign_set_rhs_code (stmt, TREE_CODE (orig));
    5110            0 :       gimple_set_num_ops (stmt, 2);
    5111            0 :       update_stmt (stmt);
    5112            0 :       remove_prop_source_from_use (op0);
    5113            0 :       return true;
    5114              :     }
    5115        20075 :   else if (code == CONSTRUCTOR
    5116        20075 :            || code == VECTOR_CST
    5117              :            || code == VIEW_CONVERT_EXPR)
    5118              :     {
    5119         5010 :       if (op0 != op1)
    5120              :         {
    5121         4813 :           if (TREE_CODE (op0) == SSA_NAME && !single_use_op0)
    5122              :             return false;
    5123              : 
    5124         4130 :           if (TREE_CODE (op1) == VECTOR_CST)
    5125              :             arg1 = op1;
    5126         3497 :           else if (TREE_CODE (op1) == SSA_NAME)
    5127              :             {
    5128         3497 :               gimple *def_stmt2 = get_prop_source_stmt (op1, true, NULL);
    5129         3497 :               if (!def_stmt2)
    5130              :                 return false;
    5131         1698 :               code2 = gimple_assign_rhs_code (def_stmt2);
    5132         1698 :               if (code2 == VIEW_CONVERT_EXPR)
    5133              :                 {
    5134            0 :                   tree rhs = gimple_assign_rhs1 (def_stmt2);
    5135            0 :                   tree name = TREE_OPERAND (rhs, 0);
    5136            0 :                   if (TREE_CODE (name) != SSA_NAME)
    5137              :                     return false;
    5138            0 :                   if (!has_single_use (name))
    5139              :                     return false;
    5140            0 :                   def_stmt2 = SSA_NAME_DEF_STMT (name);
    5141            0 :                   if (!def_stmt2 || !is_gimple_assign (def_stmt2))
    5142              :                     return false;
    5143            0 :                   if (gimple_assign_rhs_code (def_stmt2) != CONSTRUCTOR)
    5144              :                     return false;
    5145              :                 }
    5146         1698 :               else if (code2 != CONSTRUCTOR && code2 != VECTOR_CST)
    5147              :                 return false;
    5148         1485 :               if (!can_propagate_from (def_stmt2))
    5149              :                 return false;
    5150         1485 :               arg1 = gimple_assign_rhs1 (def_stmt2);
    5151              :             }
    5152              :           else
    5153              :             return false;
    5154              :         }
    5155              :       else
    5156              :         {
    5157              :           /* Already used twice in this statement.  */
    5158          197 :           if (TREE_CODE (op0) == SSA_NAME && num_imm_uses (op0) > 2)
    5159              :             return false;
    5160              :           arg1 = arg0;
    5161              :         }
    5162              : 
    5163              :       /* If there are any VIEW_CONVERT_EXPRs found when finding permutation
    5164              :          operands source, check whether it's valid to transform and prepare
    5165              :          the required new operands.  */
    5166         2247 :       if (code == VIEW_CONVERT_EXPR || code2 == VIEW_CONVERT_EXPR)
    5167              :         {
    5168              :           /* Figure out the target vector type to which operands should be
    5169              :              converted.  If both are CONSTRUCTOR, the types should be the
    5170              :              same, otherwise, use the one of CONSTRUCTOR.  */
    5171           24 :           tree tgt_type = NULL_TREE;
    5172           24 :           if (code == VIEW_CONVERT_EXPR)
    5173              :             {
    5174           24 :               gcc_assert (gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR);
    5175           24 :               code = CONSTRUCTOR;
    5176           24 :               tgt_type = TREE_TYPE (arg0);
    5177              :             }
    5178           24 :           if (code2 == VIEW_CONVERT_EXPR)
    5179              :             {
    5180            0 :               tree arg1_type = TREE_TYPE (arg1);
    5181            0 :               if (tgt_type == NULL_TREE)
    5182              :                 tgt_type = arg1_type;
    5183            0 :               else if (tgt_type != arg1_type)
    5184           23 :                 return false;
    5185              :             }
    5186              : 
    5187           24 :           if (!VECTOR_TYPE_P (tgt_type))
    5188              :             return false;
    5189           24 :           tree op2_type = TREE_TYPE (op2);
    5190              : 
    5191              :           /* Figure out the shrunk factor.  */
    5192           24 :           poly_uint64 tgt_units = TYPE_VECTOR_SUBPARTS (tgt_type);
    5193           24 :           poly_uint64 op2_units = TYPE_VECTOR_SUBPARTS (op2_type);
    5194           24 :           if (maybe_gt (tgt_units, op2_units))
    5195              :             return false;
    5196           24 :           unsigned int factor;
    5197           47 :           if (!constant_multiple_p (op2_units, tgt_units, &factor))
    5198              :             return false;
    5199              : 
    5200              :           /* Build the new permutation control vector as target vector.  */
    5201           24 :           vec_perm_builder builder;
    5202           24 :           if (!tree_to_vec_perm_builder (&builder, op2))
    5203              :             return false;
    5204           24 :           vec_perm_indices indices (builder, 2, op2_units);
    5205           24 :           vec_perm_indices new_indices;
    5206           24 :           if (new_indices.new_shrunk_vector (indices, factor))
    5207              :             {
    5208            1 :               tree mask_type = tgt_type;
    5209            1 :               if (!VECTOR_INTEGER_TYPE_P (mask_type))
    5210              :                 {
    5211            0 :                   tree elem_type = TREE_TYPE (mask_type);
    5212            0 :                   unsigned elem_size = TREE_INT_CST_LOW (TYPE_SIZE (elem_type));
    5213            0 :                   tree int_type = build_nonstandard_integer_type (elem_size, 0);
    5214            0 :                   mask_type = build_vector_type (int_type, tgt_units);
    5215              :                 }
    5216            1 :               op2 = vec_perm_indices_to_tree (mask_type, new_indices);
    5217              :             }
    5218              :           else
    5219           23 :             return false;
    5220              : 
    5221              :           /* Convert the VECTOR_CST to the appropriate vector type.  */
    5222            1 :           if (tgt_type != TREE_TYPE (arg0))
    5223            0 :             arg0 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg0);
    5224            1 :           else if (tgt_type != TREE_TYPE (arg1))
    5225            0 :             arg1 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg1);
    5226           47 :         }
    5227              : 
    5228              :       /* VIEW_CONVERT_EXPR should be updated to CONSTRUCTOR before.  */
    5229         2224 :       gcc_assert (code == CONSTRUCTOR || code == VECTOR_CST);
    5230              : 
    5231              :       /* Shuffle of a constructor.  */
    5232         2224 :       tree res_type
    5233         2224 :         = build_vector_type (TREE_TYPE (TREE_TYPE (arg0)),
    5234         2224 :                              TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)));
    5235         2224 :       tree opt = fold_ternary (VEC_PERM_EXPR, res_type, arg0, arg1, op2);
    5236         2224 :       if (!opt
    5237          280 :           || (TREE_CODE (opt) != CONSTRUCTOR && TREE_CODE (opt) != VECTOR_CST))
    5238              :         return false;
    5239              :       /* Found VIEW_CONVERT_EXPR before, need one explicit conversion.  */
    5240          280 :       if (res_type != TREE_TYPE (op0))
    5241              :         {
    5242            1 :           tree name = make_ssa_name (TREE_TYPE (opt));
    5243            1 :           gimple *ass_stmt = gimple_build_assign (name, opt);
    5244            1 :           gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
    5245            1 :           opt = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (op0), name);
    5246              :         }
    5247          280 :       gimple_assign_set_rhs_from_tree (gsi, opt);
    5248          280 :       update_stmt (gsi_stmt (*gsi));
    5249          280 :       if (TREE_CODE (op0) == SSA_NAME)
    5250            1 :         remove_prop_source_from_use (op0);
    5251          280 :       if (op0 != op1 && TREE_CODE (op1) == SSA_NAME)
    5252            0 :         remove_prop_source_from_use (op1);
    5253              :       return true;
    5254              :     }
    5255              : 
    5256              :   return false;
    5257              : }
    5258              : 
    5259              : /* Get the BIT_FIELD_REF definition of VAL, if any, looking through
    5260              :    conversions with code CONV_CODE or update it if still ERROR_MARK.
    5261              :    Return NULL_TREE if no such matching def was found.  */
    5262              : 
    5263              : static tree
    5264       437871 : get_bit_field_ref_def (tree val, enum tree_code &conv_code)
    5265              : {
    5266       437871 :   if (TREE_CODE (val) != SSA_NAME)
    5267              :     return NULL_TREE ;
    5268       407040 :   gimple *def_stmt = get_prop_source_stmt (val, false, NULL);
    5269       407040 :   if (!def_stmt)
    5270              :     return NULL_TREE;
    5271       325882 :   enum tree_code code = gimple_assign_rhs_code (def_stmt);
    5272       325882 :   if (code == FLOAT_EXPR
    5273       325882 :       || code == FIX_TRUNC_EXPR
    5274              :       || CONVERT_EXPR_CODE_P (code))
    5275              :     {
    5276       188476 :       tree op1 = gimple_assign_rhs1 (def_stmt);
    5277       188476 :       if (conv_code == ERROR_MARK)
    5278        90240 :         conv_code = code;
    5279        98236 :       else if (conv_code != code)
    5280              :         return NULL_TREE;
    5281       188451 :       if (TREE_CODE (op1) != SSA_NAME)
    5282              :         return NULL_TREE;
    5283        79950 :       def_stmt = SSA_NAME_DEF_STMT (op1);
    5284        79950 :       if (! is_gimple_assign (def_stmt))
    5285              :         return NULL_TREE;
    5286        63754 :       code = gimple_assign_rhs_code (def_stmt);
    5287              :     }
    5288       201160 :   if (code != BIT_FIELD_REF)
    5289              :     return NULL_TREE;
    5290        23822 :   return gimple_assign_rhs1 (def_stmt);
    5291              : }
    5292              : 
    5293              : /* Recognize a VEC_PERM_EXPR.  Returns true if there were any changes.  */
    5294              : 
    5295              : static bool
    5296       168113 : simplify_vector_constructor (gimple_stmt_iterator *gsi)
    5297              : {
    5298       168113 :   gimple *stmt = gsi_stmt (*gsi);
    5299       168113 :   tree op, orig[2], type;
    5300       168113 :   unsigned i;
    5301       168113 :   unsigned HOST_WIDE_INT nelts;
    5302       168113 :   unsigned HOST_WIDE_INT refnelts;
    5303       168113 :   enum tree_code conv_code;
    5304       168113 :   constructor_elt *elt;
    5305              : 
    5306       168113 :   op = gimple_assign_rhs1 (stmt);
    5307       168113 :   type = TREE_TYPE (op);
    5308       168113 :   gcc_checking_assert (TREE_CODE (op) == CONSTRUCTOR
    5309              :                        && TREE_CODE (type) == VECTOR_TYPE);
    5310              : 
    5311       168113 :   if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
    5312              :     return false;
    5313              : 
    5314       168113 :   orig[0] = NULL;
    5315       168113 :   orig[1] = NULL;
    5316       168113 :   tree orig_elem_type[2] = {};
    5317       168113 :   conv_code = ERROR_MARK;
    5318       168113 :   bool maybe_ident = true;
    5319       168113 :   bool maybe_blend[2] = { true, true };
    5320       168113 :   tree one_constant = NULL_TREE;
    5321       168113 :   tree one_nonconstant = NULL_TREE;
    5322       168113 :   tree subelt;
    5323       168113 :   auto_vec<tree> constants;
    5324       168113 :   constants.safe_grow_cleared (nelts, true);
    5325       168113 :   auto_vec<std::pair<unsigned, unsigned>, 64> elts;
    5326       168113 :   unsigned int tsubelts = 0;
    5327       471489 :   FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (op), i, elt)
    5328              :     {
    5329       437871 :       tree ref, op1;
    5330       437871 :       unsigned int elem, src_elem_size;
    5331       437871 :       unsigned HOST_WIDE_INT nsubelts = 1;
    5332              : 
    5333       437871 :       if (i >= nelts)
    5334       168113 :         return false;
    5335              : 
    5336              :       /* Look for elements extracted and possibly converted from
    5337              :          another vector.  */
    5338       437871 :       op1 = get_bit_field_ref_def (elt->value, conv_code);
    5339       442196 :       if (op1
    5340        23822 :           && TREE_CODE ((ref = TREE_OPERAND (op1, 0))) == SSA_NAME
    5341              :           /* Do not extend the live range of an SSA name that occurs in an
    5342              :              abnormal PHI.  */
    5343         5070 :           && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ref)
    5344         5020 :           && VECTOR_TYPE_P (TREE_TYPE (ref))
    5345         5005 :           && (tree_nop_conversion_p (TREE_TYPE (op1),
    5346         5005 :                                      TREE_TYPE (TREE_TYPE (ref)))
    5347          815 :               || (VECTOR_TYPE_P (TREE_TYPE (op1))
    5348          135 :                   && tree_nop_conversion_p (TREE_TYPE (TREE_TYPE (op1)),
    5349          135 :                                             TREE_TYPE (TREE_TYPE (ref)))
    5350          135 :                   && TYPE_VECTOR_SUBPARTS (TREE_TYPE (op1))
    5351          135 :                         .is_constant (&nsubelts)))
    5352         4325 :           && constant_multiple_p (bit_field_size (op1), nsubelts,
    5353              :                                   &src_elem_size)
    5354       442196 :           && constant_multiple_p (bit_field_offset (op1), src_elem_size, &elem)
    5355       442196 :           && TYPE_VECTOR_SUBPARTS (TREE_TYPE (ref)).is_constant (&refnelts))
    5356              :         {
    5357              :           unsigned int j;
    5358         4685 :           for (j = 0; j < 2; ++j)
    5359              :             {
    5360         4660 :               if (!orig[j])
    5361              :                 {
    5362         1884 :                   if (j == 0
    5363         2084 :                       || useless_type_conversion_p (TREE_TYPE (orig[0]),
    5364          200 :                                                     TREE_TYPE (ref)))
    5365              :                     break;
    5366              :                 }
    5367         2776 :               else if (ref == orig[j])
    5368              :                 break;
    5369              :             }
    5370              :           /* Found a suitable vector element.  */
    5371         4325 :           if (j < 2)
    5372              :             {
    5373         4300 :               orig[j] = ref;
    5374              :               /* Track what element type was actually extracted (which may
    5375              :                  differ in signedness from the vector's element type due to
    5376              :                  tree_nop_conversion_p).  */
    5377         4300 :               if (!orig_elem_type[j])
    5378         1880 :                 orig_elem_type[j] = TREE_TYPE (op1);
    5379         4300 :               if (elem != i || j != 0)
    5380         2241 :                 maybe_ident = false;
    5381         4300 :               if (elem != i)
    5382         2180 :                 maybe_blend[j] = false;
    5383         8861 :               for (unsigned int k = 0; k < nsubelts; ++k)
    5384         4561 :                 elts.safe_push (std::make_pair (j, elem + k));
    5385         4300 :               tsubelts += nsubelts;
    5386         4300 :               continue;
    5387         4300 :             }
    5388              :           /* Else fallthru.  */
    5389              :         }
    5390              :       /* Handle elements not extracted from a vector.
    5391              :           1. constants by permuting with constant vector
    5392              :           2. a unique non-constant element by permuting with a splat vector  */
    5393       433571 :       if (orig[1]
    5394       266311 :           && orig[1] != error_mark_node)
    5395              :         return false;
    5396       433546 :       orig[1] = error_mark_node;
    5397       433546 :       if (VECTOR_TYPE_P (TREE_TYPE (elt->value))
    5398       433546 :           && !TYPE_VECTOR_SUBPARTS (TREE_TYPE (elt->value))
    5399         5803 :                         .is_constant (&nsubelts))
    5400              :         return false;
    5401       433546 :       if (CONSTANT_CLASS_P (elt->value))
    5402              :         {
    5403        30827 :           if (one_nonconstant)
    5404              :             return false;
    5405        21859 :           if (!one_constant)
    5406         9792 :             one_constant = TREE_CODE (elt->value) == VECTOR_CST
    5407         9792 :                            ? VECTOR_CST_ELT (elt->value, 0)
    5408              :                            : elt->value;
    5409        21859 :           if (TREE_CODE (elt->value) == VECTOR_CST)
    5410              :             {
    5411          777 :               for (unsigned int k = 0; k < nsubelts; k++)
    5412          587 :                 constants[tsubelts + k] = VECTOR_CST_ELT (elt->value, k);
    5413              :             }
    5414              :           else
    5415        21669 :             constants[tsubelts] = elt->value;
    5416              :         }
    5417              :       else
    5418              :         {
    5419       402719 :           if (one_constant)
    5420              :             return false;
    5421       393448 :           subelt = VECTOR_TYPE_P (TREE_TYPE (elt->value))
    5422       393448 :                    ? ssa_uniform_vector_p (elt->value)
    5423              :                    : elt->value;
    5424       393448 :           if (!subelt
    5425              :               /* The splat of the element of a uniform vector is a new use
    5426              :                  of that element.  Do not extend the live range of an SSA
    5427              :                  name that occurs in an abnormal PHI.  */
    5428       393448 :               || (VECTOR_TYPE_P (TREE_TYPE (elt->value))
    5429          123 :                   && TREE_CODE (subelt) == SSA_NAME
    5430          123 :                   && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (subelt)))
    5431              :             return false;
    5432       388087 :           if (!one_nonconstant)
    5433              :             one_nonconstant = subelt;
    5434       235980 :           else if (!operand_equal_p (one_nonconstant, subelt, 0))
    5435              :             return false;
    5436              :         }
    5437       598551 :       for (unsigned int k = 0; k < nsubelts; ++k)
    5438       299475 :         elts.safe_push (std::make_pair (1, tsubelts + k));
    5439       299076 :       tsubelts += nsubelts;
    5440       299076 :       maybe_ident = false;
    5441              :     }
    5442              : 
    5443        67236 :   if (elts.length () < nelts)
    5444              :     return false;
    5445              : 
    5446        32446 :   if (! orig[0]
    5447        32446 :       || ! VECTOR_TYPE_P (TREE_TYPE (orig[0])))
    5448              :     return false;
    5449         1199 :   refnelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (orig[0])).to_constant ();
    5450              :   /* We currently do not handle larger destination vectors.  */
    5451         1199 :   if (refnelts < nelts)
    5452              :     return false;
    5453              : 
    5454              :   /* Determine the element type for the conversion source.
    5455              :      As orig_elem_type keeps track of the original type, check
    5456              :      if we need to perform a sign swap after permuting.
    5457              :      We need to be able to construct a vector type from the element
    5458              :      type which is not possible for e.g. BitInt or pointers
    5459              :      so pun with an integer type if needed.   */
    5460          985 :   tree perm_eltype = TREE_TYPE (TREE_TYPE (orig[0]));
    5461          985 :   bool sign_change_p = false;
    5462          985 :   if (conv_code != ERROR_MARK
    5463          213 :       && orig_elem_type[0]
    5464         1198 :       && TYPE_SIGN (orig_elem_type[0]) != TYPE_SIGN (perm_eltype))
    5465              :     {
    5466           35 :       perm_eltype = signed_or_unsigned_type_for
    5467           35 :         (TYPE_UNSIGNED (orig_elem_type[0]), perm_eltype);
    5468           35 :       sign_change_p = true;
    5469              :     }
    5470          985 :   tree conv_src_type = build_vector_type (perm_eltype, nelts);
    5471              : 
    5472          985 :   if (maybe_ident)
    5473              :     {
    5474              :       /* When there is no conversion, use the target type directly.  */
    5475          196 :       if (conv_code == ERROR_MARK && nelts != refnelts)
    5476          196 :         conv_src_type = type;
    5477          196 :       if (conv_code != ERROR_MARK
    5478          196 :           && !supportable_convert_operation (conv_code, type, conv_src_type))
    5479              :         {
    5480              :           /* Only few targets implement direct conversion patterns so try
    5481              :              some simple special cases via VEC_[UN]PACK[_FLOAT]_LO_EXPR.  */
    5482          103 :           optab optab;
    5483          103 :           insn_code icode;
    5484          103 :           tree halfvectype, dblvectype;
    5485          103 :           enum tree_code unpack_op;
    5486              : 
    5487          103 :           if (!BYTES_BIG_ENDIAN)
    5488          183 :             unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
    5489          103 :                          ? VEC_UNPACK_FLOAT_LO_EXPR
    5490              :                          : VEC_UNPACK_LO_EXPR);
    5491              :           else
    5492              :             unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
    5493              :                          ? VEC_UNPACK_FLOAT_HI_EXPR
    5494              :                          : VEC_UNPACK_HI_EXPR);
    5495              : 
    5496              :           /* Conversions between DFP and FP have no special tree code
    5497              :              but we cannot handle those since all relevant vector conversion
    5498              :              optabs only have a single mode.  */
    5499           15 :           if (CONVERT_EXPR_CODE_P (conv_code)
    5500           88 :               && FLOAT_TYPE_P (TREE_TYPE (type))
    5501          119 :               && (DECIMAL_FLOAT_TYPE_P (TREE_TYPE (type))
    5502            8 :                   != DECIMAL_FLOAT_TYPE_P (TREE_TYPE (conv_src_type))))
    5503              :             return false;
    5504              : 
    5505           15 :           if (CONVERT_EXPR_CODE_P (conv_code)
    5506           87 :               && (2 * TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
    5507           87 :                   == TYPE_PRECISION (TREE_TYPE (type)))
    5508            6 :               && orig_elem_type[0]
    5509            6 :               && useless_type_conversion_p (orig_elem_type[0],
    5510            6 :                                             TREE_TYPE (TREE_TYPE (orig[0])))
    5511            6 :               && mode_for_vector (as_a <scalar_mode>
    5512            6 :                                   (TYPE_MODE (TREE_TYPE (TREE_TYPE (orig[0])))),
    5513           12 :                                   nelts * 2).exists ()
    5514            6 :               && (dblvectype
    5515            6 :                   = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
    5516            6 :                                        nelts * 2))
    5517              :               /* Only use it for vector modes or for vector booleans
    5518              :                  represented as scalar bitmasks.  See PR95528.  */
    5519            6 :               && (VECTOR_MODE_P (TYPE_MODE (dblvectype))
    5520            0 :                   || VECTOR_BOOLEAN_TYPE_P (dblvectype))
    5521            6 :               && (optab = optab_for_tree_code (unpack_op,
    5522              :                                                dblvectype,
    5523              :                                                optab_default))
    5524            6 :               && ((icode = optab_handler (optab, TYPE_MODE (dblvectype)))
    5525              :                   != CODE_FOR_nothing)
    5526          102 :               && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
    5527              :             {
    5528            0 :               gimple_seq stmts = NULL;
    5529            0 :               tree dbl;
    5530            0 :               if (refnelts == nelts)
    5531              :                 {
    5532              :                   /* ???  Paradoxical subregs don't exist, so insert into
    5533              :                      the lower half of a wider zero vector.  */
    5534            0 :                   dbl = gimple_build (&stmts, BIT_INSERT_EXPR, dblvectype,
    5535              :                                       build_zero_cst (dblvectype), orig[0],
    5536            0 :                                       bitsize_zero_node);
    5537              :                 }
    5538            0 :               else if (refnelts == 2 * nelts)
    5539              :                 dbl = orig[0];
    5540              :               else
    5541            0 :                 dbl = gimple_build (&stmts, BIT_FIELD_REF, dblvectype,
    5542            0 :                                     orig[0], TYPE_SIZE (dblvectype),
    5543            0 :                                     bitsize_zero_node);
    5544            0 :               gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    5545            0 :               gimple_assign_set_rhs_with_ops (gsi, unpack_op, dbl);
    5546              :             }
    5547           15 :           else if (CONVERT_EXPR_CODE_P (conv_code)
    5548           87 :                    && (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
    5549           87 :                        == 2 * TYPE_PRECISION (TREE_TYPE (type)))
    5550            5 :                    && orig_elem_type[0]
    5551            5 :                    && useless_type_conversion_p (orig_elem_type[0],
    5552            5 :                                                  TREE_TYPE (TREE_TYPE (orig[0])))
    5553            1 :                    && mode_for_vector (as_a <scalar_mode>
    5554            1 :                                          (TYPE_MODE
    5555              :                                            (TREE_TYPE (TREE_TYPE (orig[0])))),
    5556            2 :                                        nelts / 2).exists ()
    5557            1 :                    && (halfvectype
    5558            1 :                          = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
    5559            1 :                                               nelts / 2))
    5560              :                    /* Only use it for vector modes or for vector booleans
    5561              :                       represented as scalar bitmasks.  See PR95528.  */
    5562            1 :                    && (VECTOR_MODE_P (TYPE_MODE (halfvectype))
    5563            0 :                        || VECTOR_BOOLEAN_TYPE_P (halfvectype))
    5564            1 :                    && (optab = optab_for_tree_code (VEC_PACK_TRUNC_EXPR,
    5565              :                                                     halfvectype,
    5566              :                                                     optab_default))
    5567            1 :                    && ((icode = optab_handler (optab, TYPE_MODE (halfvectype)))
    5568              :                        != CODE_FOR_nothing)
    5569          103 :                    && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
    5570              :             {
    5571            0 :               gimple_seq stmts = NULL;
    5572            0 :               tree low = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
    5573            0 :                                        orig[0], TYPE_SIZE (halfvectype),
    5574            0 :                                        bitsize_zero_node);
    5575            0 :               tree hig = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
    5576            0 :                                        orig[0], TYPE_SIZE (halfvectype),
    5577            0 :                                        TYPE_SIZE (halfvectype));
    5578            0 :               gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    5579            0 :               gimple_assign_set_rhs_with_ops (gsi, VEC_PACK_TRUNC_EXPR,
    5580              :                                               low, hig);
    5581              :             }
    5582              :           else
    5583          102 :             return false;
    5584            0 :           update_stmt (gsi_stmt (*gsi));
    5585            0 :           return true;
    5586              :         }
    5587           93 :       if (nelts != refnelts)
    5588              :         {
    5589           15 :           gassign *lowpart
    5590           15 :             = gimple_build_assign (make_ssa_name (conv_src_type),
    5591              :                                    build3 (BIT_FIELD_REF, conv_src_type,
    5592           15 :                                            orig[0], TYPE_SIZE (conv_src_type),
    5593              :                                            bitsize_zero_node));
    5594           15 :           gsi_insert_before (gsi, lowpart, GSI_SAME_STMT);
    5595           15 :           orig[0] = gimple_assign_lhs (lowpart);
    5596              :         }
    5597           78 :       else if (sign_change_p)
    5598              :         {
    5599            0 :           gassign *conv
    5600            0 :             = gimple_build_assign (make_ssa_name (conv_src_type),
    5601              :                                    build1 (VIEW_CONVERT_EXPR, conv_src_type,
    5602              :                                            orig[0]));
    5603            0 :           gsi_insert_before (gsi, conv, GSI_SAME_STMT);
    5604            0 :           orig[0] = gimple_assign_lhs (conv);
    5605              :         }
    5606           93 :       if (conv_code == ERROR_MARK)
    5607              :         {
    5608           76 :           tree src_type = TREE_TYPE (orig[0]);
    5609           76 :           if (!useless_type_conversion_p (type, src_type))
    5610              :             {
    5611            0 :               gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
    5612              :                                     TYPE_VECTOR_SUBPARTS (src_type))
    5613              :                           && tree_nop_conversion_p (TREE_TYPE (type),
    5614              :                                                     TREE_TYPE (src_type)));
    5615            0 :               tree rhs = build1 (VIEW_CONVERT_EXPR, type, orig[0]);
    5616            0 :               orig[0] = make_ssa_name (type);
    5617            0 :               gassign *assign = gimple_build_assign (orig[0], rhs);
    5618            0 :               gsi_insert_before (gsi, assign, GSI_SAME_STMT);
    5619              :             }
    5620           76 :           gimple_assign_set_rhs_from_tree (gsi, orig[0]);
    5621              :         }
    5622              :       else
    5623           17 :         gimple_assign_set_rhs_with_ops (gsi, conv_code, orig[0],
    5624              :                                         NULL_TREE, NULL_TREE);
    5625              :     }
    5626              :   else
    5627              :     {
    5628              :       /* If we combine a vector with a non-vector avoid cases where
    5629              :          we'll obviously end up with more GIMPLE stmts which is when
    5630              :          we'll later not fold this to a single insert into the vector
    5631              :          and we had a single extract originally.  See PR92819.  */
    5632          789 :       if (nelts == 2
    5633          335 :           && refnelts > 2
    5634          123 :           && orig[1] == error_mark_node
    5635           31 :           && !maybe_blend[0])
    5636           92 :         return false;
    5637          764 :       tree mask_type, perm_type;
    5638          764 :       perm_type = TREE_TYPE (orig[0]);
    5639          764 :       if (conv_code != ERROR_MARK
    5640          764 :           && !supportable_convert_operation (conv_code, type, conv_src_type))
    5641              :         return false;
    5642              : 
    5643              :       /* Now that we know the number of elements of the source build the
    5644              :          permute vector.
    5645              :          ???  When the second vector has constant values we can shuffle
    5646              :          it and its source indexes to make the permutation supported.
    5647              :          For now it mimics a blend.  */
    5648          754 :       vec_perm_builder sel (refnelts, refnelts, 1);
    5649          754 :       bool all_same_p = true;
    5650         8416 :       for (i = 0; i < elts.length (); ++i)
    5651              :         {
    5652         3454 :           sel.quick_push (elts[i].second + elts[i].first * refnelts);
    5653         3454 :           all_same_p &= known_eq (sel[i], sel[0]);
    5654              :         }
    5655              :       /* And fill the tail with "something".  It's really don't care,
    5656              :          and ideally we'd allow VEC_PERM to have a smaller destination
    5657              :          vector.  As a heuristic:
    5658              : 
    5659              :          (a) if what we have so far duplicates a single element, make the
    5660              :              tail do the same
    5661              : 
    5662              :          (b) otherwise preserve a uniform orig[0].  This facilitates
    5663              :              later pattern-matching of VEC_PERM_EXPR to a BIT_INSERT_EXPR.  */
    5664         1372 :       for (; i < refnelts; ++i)
    5665         1236 :         sel.quick_push (all_same_p
    5666         1854 :                         ? sel[0]
    5667          156 :                         : (elts[0].second == 0 && elts[0].first == 0
    5668          920 :                            ? 0 : refnelts) + i);
    5669          953 :       vec_perm_indices indices (sel, orig[1] ? 2 : 1, refnelts);
    5670          754 :       machine_mode vmode = TYPE_MODE (perm_type);
    5671          754 :       if ((cfun->curr_properties & PROP_gimple_lvec)
    5672          754 :           && !can_vec_perm_const_p (vmode, vmode, indices))
    5673              :         return false;
    5674          697 :       mask_type = build_vector_type (ssizetype, refnelts);
    5675          697 :       tree op2 = vec_perm_indices_to_tree (mask_type, indices);
    5676          697 :       bool converted_orig1 = false;
    5677          697 :       gimple_seq stmts = NULL;
    5678          697 :       if (!orig[1])
    5679          174 :         orig[1] = orig[0];
    5680          523 :       else if (orig[1] == error_mark_node
    5681          401 :                && one_nonconstant)
    5682              :         {
    5683              :           /* ???  We can see if we can safely convert to the original
    5684              :              element type.  */
    5685           91 :           converted_orig1 = conv_code != ERROR_MARK;
    5686           91 :           tree target_type = converted_orig1 ? type : perm_type;
    5687           91 :           tree nonconstant_for_splat = one_nonconstant;
    5688              :           /* If there's a nop conversion between the target element type and
    5689              :              the nonconstant's type, convert it.  */
    5690           91 :           if (!useless_type_conversion_p (TREE_TYPE (target_type),
    5691           91 :                                           TREE_TYPE (one_nonconstant)))
    5692            0 :             nonconstant_for_splat
    5693            0 :               = gimple_build (&stmts, NOP_EXPR, TREE_TYPE (target_type),
    5694              :                               one_nonconstant);
    5695           91 :           orig[1] = gimple_build_vector_from_val (&stmts, UNKNOWN_LOCATION,
    5696              :                                                   target_type,
    5697              :                                                   nonconstant_for_splat);
    5698           91 :         }
    5699          432 :       else if (orig[1] == error_mark_node)
    5700              :         {
    5701              :           /* ???  See if we can convert the vector to the original type.  */
    5702          310 :           converted_orig1 = conv_code != ERROR_MARK;
    5703          310 :           unsigned n = converted_orig1 ? nelts : refnelts;
    5704          293 :           tree target_type = converted_orig1 ? type : perm_type;
    5705          310 :           tree_vector_builder vec (target_type, n, 1);
    5706         2508 :           for (unsigned i = 0; i < n; ++i)
    5707         3652 :             if (i < nelts && constants[i])
    5708              :               {
    5709         1056 :                 tree constant = constants[i];
    5710              :                 /* If there's a nop conversion, convert the constant.  */
    5711         1056 :                 if (!useless_type_conversion_p (TREE_TYPE (target_type),
    5712         1056 :                                                 TREE_TYPE (constant)))
    5713            2 :                   constant = fold_convert (TREE_TYPE (target_type), constant);
    5714         1056 :                 vec.quick_push (constant);
    5715              :               }
    5716              :             else
    5717              :               {
    5718              :                 /* ??? Push a don't-care value.  */
    5719          832 :                 tree constant = one_constant;
    5720          832 :                 if (!useless_type_conversion_p (TREE_TYPE (target_type),
    5721          832 :                                                 TREE_TYPE (constant)))
    5722            2 :                   constant = fold_convert (TREE_TYPE (target_type), constant);
    5723          832 :                 vec.quick_push (constant);
    5724              :               }
    5725          310 :           orig[1] = vec.build ();
    5726          310 :         }
    5727          575 :       tree blend_op2 = NULL_TREE;
    5728          575 :       if (converted_orig1)
    5729              :         {
    5730              :           /* Make sure we can do a blend in the target type.  */
    5731           34 :           vec_perm_builder sel (nelts, nelts, 1);
    5732          136 :           for (i = 0; i < elts.length (); ++i)
    5733          102 :             sel.quick_push (elts[i].first
    5734          102 :                             ? elts[i].second + nelts : i);
    5735           34 :           vec_perm_indices indices (sel, 2, nelts);
    5736           34 :           machine_mode vmode = TYPE_MODE (type);
    5737           34 :           if ((cfun->curr_properties & PROP_gimple_lvec)
    5738           34 :               && !can_vec_perm_const_p (vmode, vmode, indices))
    5739            0 :             return false;
    5740           34 :           mask_type = build_vector_type (ssizetype, nelts);
    5741           34 :           blend_op2 = vec_perm_indices_to_tree (mask_type, indices);
    5742           34 :         }
    5743              : 
    5744              :       /* For a real orig[1] (no splat, constant etc.) we might need to
    5745              :          nop-convert it.  Do so here.  */
    5746          697 :       if (orig[1] && orig[1] != error_mark_node
    5747          697 :           && !converted_orig1
    5748          663 :           && !useless_type_conversion_p (perm_type, TREE_TYPE (orig[1]))
    5749          697 :           && tree_nop_conversion_p (TREE_TYPE (perm_type),
    5750            0 :                                     TREE_TYPE (TREE_TYPE (orig[1]))))
    5751            0 :         orig[1] = gimple_build (&stmts, VIEW_CONVERT_EXPR, perm_type,
    5752              :                                 orig[1]);
    5753              : 
    5754          697 :       tree orig1_for_perm
    5755          697 :         = converted_orig1 ? build_zero_cst (perm_type) : orig[1];
    5756          697 :       tree res = gimple_build (&stmts, VEC_PERM_EXPR, perm_type,
    5757              :                                orig[0], orig1_for_perm, op2);
    5758              :       /* If we're building a smaller vector, extract the element
    5759              :          with the proper type.  */
    5760          697 :       if (nelts != refnelts)
    5761          246 :         res = gimple_build (&stmts, BIT_FIELD_REF,
    5762              :                             conv_code != ERROR_MARK ? conv_src_type : type,
    5763              :                             res,
    5764          123 :                             TYPE_SIZE (conv_code != ERROR_MARK ? conv_src_type
    5765              :                                                                : type),
    5766          123 :                             bitsize_zero_node);
    5767              :       /* Otherwise, we can still have an intermediate sign change.
    5768              :          ??? In that case we have two subsequent conversions.
    5769              :          We should be able to merge them.  */
    5770          574 :       else if (sign_change_p)
    5771           15 :         res = gimple_build (&stmts, VIEW_CONVERT_EXPR, conv_src_type, res);
    5772              :       /* Finally, apply the conversion.  */
    5773          697 :       if (conv_code != ERROR_MARK)
    5774           68 :         res = gimple_build (&stmts, conv_code, type, res);
    5775          629 :       else if (!useless_type_conversion_p (type, TREE_TYPE (res)))
    5776              :         {
    5777            3 :           gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
    5778              :                                 TYPE_VECTOR_SUBPARTS (perm_type))
    5779              :                       && tree_nop_conversion_p (TREE_TYPE (type),
    5780              :                                                 TREE_TYPE (perm_type)));
    5781            3 :           res = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, res);
    5782              :         }
    5783              :       /* Blend in the actual constant.  */
    5784          697 :       if (converted_orig1)
    5785           34 :         res = gimple_build (&stmts, VEC_PERM_EXPR, type,
    5786           34 :                             res, orig[1], blend_op2);
    5787          697 :       gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    5788          697 :       gimple_assign_set_rhs_with_ops (gsi, SSA_NAME, res);
    5789          754 :     }
    5790          790 :   update_stmt (gsi_stmt (*gsi));
    5791          790 :   return true;
    5792       168113 : }
    5793              : 
    5794              : /* Prepare a TARGET_MEM_REF ref so that it can be subsetted as
    5795              :    lvalue.  This splits out an address computation stmt before *GSI
    5796              :    and returns a MEM_REF wrapping the address.  */
    5797              : 
    5798              : static tree
    5799         1143 : prepare_target_mem_ref_lvalue (tree ref, gimple_stmt_iterator *gsi)
    5800              : {
    5801         1143 :   if (TREE_CODE (TREE_OPERAND (ref, 0)) == ADDR_EXPR)
    5802          189 :     mark_addressable (TREE_OPERAND (TREE_OPERAND (ref, 0), 0));
    5803         1143 :   tree ptrtype = build_pointer_type (TREE_TYPE (ref));
    5804         1143 :   tree tem = make_ssa_name (ptrtype);
    5805         1143 :   gimple *new_stmt
    5806         1143 :     = gimple_build_assign (tem, build1 (ADDR_EXPR, TREE_TYPE (tem),
    5807              :                                         unshare_expr (ref)));
    5808         1143 :   gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    5809         2286 :   ref = build2_loc (EXPR_LOCATION (ref),
    5810         1143 :                     MEM_REF, TREE_TYPE (ref), tem,
    5811         1143 :                     build_int_cst (TREE_TYPE (TREE_OPERAND (ref, 1)), 0));
    5812         1143 :   return ref;
    5813              : }
    5814              : 
    5815              : /* Rewrite the vector load at *GSI to component-wise loads if the load
    5816              :    is only used in BIT_FIELD_REF extractions with eventual intermediate
    5817              :    widening.  */
    5818              : 
    5819              : static void
    5820       282909 : optimize_vector_load (gimple_stmt_iterator *gsi)
    5821              : {
    5822       282909 :   gimple *stmt = gsi_stmt (*gsi);
    5823       282909 :   tree lhs = gimple_assign_lhs (stmt);
    5824       282909 :   tree rhs = gimple_assign_rhs1 (stmt);
    5825       282909 :   tree vuse = gimple_vuse (stmt);
    5826              : 
    5827              :   /* Gather BIT_FIELD_REFs to rewrite, looking through
    5828              :      VEC_UNPACK_{LO,HI}_EXPR.  */
    5829       282909 :   use_operand_p use_p;
    5830       282909 :   imm_use_iterator iter;
    5831       282909 :   bool rewrite = true;
    5832       282909 :   bool scalar_use = false;
    5833       282909 :   bool unpack_use = false;
    5834       282909 :   auto_vec<gimple *, 8> bf_stmts;
    5835       282909 :   auto_vec<tree, 8> worklist;
    5836       282909 :   worklist.quick_push (lhs);
    5837       284740 :   do
    5838              :     {
    5839       284740 :       tree def = worklist.pop ();
    5840       284740 :       unsigned HOST_WIDE_INT def_eltsize
    5841       284740 :         = TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
    5842       364165 :       FOR_EACH_IMM_USE_FAST (use_p, iter, def)
    5843              :         {
    5844       342941 :           gimple *use_stmt = USE_STMT (use_p);
    5845       342941 :           if (is_gimple_debug (use_stmt))
    5846        79425 :             continue;
    5847       341297 :           tree use_lhs;
    5848       341297 :           if (!is_gimple_assign (use_stmt)
    5849              :               /* For alias reasons we move the use to the place of the
    5850              :                  load.  Avoid this when abnormals are involved.  */
    5851       341297 :               || ((TREE_CODE ((use_lhs = gimple_assign_lhs (use_stmt)))
    5852              :                    == SSA_NAME)
    5853       240853 :                   && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)))
    5854              :             {
    5855              :               rewrite = false;
    5856       263516 :               break;
    5857              :             }
    5858       311188 :           enum tree_code use_code = gimple_assign_rhs_code (use_stmt);
    5859       311188 :           tree use_rhs = gimple_assign_rhs1 (use_stmt);
    5860       385324 :           if (use_code == BIT_FIELD_REF
    5861        74137 :               && TREE_OPERAND (use_rhs, 0) == def
    5862              :               /* If its on the VEC_UNPACK_{HI,LO}_EXPR
    5863              :                  def need to verify it is element aligned.  */
    5864       385325 :               && (def == lhs
    5865          153 :                   || (known_eq (bit_field_size (use_rhs), def_eltsize)
    5866        74289 :                       && constant_multiple_p (bit_field_offset (use_rhs),
    5867              :                                               def_eltsize)
    5868              :                       /* We can simulate the VEC_UNPACK_{HI,LO}_EXPR
    5869              :                          via a NOP_EXPR only for integral types.
    5870              :                          ???  Support VEC_UNPACK_FLOAT_{HI,LO}_EXPR.  */
    5871          153 :                       && INTEGRAL_TYPE_P (TREE_TYPE (use_rhs)))))
    5872              :             {
    5873        74136 :               if (!VECTOR_TYPE_P (TREE_TYPE (gimple_assign_lhs (use_stmt))))
    5874        71739 :                 scalar_use = true;
    5875        74136 :               bf_stmts.safe_push (use_stmt);
    5876        74136 :               continue;
    5877              :             }
    5878              :           /* Walk through one level of VEC_UNPACK_{LO,HI}_EXPR.  */
    5879       237052 :           if (def == lhs
    5880       235302 :               && (use_code == VEC_UNPACK_HI_EXPR
    5881       235302 :                   || use_code == VEC_UNPACK_LO_EXPR)
    5882         3645 :               && use_rhs == lhs)
    5883              :             {
    5884         3645 :               unpack_use = true;
    5885         3645 :               worklist.safe_push (gimple_assign_lhs (use_stmt));
    5886         3645 :               continue;
    5887              :             }
    5888              :           rewrite = false;
    5889              :           break;
    5890       284740 :         }
    5891       284740 :       if (!rewrite)
    5892              :         break;
    5893              :     }
    5894        42448 :   while (!worklist.is_empty ());
    5895              : 
    5896       282909 :   rewrite = rewrite && (scalar_use
    5897        19393 :                         || unpack_use
    5898          674 :                         || !can_implement_p (mov_optab,
    5899          674 :                                              TYPE_MODE (TREE_TYPE (lhs))));
    5900       263730 :   if (!rewrite)
    5901              :     {
    5902       263730 :       gsi_next (gsi);
    5903       263730 :       return;
    5904              :     }
    5905              :   /* We now have all ultimate uses of the load to rewrite in bf_stmts.  */
    5906              : 
    5907              :   /* Prepare the original ref to be wrapped in adjusted BIT_FIELD_REFs.
    5908              :      For TARGET_MEM_REFs we have to separate the LEA from the reference.  */
    5909        19179 :   tree load_rhs = rhs;
    5910        19179 :   if (TREE_CODE (load_rhs) == TARGET_MEM_REF)
    5911         1142 :     load_rhs = prepare_target_mem_ref_lvalue (load_rhs, gsi);
    5912              : 
    5913              :   /* Rewrite the BIT_FIELD_REFs to be actual loads, re-emitting them at
    5914              :      the place of the original load.  */
    5915       127409 :   for (gimple *use_stmt : bf_stmts)
    5916              :     {
    5917        69872 :       tree bfr = gimple_assign_rhs1 (use_stmt);
    5918        69872 :       tree new_rhs = unshare_expr (load_rhs);
    5919        69872 :       if (TREE_OPERAND (bfr, 0) != lhs)
    5920              :         {
    5921              :           /* When the BIT_FIELD_REF is on the promoted vector we have to
    5922              :              adjust it and emit a conversion afterwards.  */
    5923          152 :           gimple *def_stmt
    5924          152 :               = SSA_NAME_DEF_STMT (TREE_OPERAND (bfr, 0));
    5925          152 :           enum tree_code def_code
    5926          152 :               = gimple_assign_rhs_code (def_stmt);
    5927              : 
    5928              :           /* The adjusted BIT_FIELD_REF is of the promotion source
    5929              :              vector size and at half of the offset...  */
    5930          152 :           new_rhs = fold_build3 (BIT_FIELD_REF,
    5931              :                                  TREE_TYPE (TREE_TYPE (lhs)),
    5932              :                                  new_rhs,
    5933              :                                  TYPE_SIZE (TREE_TYPE (TREE_TYPE (lhs))),
    5934              :                                  size_binop (EXACT_DIV_EXPR,
    5935              :                                              TREE_OPERAND (bfr, 2),
    5936              :                                              bitsize_int (2)));
    5937              :           /* ... and offsetted by half of the vector if VEC_UNPACK_HI_EXPR.  */
    5938          152 :           if (def_code == (!BYTES_BIG_ENDIAN
    5939              :                            ? VEC_UNPACK_HI_EXPR : VEC_UNPACK_LO_EXPR))
    5940           76 :             TREE_OPERAND (new_rhs, 2)
    5941          152 :               = size_binop (PLUS_EXPR, TREE_OPERAND (new_rhs, 2),
    5942              :                             size_binop (EXACT_DIV_EXPR,
    5943              :                                         TYPE_SIZE (TREE_TYPE (lhs)),
    5944              :                                         bitsize_int (2)));
    5945          152 :           tree tem = make_ssa_name (TREE_TYPE (TREE_TYPE (lhs)));
    5946          152 :           gimple *new_stmt = gimple_build_assign (tem, new_rhs);
    5947          152 :           location_t loc = gimple_location (use_stmt);
    5948          152 :           gimple_set_location (new_stmt, loc);
    5949          152 :           gimple_set_vuse (new_stmt, vuse);
    5950          152 :           gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    5951              :           /* Perform scalar promotion.  */
    5952          152 :           new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
    5953              :                                           NOP_EXPR, tem);
    5954          152 :           gimple_set_location (new_stmt, loc);
    5955          152 :           gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    5956              :         }
    5957              :       else
    5958              :         {
    5959              :           /* When the BIT_FIELD_REF is on the original load result
    5960              :              we can just wrap that.  */
    5961        69720 :           tree new_rhs = fold_build3 (BIT_FIELD_REF, TREE_TYPE (bfr),
    5962              :                                       unshare_expr (load_rhs),
    5963              :                                       TREE_OPERAND (bfr, 1),
    5964              :                                       TREE_OPERAND (bfr, 2));
    5965        69720 :           gimple *new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
    5966              :                                                   new_rhs);
    5967        69720 :           location_t loc = gimple_location (use_stmt);
    5968        69720 :           gimple_set_location (new_stmt, loc);
    5969        69720 :           gimple_set_vuse (new_stmt, vuse);
    5970        69720 :           gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    5971              :         }
    5972        69872 :       gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    5973        69872 :       unlink_stmt_vdef (use_stmt);
    5974        69872 :       gsi_remove (&gsi2, true);
    5975              :     }
    5976              : 
    5977              :   /* Finally get rid of the intermediate stmts.  */
    5978        19179 :   gimple *use_stmt;
    5979        19915 :   FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    5980              :     {
    5981          736 :       if (is_gimple_debug (use_stmt))
    5982              :         {
    5983          674 :           if (gimple_debug_bind_p (use_stmt))
    5984              :             {
    5985          674 :               gimple_debug_bind_reset_value (use_stmt);
    5986          674 :               update_stmt (use_stmt);
    5987              :             }
    5988          674 :           continue;
    5989              :         }
    5990           62 :       gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    5991           62 :       unlink_stmt_vdef (use_stmt);
    5992           62 :       release_defs (use_stmt);
    5993           62 :       gsi_remove (&gsi2, true);
    5994        19179 :     }
    5995              :   /* And the original load.  */
    5996        19179 :   release_defs (stmt);
    5997        19179 :   gsi_remove (gsi, true);
    5998       282909 : }
    5999              : 
    6000              : 
    6001              : /* Primitive "lattice" function for gimple_simplify.  */
    6002              : 
    6003              : static tree
    6004   1785901107 : fwprop_ssa_val (tree name)
    6005              : {
    6006              :   /* First valueize NAME.  */
    6007   1785901107 :   if (TREE_CODE (name) == SSA_NAME
    6008   1785901107 :       && SSA_NAME_VERSION (name) < lattice.length ())
    6009              :     {
    6010   1784973014 :       tree val = lattice[SSA_NAME_VERSION (name)];
    6011   1784973014 :       if (val)
    6012   1785901107 :         name = val;
    6013              :     }
    6014              :   /* We continue matching along SSA use-def edges for SSA names
    6015              :      that are not single-use.  Currently there are no patterns
    6016              :      that would cause any issues with that.  */
    6017   1785901107 :   return name;
    6018              : }
    6019              : 
    6020              : /* Search for opportunities to free half of the lanes in the following pattern:
    6021              : 
    6022              :      v_in = {e0, e1, e2, e3}
    6023              :      v_1 = VEC_PERM <v_in, v_in, {0, 2, 0, 2}>
    6024              :      // v_1 = {e0, e2, e0, e2}
    6025              :      v_2 = VEC_PERM <v_in, v_in, {1, 3, 1, 3}>
    6026              :      // v_2 = {e1, e3, e1, e3}
    6027              : 
    6028              :      v_x = v_1 + v_2
    6029              :      // v_x = {e0+e1, e2+e3, e0+e1, e2+e3}
    6030              :      v_y = v_1 - v_2
    6031              :      // v_y = {e0-e1, e2-e3, e0-e1, e2-e3}
    6032              : 
    6033              :      v_out = VEC_PERM <v_x, v_y, {0, 1, 6, 7}>
    6034              :      // v_out = {e0+e1, e2+e3, e0-e1, e2-e3}
    6035              : 
    6036              :    The last statement could be simplified to:
    6037              :      v_out' = VEC_PERM <v_x, v_y, {0, 1, 4, 5}>
    6038              :      // v_out' = {e0+e1, e2+e3, e0-e1, e2-e3}
    6039              : 
    6040              :    Characteristic properties:
    6041              :    - v_1 and v_2 are created from the same input vector v_in and introduce the
    6042              :      lane duplication (in the selection operand) that we can eliminate.
    6043              :    - v_x and v_y are results from lane-preserving operations that use v_1 and
    6044              :      v_2 as inputs.
    6045              :    - v_out is created by selecting from duplicated lanes.  */
    6046              : 
    6047              : static bool
    6048       190450 : recognise_vec_perm_simplify_seq (gassign *stmt, vec_perm_simplify_seq *seq)
    6049              : {
    6050       190450 :   unsigned HOST_WIDE_INT nelts;
    6051              : 
    6052       190450 :   gcc_checking_assert (stmt);
    6053       190450 :   gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
    6054       190450 :   basic_block bb = gimple_bb (stmt);
    6055              : 
    6056              :   /* Decompose the final vec permute statement.  */
    6057       190450 :   tree v_x = gimple_assign_rhs1 (stmt);
    6058       190450 :   tree v_y = gimple_assign_rhs2 (stmt);
    6059       190450 :   tree sel = gimple_assign_rhs3 (stmt);
    6060              : 
    6061       190450 :   if (TREE_CODE (sel) != VECTOR_CST
    6062       265192 :       || !VECTOR_CST_NELTS (sel).is_constant (&nelts)
    6063       187658 :       || TREE_CODE (v_x) != SSA_NAME
    6064       185568 :       || TREE_CODE (v_y) != SSA_NAME
    6065       180091 :       || !has_single_use (v_x)
    6066       303133 :       || !has_single_use (v_y))
    6067              :     return false;
    6068              : 
    6069              :   /* Don't analyse sequences with many lanes.  */
    6070       110826 :   if (nelts > 4)
    6071              :     return false;
    6072              : 
    6073              :   /* Lookup the definition of v_x and v_y.  */
    6074       106597 :   gassign *v_x_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x));
    6075       106597 :   gassign *v_y_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_y));
    6076       106208 :   if (!v_x_stmt || gimple_bb (v_x_stmt) != bb
    6077       212805 :       || !v_y_stmt || gimple_bb (v_y_stmt) != bb)
    6078              :     return false;
    6079              : 
    6080              :   /* Check the operations that define v_x and v_y.  */
    6081       106201 :   if (TREE_CODE_CLASS (gimple_assign_rhs_code (v_x_stmt)) != tcc_binary
    6082       108306 :       || TREE_CODE_CLASS (gimple_assign_rhs_code (v_y_stmt)) != tcc_binary)
    6083              :     return false;
    6084              : 
    6085         2105 :   tree v_x_1 = gimple_assign_rhs1 (v_x_stmt);
    6086         2105 :   tree v_x_2 = gimple_assign_rhs2 (v_x_stmt);
    6087         2105 :   tree v_y_1 = gimple_assign_rhs1 (v_y_stmt);
    6088         2105 :   tree v_y_2 = gimple_assign_rhs2 (v_y_stmt);
    6089              : 
    6090         2105 :   if (v_x_stmt == v_y_stmt
    6091         2105 :       || TREE_CODE (v_x_1) != SSA_NAME
    6092         2102 :       || TREE_CODE (v_x_2) != SSA_NAME
    6093         2078 :       || num_imm_uses (v_x_1) != 2
    6094         4025 :       || num_imm_uses (v_x_2) != 2)
    6095              :     return false;
    6096              : 
    6097         1882 :   if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
    6098              :     {
    6099              :       /* Allow operands of commutative operators to swap.  */
    6100          656 :       if (commutative_tree_code (gimple_assign_rhs_code (v_x_stmt)))
    6101              :         {
    6102              :           /* Keep v_x_1 the first operand for non-commutative operators.  */
    6103          253 :           std::swap (v_x_1, v_x_2);
    6104          253 :           if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
    6105              :             return false;
    6106              :         }
    6107          403 :       else if (commutative_tree_code (gimple_assign_rhs_code (v_y_stmt)))
    6108              :         {
    6109          403 :           if (v_x_1 != v_y_2 || v_x_2 != v_y_1)
    6110              :             return false;
    6111              :         }
    6112              :       else
    6113              :         return false;
    6114              :     }
    6115         1882 :   gassign *v_1_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_1));
    6116         1882 :   gassign *v_2_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_2));
    6117         1818 :   if (!v_1_stmt || gimple_bb (v_1_stmt) != bb
    6118         3700 :       || !v_2_stmt || gimple_bb (v_2_stmt) != bb)
    6119              :     return false;
    6120              : 
    6121         1812 :   if (gimple_assign_rhs_code (v_1_stmt) != VEC_PERM_EXPR
    6122         1938 :       || gimple_assign_rhs_code (v_2_stmt) != VEC_PERM_EXPR)
    6123              :     return false;
    6124              : 
    6125              :   /* Decompose initial VEC_PERM_EXPRs.  */
    6126          112 :   tree v_in = gimple_assign_rhs1 (v_1_stmt);
    6127          112 :   tree v_1_sel = gimple_assign_rhs3 (v_1_stmt);
    6128          112 :   tree v_2_sel = gimple_assign_rhs3 (v_2_stmt);
    6129          112 :   if (v_in != gimple_assign_rhs2 (v_1_stmt)
    6130          107 :       || v_in != gimple_assign_rhs1 (v_2_stmt)
    6131          217 :       || v_in != gimple_assign_rhs2 (v_2_stmt))
    6132              :     return false;
    6133              : 
    6134          105 :   unsigned HOST_WIDE_INT v_1_nelts, v_2_nelts;
    6135          105 :   if (TREE_CODE (v_1_sel) != VECTOR_CST
    6136          105 :       || !VECTOR_CST_NELTS (v_1_sel).is_constant (&v_1_nelts)
    6137          105 :       || TREE_CODE (v_2_sel) != VECTOR_CST
    6138          210 :       || !VECTOR_CST_NELTS (v_2_sel).is_constant (&v_2_nelts))
    6139              :     return false;
    6140              : 
    6141          105 :   if (nelts != v_1_nelts || nelts != v_2_nelts)
    6142              :     return false;
    6143              : 
    6144              :   /* Create the new selector.  */
    6145          105 :   vec_perm_builder new_sel_perm (nelts, nelts, 1);
    6146          105 :   auto_vec<bool> lanes (nelts);
    6147          105 :   lanes.quick_grow_cleared (nelts);
    6148          630 :   for (unsigned int i = 0; i < nelts; i++)
    6149              :     {
    6150              :       /* Extract the i-th value from the selector.  */
    6151          420 :       unsigned int sel_cst = TREE_INT_CST_LOW (VECTOR_CST_ELT (sel, i));
    6152          420 :       unsigned int lane = sel_cst % nelts;
    6153          420 :       unsigned int offs = sel_cst / nelts;
    6154              : 
    6155              :       /* Check what's in the lane.  */
    6156          420 :       unsigned int e_1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, lane));
    6157          420 :       unsigned int e_2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, lane));
    6158              : 
    6159              :       /* Reuse previous lane (if any).  */
    6160          420 :       unsigned int l = 0;
    6161          711 :       for (; l < lane; l++)
    6162              :         {
    6163          489 :           if ((TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, l)) == e_1)
    6164          489 :               && (TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, l)) == e_2))
    6165              :             break;
    6166              :         }
    6167              : 
    6168              :       /* Add to narrowed selector.  */
    6169          420 :       new_sel_perm.quick_push (l + offs * nelts);
    6170              : 
    6171              :       /* Mark lane as used.  */
    6172          420 :       lanes[l] = true;
    6173              :     }
    6174              : 
    6175              :   /* Count how many lanes are need.  */
    6176              :   unsigned int cnt = 0;
    6177          525 :   for (unsigned int i = 0; i < nelts; i++)
    6178          420 :     cnt += lanes[i];
    6179              : 
    6180              :   /* If more than (nelts/2) lanes are needed, skip the sequence.  */
    6181          105 :   if (cnt > nelts / 2)
    6182              :     return false;
    6183              : 
    6184              :   /* Check if the resulting permutation is cheap.  */
    6185          105 :   vec_perm_indices new_indices (new_sel_perm, 2, nelts);
    6186          105 :   tree vectype = TREE_TYPE (gimple_assign_lhs (stmt));
    6187          105 :   machine_mode vmode = TYPE_MODE (vectype);
    6188          105 :   if (!can_vec_perm_const_p (vmode, vmode, new_indices, false))
    6189              :     return false;
    6190              : 
    6191          105 :   *seq = XNEW (struct _vec_perm_simplify_seq);
    6192          105 :   (*seq)->stmt = stmt;
    6193          105 :   (*seq)->v_1_stmt = v_1_stmt;
    6194          105 :   (*seq)->v_2_stmt = v_2_stmt;
    6195          105 :   (*seq)->v_x_stmt = v_x_stmt;
    6196          105 :   (*seq)->v_y_stmt = v_y_stmt;
    6197          105 :   (*seq)->nelts = nelts;
    6198          105 :   (*seq)->new_sel = vect_gen_perm_mask_checked (vectype, new_indices);
    6199              : 
    6200          105 :   if (dump_file)
    6201              :     {
    6202           32 :       fprintf (dump_file, "Found vec perm simplify sequence ending with:\n\t");
    6203           32 :       print_gimple_stmt (dump_file, stmt, 0);
    6204              : 
    6205           32 :       if (dump_flags & TDF_DETAILS)
    6206              :         {
    6207           32 :           fprintf (dump_file, "\tNarrowed vec_perm selector: ");
    6208           32 :           print_generic_expr (dump_file, (*seq)->new_sel);
    6209           32 :           fprintf (dump_file, "\n");
    6210              :         }
    6211              :     }
    6212              : 
    6213              :   return true;
    6214          210 : }
    6215              : 
    6216              : /* Reduce the lane consumption of a simplifiable vec perm sequence.  */
    6217              : 
    6218              : static void
    6219           78 : narrow_vec_perm_simplify_seq (const vec_perm_simplify_seq &seq)
    6220              : {
    6221           78 :   gassign *stmt = seq->stmt;
    6222           78 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6223              :     {
    6224           26 :       fprintf (dump_file, "Updating VEC_PERM statement:\n");
    6225           26 :       fprintf (dump_file, "Old stmt: ");
    6226           26 :       print_gimple_stmt (dump_file, stmt, 0);
    6227              :     }
    6228              : 
    6229              :   /* Update the last VEC_PERM statement.  */
    6230           78 :   gimple_assign_set_rhs3 (stmt, seq->new_sel);
    6231           78 :   update_stmt (stmt);
    6232              : 
    6233           78 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6234              :     {
    6235           26 :       fprintf (dump_file, "New stmt: ");
    6236           26 :       print_gimple_stmt (dump_file, stmt, 0);
    6237              :     }
    6238           78 : }
    6239              : 
    6240              : /* Test if we can blend two simplifiable vec permute sequences.
    6241              :    NEED_SWAP will be set, if sequences must be swapped for blending.  */
    6242              : 
    6243              : static bool
    6244           49 : can_blend_vec_perm_simplify_seqs_p (vec_perm_simplify_seq seq1,
    6245              :                                     vec_perm_simplify_seq seq2,
    6246              :                                     bool *need_swap)
    6247              : {
    6248           49 :   unsigned int nelts = seq1->nelts;
    6249           49 :   basic_block bb = gimple_bb (seq1->stmt);
    6250              : 
    6251           49 :   gcc_assert (gimple_bb (seq2->stmt) == bb);
    6252              : 
    6253              :   /* BBs and number of elements must be equal.  */
    6254           49 :   if (gimple_bb (seq2->stmt) != bb || seq2->nelts != nelts)
    6255              :     return false;
    6256              : 
    6257              :   /* We need vectors of the same type.  */
    6258           49 :   if (TREE_TYPE (gimple_assign_lhs (seq1->stmt))
    6259           49 :       != TREE_TYPE (gimple_assign_lhs (seq2->stmt)))
    6260              :     return false;
    6261              : 
    6262              :   /* We require isomorphic operators.  */
    6263           43 :   if (((gimple_assign_rhs_code (seq1->v_x_stmt)
    6264           43 :         != gimple_assign_rhs_code (seq2->v_x_stmt))
    6265           43 :        || (gimple_assign_rhs_code (seq1->v_y_stmt)
    6266           43 :            != gimple_assign_rhs_code (seq2->v_y_stmt))))
    6267              :     return false;
    6268              : 
    6269              :   /* We cannot have any dependencies between the sequences.
    6270              : 
    6271              :      For merging, we will reuse seq1->v_1_stmt and seq1->v_2_stmt.
    6272              :      seq1's v_in is defined before these statements, but we need
    6273              :      to check if seq2's v_in is defined before them as well.
    6274              : 
    6275              :      Further, we will reuse seq2->stmt.  We need to ensure that
    6276              :      seq1->v_x_stmt and seq1->v_y_stmt are before it.
    6277              : 
    6278              :      Note, that we don't need to check the BBs here, because all
    6279              :      statements of both sequences have to be in the same BB.  */
    6280              : 
    6281           43 :   tree seq2_v_in = gimple_assign_rhs1 (seq2->v_1_stmt);
    6282           43 :   if (TREE_CODE (seq2_v_in) != SSA_NAME)
    6283              :     return false;
    6284              : 
    6285           43 :   gassign *seq2_v_in_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq2_v_in));
    6286           43 :   if (!seq2_v_in_stmt || gimple_bb (seq2_v_in_stmt) != bb
    6287           43 :       || (gimple_uid (seq2_v_in_stmt) > gimple_uid (seq1->v_1_stmt))
    6288           39 :       || (gimple_uid (seq1->v_x_stmt) > gimple_uid (seq2->stmt))
    6289           39 :       || (gimple_uid (seq1->v_y_stmt) > gimple_uid (seq2->stmt)))
    6290              :     {
    6291            4 :       tree seq1_v_in = gimple_assign_rhs1 (seq1->v_1_stmt);
    6292            4 :       if (TREE_CODE (seq1_v_in) != SSA_NAME)
    6293              :         return false;
    6294              : 
    6295            4 :       gassign *seq1_v_in_stmt
    6296            4 :         = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq1_v_in));
    6297              :       /* Let's try to see if we succeed when swapping the sequences.  */
    6298            4 :       if (!seq1_v_in_stmt || gimple_bb (seq1_v_in_stmt)
    6299            0 :           || (gimple_uid (seq1_v_in_stmt) > gimple_uid (seq2->v_1_stmt))
    6300            0 :           || (gimple_uid (seq2->v_x_stmt) > gimple_uid (seq1->stmt))
    6301            0 :           || (gimple_uid (seq2->v_y_stmt) > gimple_uid (seq1->stmt)))
    6302              :         return false;
    6303            0 :       *need_swap = true;
    6304              :     }
    6305              :   else
    6306           39 :     *need_swap = false;
    6307              : 
    6308           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6309           13 :     fprintf (dump_file, "Found vec perm simplify sequence pair.\n");
    6310              : 
    6311              :   return true;
    6312              : }
    6313              : 
    6314              : /* Calculate the permutations for blending the two given vec permute
    6315              :    sequences.  This may fail if the resulting permutation is not
    6316              :    supported.  */
    6317              : 
    6318              : static bool
    6319           39 : calc_perm_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
    6320              :                                   vec_perm_simplify_seq seq2,
    6321              :                                   vec_perm_indices *seq2_stmt_indices,
    6322              :                                   vec_perm_indices *seq1_v_1_stmt_indices,
    6323              :                                   vec_perm_indices *seq1_v_2_stmt_indices)
    6324              : {
    6325           39 :   unsigned int i;
    6326           39 :   unsigned int nelts = seq1->nelts;
    6327           39 :   auto_vec<unsigned int> lane_assignment;
    6328           39 :   lane_assignment.create (nelts);
    6329              : 
    6330              :   /* Mark all lanes as free.  */
    6331           39 :   lane_assignment.quick_grow_cleared (nelts);
    6332              : 
    6333              :   /* Allocate lanes for seq1.  */
    6334          234 :   for (i = 0; i < nelts; i++)
    6335              :     {
    6336          156 :       unsigned int l = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq1->new_sel, i));
    6337          156 :       l %= nelts;
    6338          156 :       lane_assignment[l] = 1;
    6339              :     }
    6340              : 
    6341              :   /* Allocate lanes for seq2 and calculate selector for seq2->stmt.  */
    6342           39 :   vec_perm_builder seq2_stmt_sel_perm (nelts, nelts, 1);
    6343          195 :   for (i = 0; i < nelts; i++)
    6344              :     {
    6345          156 :       unsigned int sel = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, i));
    6346          156 :       unsigned int lane = sel % nelts;
    6347          156 :       unsigned int offs = sel / nelts;
    6348          156 :       unsigned int new_sel;
    6349              : 
    6350              :       /* Check if we already allocated the lane for seq2.  */
    6351          156 :       unsigned int j = 0;
    6352          279 :       for (; j < i; j++)
    6353              :         {
    6354          201 :           unsigned int sel_old;
    6355          201 :           sel_old = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, j));
    6356          201 :           unsigned int lane_old = sel_old % nelts;
    6357          201 :           if (lane == lane_old)
    6358              :             {
    6359           78 :               new_sel = seq2_stmt_sel_perm[j].to_constant ();
    6360           78 :               new_sel = (new_sel % nelts) + offs * nelts;
    6361           78 :               break;
    6362              :             }
    6363              :         }
    6364              : 
    6365              :       /* If the lane is not allocated, we need to do that now.  */
    6366          156 :       if (j == i)
    6367              :         {
    6368              :           unsigned int l_orig = lane;
    6369          194 :           while (lane_assignment[lane] != 0)
    6370              :             {
    6371          116 :               lane = (lane + 1) % nelts;
    6372              : 
    6373              :               /* This should not happen if both sequences utilize no more than
    6374              :                  half of the lanes.  Test anyway to guarantee termination.  */
    6375          116 :               if (lane == l_orig)
    6376           39 :                 return false;
    6377              :             }
    6378              : 
    6379              :           /* Allocate lane.  */
    6380           78 :           lane_assignment[lane] = 2 + l_orig;
    6381           78 :           new_sel = lane + offs * nelts;
    6382              :         }
    6383              : 
    6384          156 :       seq2_stmt_sel_perm.quick_push (new_sel);
    6385              :     }
    6386              : 
    6387              :   /* Check if the resulting permutation is cheap.  */
    6388           39 :   seq2_stmt_indices->new_vector (seq2_stmt_sel_perm, 2, nelts);
    6389           39 :   tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
    6390           39 :   machine_mode vmode = TYPE_MODE (vectype);
    6391           39 :   if (!can_vec_perm_const_p (vmode, vmode, *seq2_stmt_indices, false))
    6392              :     return false;
    6393              : 
    6394              :   /* Calculate selectors for seq1->v_1_stmt and seq1->v_2_stmt.  */
    6395           39 :   vec_perm_builder seq1_v_1_stmt_sel_perm (nelts, nelts, 1);
    6396           39 :   vec_perm_builder seq1_v_2_stmt_sel_perm (nelts, nelts, 1);
    6397          195 :   for (i = 0; i < nelts; i++)
    6398              :     {
    6399          156 :       bool use_seq1 = lane_assignment[i] < 2;
    6400          156 :       unsigned int l1, l2;
    6401              : 
    6402          156 :       if (use_seq1)
    6403              :         {
    6404              :           /* Just reuse the selector indices.  */
    6405           78 :           tree s1 = gimple_assign_rhs3 (seq1->v_1_stmt);
    6406           78 :           tree s2 = gimple_assign_rhs3 (seq1->v_2_stmt);
    6407           78 :           l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, i));
    6408           78 :           l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, i));
    6409              :         }
    6410              :       else
    6411              :         {
    6412              :           /* We moved the lanes for seq2, so we need to adjust for that.  */
    6413           78 :           tree s1 = gimple_assign_rhs3 (seq2->v_1_stmt);
    6414           78 :           tree s2 = gimple_assign_rhs3 (seq2->v_2_stmt);
    6415           78 :           l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, lane_assignment[i] - 2));
    6416           78 :           l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, lane_assignment[i] - 2));
    6417              :         }
    6418              : 
    6419          156 :       l1 %= nelts;
    6420          156 :       l2 %= nelts;
    6421          234 :       seq1_v_1_stmt_sel_perm.quick_push (l1 + (use_seq1 ? 0 : nelts));
    6422          156 :       seq1_v_2_stmt_sel_perm.quick_push (l2 + (use_seq1 ? 0 : nelts));
    6423              :     }
    6424              : 
    6425           39 :   seq1_v_1_stmt_indices->new_vector (seq1_v_1_stmt_sel_perm, 2, nelts);
    6426           39 :   vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
    6427           39 :   vmode = TYPE_MODE (vectype);
    6428           39 :   if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_1_stmt_indices, false))
    6429              :     return false;
    6430              : 
    6431           39 :   seq1_v_2_stmt_indices->new_vector (seq1_v_2_stmt_sel_perm, 2, nelts);
    6432           39 :   vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
    6433           39 :   vmode = TYPE_MODE (vectype);
    6434           39 :   if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_2_stmt_indices, false))
    6435              :     return false;
    6436              : 
    6437              :   return true;
    6438           78 : }
    6439              : 
    6440              : /* Blend the two given simplifiable vec permute sequences using the
    6441              :    given permutations.  */
    6442              : 
    6443              : static void
    6444           39 : blend_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
    6445              :                               vec_perm_simplify_seq seq2,
    6446              :                               const vec_perm_indices &seq2_stmt_indices,
    6447              :                               const vec_perm_indices &seq1_v_1_stmt_indices,
    6448              :                               const vec_perm_indices &seq1_v_2_stmt_indices)
    6449              : {
    6450              :   /* We don't need to adjust seq1->stmt because its lanes consumption
    6451              :      was already narrowed before entering this function.  */
    6452              : 
    6453              :   /* Adjust seq2->stmt: copy RHS1/RHS2 from seq1->stmt and set new sel.  */
    6454           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6455              :     {
    6456           13 :       fprintf (dump_file, "Updating VEC_PERM statement:\n");
    6457           13 :       fprintf (dump_file, "Old stmt: ");
    6458           13 :       print_gimple_stmt (dump_file, seq2->stmt, 0);
    6459              :     }
    6460              : 
    6461           39 :   gimple_assign_set_rhs1 (seq2->stmt, gimple_assign_rhs1 (seq1->stmt));
    6462           78 :   gimple_assign_set_rhs2 (seq2->stmt, gimple_assign_rhs2 (seq1->stmt));
    6463           39 :   tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
    6464           39 :   tree sel = vect_gen_perm_mask_checked (vectype, seq2_stmt_indices);
    6465           39 :   gimple_assign_set_rhs3 (seq2->stmt, sel);
    6466           39 :   update_stmt (seq2->stmt);
    6467              : 
    6468           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6469              :     {
    6470           13 :       fprintf (dump_file, "New stmt: ");
    6471           13 :       print_gimple_stmt (dump_file, seq2->stmt, 0);
    6472              :     }
    6473              : 
    6474              :   /* Adjust seq1->v_1_stmt: copy RHS2 from seq2->v_1_stmt and set new sel.  */
    6475           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6476              :     {
    6477           13 :       fprintf (dump_file, "Updating VEC_PERM statement:\n");
    6478           13 :       fprintf (dump_file, "Old stmt: ");
    6479           13 :       print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
    6480              :     }
    6481              : 
    6482           39 :   gimple_assign_set_rhs2 (seq1->v_1_stmt, gimple_assign_rhs1 (seq2->v_1_stmt));
    6483           39 :   vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
    6484           39 :   sel = vect_gen_perm_mask_checked (vectype, seq1_v_1_stmt_indices);
    6485           39 :   gimple_assign_set_rhs3 (seq1->v_1_stmt, sel);
    6486           39 :   update_stmt (seq1->v_1_stmt);
    6487              : 
    6488           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6489              :     {
    6490           13 :       fprintf (dump_file, "New stmt: ");
    6491           13 :       print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
    6492              :     }
    6493              : 
    6494              :   /* Adjust seq1->v_2_stmt: copy RHS2 from seq2->v_2_stmt and set new sel.  */
    6495           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6496              :     {
    6497           13 :       fprintf (dump_file, "Updating VEC_PERM statement:\n");
    6498           13 :       fprintf (dump_file, "Old stmt: ");
    6499           13 :       print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
    6500              :     }
    6501              : 
    6502           39 :   gimple_assign_set_rhs2 (seq1->v_2_stmt, gimple_assign_rhs1 (seq2->v_2_stmt));
    6503           39 :   vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
    6504           39 :   sel = vect_gen_perm_mask_checked (vectype, seq1_v_2_stmt_indices);
    6505           39 :   gimple_assign_set_rhs3 (seq1->v_2_stmt, sel);
    6506           39 :   update_stmt (seq1->v_2_stmt);
    6507              : 
    6508           39 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6509              :     {
    6510           13 :       fprintf (dump_file, "New stmt: ");
    6511           13 :       print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
    6512              :     }
    6513              : 
    6514              :   /* At this point, we have four unmodified seq2 stmts, which will be
    6515              :      eliminated by DCE.  */
    6516              : 
    6517           39 :   if (dump_file)
    6518           13 :     fprintf (dump_file, "Vec perm simplify sequences have been blended.\n\n");
    6519           39 : }
    6520              : 
    6521              : /* Try to blend narrowed vec_perm_simplify_seqs pairwise.
    6522              :    The provided list will be empty after this call.  */
    6523              : 
    6524              : static void
    6525    139509286 : process_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l)
    6526              : {
    6527    139509286 :   unsigned int i, j;
    6528    139509286 :   vec_perm_simplify_seq seq1, seq2;
    6529              : 
    6530    139509286 :   if (l->is_empty ())
    6531    139509239 :     return;
    6532              : 
    6533           47 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6534           15 :     fprintf (dump_file, "\nProcessing %u vec perm simplify sequences.\n",
    6535              :              l->length ());
    6536              : 
    6537          160 :   FOR_EACH_VEC_ELT (*l, i, seq1)
    6538              :     {
    6539           66 :       if (i + 1 < l->length ())
    6540              :         {
    6541          119 :           FOR_EACH_VEC_ELT_FROM (*l, j, seq2, i + 1)
    6542              :             {
    6543           49 :               bool swap = false;
    6544           49 :               if (can_blend_vec_perm_simplify_seqs_p (seq1, seq2, &swap))
    6545              :                 {
    6546           39 :                   vec_perm_indices seq2_stmt_indices;
    6547           39 :                   vec_perm_indices seq1_v_1_stmt_indices;
    6548           39 :                   vec_perm_indices seq1_v_2_stmt_indices;
    6549          117 :                   if (calc_perm_vec_perm_simplify_seqs (swap ? seq2 : seq1,
    6550              :                                                         swap ? seq1 : seq2,
    6551              :                                                         &seq2_stmt_indices,
    6552              :                                                         &seq1_v_1_stmt_indices,
    6553              :                                                         &seq1_v_2_stmt_indices))
    6554              :                     {
    6555              :                       /* Narrow lane usage.  */
    6556           39 :                       narrow_vec_perm_simplify_seq (seq1);
    6557           39 :                       narrow_vec_perm_simplify_seq (seq2);
    6558              : 
    6559              :                       /* Blend sequences.  */
    6560           39 :                       blend_vec_perm_simplify_seqs (swap ? seq2 : seq1,
    6561              :                                                     swap ? seq1 : seq2,
    6562              :                                                     seq2_stmt_indices,
    6563              :                                                     seq1_v_1_stmt_indices,
    6564              :                                                     seq1_v_2_stmt_indices);
    6565              : 
    6566              :                       /* We can use unordered_remove as we break the loop.  */
    6567           39 :                       l->unordered_remove (j);
    6568           39 :                       XDELETE (seq2);
    6569           39 :                       break;
    6570              :                     }
    6571           39 :                 }
    6572              :             }
    6573              :         }
    6574              : 
    6575              :       /* We don't need to call l->remove for seq1.  */
    6576           66 :       XDELETE (seq1);
    6577              :     }
    6578              : 
    6579           47 :   l->truncate (0);
    6580              : }
    6581              : 
    6582              : static void
    6583          105 : append_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l,
    6584              :                                    const vec_perm_simplify_seq &seq)
    6585              : {
    6586              :   /* If no space on list left, then process the list.  */
    6587          105 :   if (!l->space (1))
    6588            0 :       process_vec_perm_simplify_seq_list (l);
    6589              : 
    6590          105 :   l->quick_push (seq);
    6591          105 : }
    6592              : 
    6593              : /* Main entry point for the forward propagation and statement combine
    6594              :    optimizer.  */
    6595              : 
    6596              : namespace {
    6597              : 
    6598              : const pass_data pass_data_forwprop =
    6599              : {
    6600              :   GIMPLE_PASS, /* type */
    6601              :   "forwprop", /* name */
    6602              :   OPTGROUP_NONE, /* optinfo_flags */
    6603              :   TV_TREE_FORWPROP, /* tv_id */
    6604              :   ( PROP_cfg | PROP_ssa ), /* properties_required */
    6605              :   0, /* properties_provided */
    6606              :   0, /* properties_destroyed */
    6607              :   0, /* todo_flags_start */
    6608              :   0, /* todo_flags_finish */
    6609              : };
    6610              : 
    6611              : class pass_forwprop : public gimple_opt_pass
    6612              : {
    6613              : public:
    6614      1485900 :   pass_forwprop (gcc::context *ctxt)
    6615      2971800 :     : gimple_opt_pass (pass_data_forwprop, ctxt), last_p (false)
    6616              :   {}
    6617              : 
    6618              :   /* opt_pass methods: */
    6619      1188720 :   opt_pass * clone () final override { return new pass_forwprop (m_ctxt); }
    6620      1783080 :   void set_pass_param (unsigned int n, bool param) final override
    6621              :     {
    6622      1783080 :       switch (n)
    6623              :         {
    6624      1188720 :           case 0:
    6625      1188720 :             m_full_walk = param;
    6626      1188720 :             break;
    6627       594360 :           case 1:
    6628       594360 :             last_p = param;
    6629       594360 :             break;
    6630            0 :           default:
    6631            0 :           gcc_unreachable();
    6632              :         }
    6633      1783080 :     }
    6634      5716205 :   bool gate (function *) final override { return flag_tree_forwprop; }
    6635              :   unsigned int execute (function *) final override;
    6636              : 
    6637              :  private:
    6638              :   /* Determines whether the pass instance should set PROP_last_full_fold.  */
    6639              :   bool last_p;
    6640              : 
    6641              :   /* True if the aggregate props are doing a full walk or not.  */
    6642              :   bool m_full_walk = false;
    6643              : }; // class pass_forwprop
    6644              : 
    6645              : /* Attempt to make the BB block of __builtin_unreachable unreachable by changing
    6646              :    the incoming jumps.  Return true if at least one jump was changed.  */
    6647              : 
    6648              : static bool
    6649         1236 : optimize_unreachable (basic_block bb)
    6650              : {
    6651         1236 :   gimple_stmt_iterator gsi;
    6652         1236 :   gimple *stmt;
    6653         1236 :   edge_iterator ei;
    6654         1236 :   edge e;
    6655         1236 :   bool ret;
    6656              : 
    6657         1236 :   ret = false;
    6658         2554 :   FOR_EACH_EDGE (e, ei, bb->preds)
    6659              :     {
    6660         1318 :       gsi = gsi_last_bb (e->src);
    6661         1318 :       if (gsi_end_p (gsi))
    6662          327 :         continue;
    6663              : 
    6664          991 :       stmt = gsi_stmt (gsi);
    6665          991 :       if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
    6666              :         {
    6667              :           /* If the condition is already true/false
    6668              :              ignore it. This can happen during copy prop of forwprop. */
    6669          699 :           if (gimple_cond_true_p (cond_stmt)
    6670          691 :               || gimple_cond_false_p (cond_stmt))
    6671            8 :             continue;
    6672          683 :           else if (e->flags & EDGE_TRUE_VALUE)
    6673          599 :             gimple_cond_make_false (cond_stmt);
    6674           84 :           else if (e->flags & EDGE_FALSE_VALUE)
    6675           84 :             gimple_cond_make_true (cond_stmt);
    6676              :           else
    6677            0 :             gcc_unreachable ();
    6678          683 :           update_stmt (cond_stmt);
    6679              :         }
    6680              :       else
    6681              :         {
    6682              :           /* Todo: handle other cases.  e.g. switch.  */
    6683          300 :           continue;
    6684              :         }
    6685              : 
    6686          683 :       ret = true;
    6687              :     }
    6688              : 
    6689         1236 :   return ret;
    6690              : }
    6691              : 
    6692              : unsigned int
    6693      5713356 : pass_forwprop::execute (function *fun)
    6694              : {
    6695      5713356 :   unsigned int todoflags = 0;
    6696              :   /* Handle a full walk only when expensive optimizations are on.  */
    6697      5713356 :   bool full_walk = m_full_walk && flag_expensive_optimizations;
    6698              : 
    6699      5713356 :   cfg_changed = false;
    6700      5713356 :   if (last_p)
    6701      1064909 :     fun->curr_properties |= PROP_last_full_fold;
    6702              : 
    6703      5713356 :   calculate_dominance_info (CDI_DOMINATORS);
    6704              : 
    6705              :   /* Combine stmts with the stmts defining their operands.  Do that
    6706              :      in an order that guarantees visiting SSA defs before SSA uses.  */
    6707     11426712 :   lattice.create (num_ssa_names);
    6708     11426712 :   lattice.quick_grow_cleared (num_ssa_names);
    6709      5713356 :   int *postorder = XNEWVEC (int, n_basic_blocks_for_fn (fun));
    6710      5713356 :   int postorder_num = pre_and_rev_post_order_compute_fn (fun, NULL,
    6711              :                                                          postorder, false);
    6712      5713356 :   int *bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fun));
    6713     57482102 :   for (int i = 0; i < postorder_num; ++i)
    6714              :     {
    6715     46055390 :       bb_to_rpo[postorder[i]] = i;
    6716     46055390 :       edge_iterator ei;
    6717     46055390 :       edge e;
    6718    110858816 :       FOR_EACH_EDGE (e, ei, BASIC_BLOCK_FOR_FN (fun, postorder[i])->succs)
    6719     64803426 :         e->flags &= ~EDGE_EXECUTABLE;
    6720              :     }
    6721      5713356 :   single_succ_edge (BASIC_BLOCK_FOR_FN (fun, ENTRY_BLOCK))->flags
    6722      5713356 :     |= EDGE_EXECUTABLE;
    6723      5713356 :   auto_vec<gimple *, 4> to_fixup;
    6724      5713356 :   auto_vec<gimple *, 32> to_remove;
    6725      5713356 :   auto_vec<unsigned, 32> to_remove_defs;
    6726      5713356 :   auto_vec<std::pair<int, int>, 10> edges_to_remove;
    6727      5713356 :   auto_bitmap simple_dce_worklist;
    6728      5713356 :   auto_bitmap need_ab_cleanup;
    6729      5713356 :   to_purge = BITMAP_ALLOC (NULL);
    6730      5713356 :   auto_vec<vec_perm_simplify_seq, 8> vec_perm_simplify_seq_list;
    6731     51768746 :   for (int i = 0; i < postorder_num; ++i)
    6732              :     {
    6733     46055390 :       gimple_stmt_iterator gsi;
    6734     46055390 :       basic_block bb = BASIC_BLOCK_FOR_FN (fun, postorder[i]);
    6735     46055390 :       edge_iterator ei;
    6736     46055390 :       edge e;
    6737              : 
    6738              :       /* Skip processing not executable blocks.  We could improve
    6739              :          single_use tracking by at least unlinking uses from unreachable
    6740              :          blocks but since blocks with uses are not processed in a
    6741              :          meaningful order this is probably not worth it.  */
    6742     46055390 :       bool any = false;
    6743     47267718 :       FOR_EACH_EDGE (e, ei, bb->preds)
    6744              :         {
    6745     47240888 :           if ((e->flags & EDGE_EXECUTABLE)
    6746              :               /* We can handle backedges in natural loops correctly but
    6747              :                  for irreducible regions we have to take all backedges
    6748              :                  conservatively when we did not visit the source yet.  */
    6749     47240888 :               || (bb_to_rpo[e->src->index] > i
    6750       727420 :                   && !dominated_by_p (CDI_DOMINATORS, e->src, e->dest)))
    6751              :             {
    6752              :               any = true;
    6753              :               break;
    6754              :             }
    6755              :         }
    6756     46055390 :       if (!any)
    6757        27475 :         continue;
    6758              : 
    6759              :       /* Remove conditions that go directly to unreachable when this is the last forwprop.  */
    6760     46028560 :       if (last_p
    6761     10138628 :           && !(flag_sanitize & SANITIZE_UNREACHABLE))
    6762              :         {
    6763     10133656 :           gimple_stmt_iterator gsi;
    6764     10133656 :           gsi = gsi_start_nondebug_after_labels_bb (bb);
    6765     10134301 :           if (!gsi_end_p (gsi)
    6766      9165272 :               && gimple_call_builtin_p (*gsi, BUILT_IN_UNREACHABLE)
    6767     10134892 :               && optimize_unreachable (bb))
    6768              :             {
    6769          645 :               cfg_changed = true;
    6770          645 :               continue;
    6771              :             }
    6772              :         }
    6773              : 
    6774              :       /* Fold PHI-form long-multiply carries and record degenerate
    6775              :          PHIs in the lattice.  Iterator advanced up front so a folded
    6776              :          PHI can be removed in-flight; a long-mul carry PHI is never
    6777              :          degenerate, so the two cases are disjoint.  */
    6778     62324210 :       for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);)
    6779              :         {
    6780     16296295 :           gphi *phi = si.phi ();
    6781     16296295 :           gsi_next (&si);
    6782     16296295 :           tree res = gimple_phi_result (phi);
    6783     32592590 :           if (virtual_operand_p (res))
    6784      7440055 :             continue;
    6785      8856254 :           if (match_long_mul_phi (phi))
    6786           14 :             continue;
    6787              : 
    6788      8856240 :           tree first = NULL_TREE;
    6789      8856240 :           bool all_same = true;
    6790      8856240 :           edge_iterator ei;
    6791      8856240 :           edge e;
    6792     18228692 :           FOR_EACH_EDGE (e, ei, bb->preds)
    6793              :             {
    6794              :               /* Ignore not executable forward edges.  */
    6795     17974861 :               if (!(e->flags & EDGE_EXECUTABLE))
    6796              :                 {
    6797      4182173 :                   if (bb_to_rpo[e->src->index] < i)
    6798        17268 :                     continue;
    6799              :                   /* Avoid equivalences from backedges - while we might
    6800              :                      be able to make irreducible regions reducible and
    6801              :                      thus turning a back into a forward edge we do not
    6802              :                      want to deal with the intermediate SSA issues that
    6803              :                      exposes.  */
    6804              :                   all_same = false;
    6805              :                 }
    6806     17957593 :               tree use = PHI_ARG_DEF_FROM_EDGE (phi, e);
    6807     17957593 :               if (use == res)
    6808              :                 /* The PHI result can also appear on a backedge, if so
    6809              :                    we can ignore this case for the purpose of determining
    6810              :                    the singular value.  */
    6811              :                 ;
    6812     17951683 :               else if (! first)
    6813              :                 first = use;
    6814      9095443 :               else if (! operand_equal_p (first, use, 0))
    6815              :                 {
    6816              :                   all_same = false;
    6817              :                   break;
    6818              :                 }
    6819              :             }
    6820      8856240 :           if (all_same)
    6821              :             {
    6822       247961 :               if (may_propagate_copy (res, first))
    6823       247290 :                 to_remove_defs.safe_push (SSA_NAME_VERSION (res));
    6824       247961 :               fwprop_set_lattice_val (res, first);
    6825              :             }
    6826              :         }
    6827              : 
    6828              :       /* Apply forward propagation to all stmts in the basic-block.
    6829              :          Note we update GSI within the loop as necessary.  */
    6830     46027915 :       unsigned int uid = 1;
    6831    450069200 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); )
    6832              :         {
    6833    358013370 :           gimple *stmt = gsi_stmt (gsi);
    6834    358013370 :           tree lhs, rhs;
    6835    358013370 :           enum tree_code code;
    6836              : 
    6837    358013370 :           if (is_gimple_debug (stmt))
    6838              :             {
    6839    195977768 :               gsi_next (&gsi);
    6840    195977768 :               continue;
    6841              :             }
    6842              : 
    6843    162035602 :           gimple_set_uid (stmt, uid++);
    6844              : 
    6845    162035602 :           if (!is_gimple_assign (stmt))
    6846              :             {
    6847     53476210 :               process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
    6848     53476210 :               gsi_next (&gsi);
    6849     53476210 :               continue;
    6850              :             }
    6851              : 
    6852    108559392 :           lhs = gimple_assign_lhs (stmt);
    6853    108559392 :           rhs = gimple_assign_rhs1 (stmt);
    6854    108559392 :           code = gimple_assign_rhs_code (stmt);
    6855              : 
    6856    148564553 :           if (TREE_CODE (lhs) != SSA_NAME
    6857    108559392 :               || has_zero_uses (lhs))
    6858              :             {
    6859     40005161 :               process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
    6860     40005161 :               gsi_next (&gsi);
    6861     40005161 :               continue;
    6862              :             }
    6863              : 
    6864              :           /* If this statement sets an SSA_NAME to an address,
    6865              :              try to propagate the address into the uses of the SSA_NAME.  */
    6866     68554231 :           if ((code == ADDR_EXPR
    6867              :                /* Handle pointer conversions on invariant addresses
    6868              :                   as well, as this is valid gimple.  */
    6869     66222683 :                || (CONVERT_EXPR_CODE_P (code)
    6870      9165536 :                    && TREE_CODE (rhs) == ADDR_EXPR
    6871       363787 :                    && POINTER_TYPE_P (TREE_TYPE (lhs))))
    6872     68554455 :               && TREE_CODE (TREE_OPERAND (rhs, 0)) != TARGET_MEM_REF)
    6873              :             {
    6874      2331153 :               tree base = get_base_address (TREE_OPERAND (rhs, 0));
    6875      2331153 :               if ((!base
    6876      2331153 :                    || !DECL_P (base)
    6877       133137 :                    || decl_address_invariant_p (base))
    6878      2331153 :                   && !stmt_references_abnormal_ssa_name (stmt)
    6879      4662290 :                   && forward_propagate_addr_expr (lhs, rhs, true))
    6880              :                 {
    6881       477056 :                   fwprop_invalidate_lattice (gimple_get_lhs (stmt));
    6882       477056 :                   release_defs (stmt);
    6883       477056 :                   gsi_remove (&gsi, true);
    6884              :                 }
    6885              :               else
    6886      1854097 :                 gsi_next (&gsi);
    6887              :             }
    6888     66223078 :           else if (code == POINTER_PLUS_EXPR)
    6889              :             {
    6890      3697060 :               tree off = gimple_assign_rhs2 (stmt);
    6891      3697060 :               if (TREE_CODE (off) == INTEGER_CST
    6892      1136858 :                   && can_propagate_from (stmt)
    6893      1136507 :                   && !simple_iv_increment_p (stmt)
    6894              :                   /* ???  Better adjust the interface to that function
    6895              :                      instead of building new trees here.  */
    6896      4534863 :                   && forward_propagate_addr_expr
    6897      2513409 :                        (lhs,
    6898              :                         build1_loc (gimple_location (stmt),
    6899       837803 :                                     ADDR_EXPR, TREE_TYPE (rhs),
    6900       837803 :                                     fold_build2 (MEM_REF,
    6901              :                                                  TREE_TYPE (TREE_TYPE (rhs)),
    6902              :                                                  rhs,
    6903              :                                                  fold_convert (ptr_type_node,
    6904              :                                                                off))), true))
    6905              :                 {
    6906       317822 :                   fwprop_invalidate_lattice (gimple_get_lhs (stmt));
    6907       317822 :                   release_defs (stmt);
    6908       317822 :                   gsi_remove (&gsi, true);
    6909              :                 }
    6910      3379238 :               else if (is_gimple_min_invariant (rhs))
    6911              :                 {
    6912              :                   /* Make sure to fold &a[0] + off_1 here.  */
    6913       416265 :                   fold_stmt_inplace (&gsi);
    6914       416265 :                   update_stmt (stmt);
    6915       416265 :                   if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
    6916       416241 :                     gsi_next (&gsi);
    6917              :                 }
    6918              :               else
    6919      2962973 :                 gsi_next (&gsi);
    6920              :             }
    6921     62526018 :           else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE
    6922       214531 :                    && gimple_assign_load_p (stmt)
    6923       135192 :                    && !gimple_has_volatile_ops (stmt)
    6924        41117 :                    && TREE_CODE (rhs) != TARGET_MEM_REF
    6925        41087 :                    && TREE_CODE (rhs) != BIT_FIELD_REF
    6926     62567101 :                    && !stmt_can_throw_internal (fun, stmt))
    6927              :             {
    6928              :               /* Rewrite loads used only in real/imagpart extractions to
    6929              :                  component-wise loads.  */
    6930        40965 :               use_operand_p use_p;
    6931        40965 :               imm_use_iterator iter;
    6932        40965 :               tree vuse = gimple_vuse (stmt);
    6933        40965 :               bool rewrite = true;
    6934        46151 :               FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
    6935              :                 {
    6936        44022 :                   gimple *use_stmt = USE_STMT (use_p);
    6937        44022 :                   if (is_gimple_debug (use_stmt))
    6938         1011 :                     continue;
    6939        43011 :                   if (!is_gimple_assign (use_stmt)
    6940        28301 :                       || (gimple_assign_rhs_code (use_stmt) != REALPART_EXPR
    6941        26186 :                           && gimple_assign_rhs_code (use_stmt) != IMAGPART_EXPR)
    6942         4186 :                       || TREE_OPERAND (gimple_assign_rhs1 (use_stmt), 0) != lhs
    6943              :                       /* We are moving the def, so avoid abnormals.  */
    6944        47197 :                       || SSA_NAME_OCCURS_IN_ABNORMAL_PHI
    6945              :                            (gimple_assign_lhs (use_stmt)))
    6946              :                     {
    6947              :                       rewrite = false;
    6948              :                       break;
    6949              :                     }
    6950        40965 :                 }
    6951        40965 :               if (rewrite)
    6952              :                 {
    6953         2129 :                   gimple *use_stmt;
    6954         6769 :                   FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    6955              :                     {
    6956         4640 :                       if (is_gimple_debug (use_stmt))
    6957              :                         {
    6958          498 :                           if (gimple_debug_bind_p (use_stmt))
    6959              :                             {
    6960          498 :                               gimple_debug_bind_reset_value (use_stmt);
    6961          498 :                               update_stmt (use_stmt);
    6962              :                             }
    6963          498 :                           continue;
    6964              :                         }
    6965              : 
    6966         8284 :                       tree new_rhs = build1 (gimple_assign_rhs_code (use_stmt),
    6967         4142 :                                              TREE_TYPE (TREE_TYPE (rhs)),
    6968              :                                              unshare_expr (rhs));
    6969         4142 :                       gimple *new_stmt
    6970         4142 :                         = gimple_build_assign (gimple_assign_lhs (use_stmt),
    6971              :                                                new_rhs);
    6972              : 
    6973         4142 :                       location_t loc = gimple_location (use_stmt);
    6974         4142 :                       gimple_set_location (new_stmt, loc);
    6975         4142 :                       gimple_set_vuse (new_stmt, vuse);
    6976         4142 :                       gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    6977         4142 :                       unlink_stmt_vdef (use_stmt);
    6978         4142 :                       gsi_remove (&gsi2, true);
    6979              : 
    6980         4142 :                       gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
    6981         2129 :                     }
    6982              : 
    6983         2129 :                   release_defs (stmt);
    6984         2129 :                   gsi_remove (&gsi, true);
    6985              :                 }
    6986              :               else
    6987        38836 :                 gsi_next (&gsi);
    6988              :             }
    6989     62485053 :           else if (TREE_CODE (TREE_TYPE (lhs)) == VECTOR_TYPE
    6990      1816617 :                    && (TYPE_MODE (TREE_TYPE (lhs)) == BLKmode
    6991              :                        /* After vector lowering rewrite all loads, but
    6992              :                           initially do not since this conflicts with
    6993              :                           vector CONSTRUCTOR to shuffle optimization.  */
    6994      1788774 :                        || (fun->curr_properties & PROP_gimple_lvec))
    6995       922747 :                    && gimple_assign_load_p (stmt)
    6996       298124 :                    && !gimple_has_volatile_ops (stmt)
    6997       283412 :                    && !stmt_can_throw_internal (fun, stmt)
    6998     62768465 :                    && (!VAR_P (rhs) || !DECL_HARD_REGISTER (rhs)))
    6999       282909 :             optimize_vector_load (&gsi);
    7000              : 
    7001     62202144 :           else if (code == COMPLEX_EXPR)
    7002              :             {
    7003              :               /* Rewrite stores of a single-use complex build expression
    7004              :                  to component-wise stores.  */
    7005        37954 :               use_operand_p use_p;
    7006        37954 :               gimple *use_stmt, *def1, *def2;
    7007        37954 :               tree rhs2;
    7008        37954 :               if (single_imm_use (lhs, &use_p, &use_stmt)
    7009        35771 :                   && gimple_store_p (use_stmt)
    7010        42098 :                   && !gimple_has_volatile_ops (use_stmt)
    7011         3133 :                   && is_gimple_assign (use_stmt)
    7012         3129 :                   && (TREE_CODE (TREE_TYPE (gimple_assign_lhs (use_stmt)))
    7013              :                       == COMPLEX_TYPE)
    7014        41078 :                   && (TREE_CODE (gimple_assign_lhs (use_stmt))
    7015              :                       != TARGET_MEM_REF))
    7016              :                 {
    7017         3120 :                   tree use_lhs = gimple_assign_lhs (use_stmt);
    7018         3120 :                   if (auto_var_p (use_lhs))
    7019          601 :                     DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
    7020         6240 :                   tree new_lhs = build1 (REALPART_EXPR,
    7021         3120 :                                          TREE_TYPE (TREE_TYPE (use_lhs)),
    7022              :                                          unshare_expr (use_lhs));
    7023         3120 :                   gimple *new_stmt = gimple_build_assign (new_lhs, rhs);
    7024         3120 :                   location_t loc = gimple_location (use_stmt);
    7025         3120 :                   gimple_set_location (new_stmt, loc);
    7026         6240 :                   gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
    7027         3120 :                   gimple_set_vdef (new_stmt, make_ssa_name (gimple_vop (fun)));
    7028         6240 :                   SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
    7029         6240 :                   gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
    7030         3120 :                   gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    7031         3120 :                   gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
    7032              : 
    7033         6240 :                   new_lhs = build1 (IMAGPART_EXPR,
    7034         3120 :                                     TREE_TYPE (TREE_TYPE (use_lhs)),
    7035              :                                     unshare_expr (use_lhs));
    7036         3120 :                   gimple_assign_set_lhs (use_stmt, new_lhs);
    7037         3120 :                   gimple_assign_set_rhs1 (use_stmt, gimple_assign_rhs2 (stmt));
    7038         3120 :                   update_stmt (use_stmt);
    7039              : 
    7040         3120 :                   release_defs (stmt);
    7041         3120 :                   gsi_remove (&gsi, true);
    7042              :                 }
    7043              :               /* Rewrite a component-wise load of a complex to a complex
    7044              :                  load if the components are not used separately.  */
    7045        34834 :               else if (TREE_CODE (rhs) == SSA_NAME
    7046        34368 :                        && has_single_use (rhs)
    7047        30818 :                        && ((rhs2 = gimple_assign_rhs2 (stmt)), true)
    7048        30818 :                        && TREE_CODE (rhs2) == SSA_NAME
    7049        28997 :                        && has_single_use (rhs2)
    7050        28577 :                        && (def1 = SSA_NAME_DEF_STMT (rhs),
    7051        28577 :                            gimple_assign_load_p (def1))
    7052         1093 :                        && (def2 = SSA_NAME_DEF_STMT (rhs2),
    7053         1093 :                            gimple_assign_load_p (def2))
    7054         1598 :                        && (gimple_vuse (def1) == gimple_vuse (def2))
    7055          796 :                        && !gimple_has_volatile_ops (def1)
    7056          796 :                        && !gimple_has_volatile_ops (def2)
    7057          796 :                        && !stmt_can_throw_internal (fun, def1)
    7058          796 :                        && !stmt_can_throw_internal (fun, def2)
    7059          796 :                        && gimple_assign_rhs_code (def1) == REALPART_EXPR
    7060          550 :                        && gimple_assign_rhs_code (def2) == IMAGPART_EXPR
    7061        35384 :                        && operand_equal_p (TREE_OPERAND (gimple_assign_rhs1
    7062              :                                                                  (def1), 0),
    7063          550 :                                            TREE_OPERAND (gimple_assign_rhs1
    7064              :                                                                  (def2), 0)))
    7065              :                 {
    7066          550 :                   tree cl = TREE_OPERAND (gimple_assign_rhs1 (def1), 0);
    7067          550 :                   gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (cl));
    7068          550 :                   gcc_assert (gsi_stmt (gsi) == stmt);
    7069         1100 :                   gimple_set_vuse (stmt, gimple_vuse (def1));
    7070          550 :                   gimple_set_modified (stmt, true);
    7071          550 :                   gimple_stmt_iterator gsi2 = gsi_for_stmt (def1);
    7072          550 :                   gsi_remove (&gsi, false);
    7073          550 :                   gsi_insert_after (&gsi2, stmt, GSI_SAME_STMT);
    7074              :                 }
    7075              :               else
    7076        34284 :                 gsi_next (&gsi);
    7077              :             }
    7078     62164190 :           else if (code == CONSTRUCTOR
    7079       170514 :                    && VECTOR_TYPE_P (TREE_TYPE (rhs))
    7080       170514 :                    && TYPE_MODE (TREE_TYPE (rhs)) == BLKmode
    7081         4749 :                    && CONSTRUCTOR_NELTS (rhs) > 0
    7082     62168939 :                    && (!VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
    7083         2104 :                        || (TYPE_MODE (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
    7084              :                            != BLKmode)))
    7085              :             {
    7086              :               /* Rewrite stores of a single-use vector constructors
    7087              :                  to component-wise stores if the mode isn't supported.  */
    7088         4392 :               use_operand_p use_p;
    7089         4392 :               gimple *use_stmt;
    7090         4392 :               if (single_imm_use (lhs, &use_p, &use_stmt)
    7091         3928 :                   && gimple_store_p (use_stmt)
    7092         3428 :                   && !gimple_has_volatile_ops (use_stmt)
    7093         1708 :                   && !stmt_can_throw_internal (fun, use_stmt)
    7094         6093 :                   && is_gimple_assign (use_stmt))
    7095              :                 {
    7096         1701 :                   tree elt_t = TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value);
    7097         1701 :                   unsigned HOST_WIDE_INT elt_w
    7098         1701 :                     = tree_to_uhwi (TYPE_SIZE (elt_t));
    7099         1701 :                   unsigned HOST_WIDE_INT n
    7100         1701 :                     = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
    7101         1701 :                   tree use_lhs = gimple_assign_lhs (use_stmt);
    7102         1701 :                   if (auto_var_p (use_lhs))
    7103          575 :                     DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
    7104         1126 :                   else if (TREE_CODE (use_lhs) == TARGET_MEM_REF)
    7105              :                     {
    7106            1 :                       gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    7107            1 :                       use_lhs = prepare_target_mem_ref_lvalue (use_lhs, &gsi2);
    7108              :                     }
    7109        43684 :                   for (unsigned HOST_WIDE_INT bi = 0; bi < n; bi += elt_w)
    7110              :                     {
    7111        41983 :                       unsigned HOST_WIDE_INT ci = bi / elt_w;
    7112        41983 :                       tree new_rhs;
    7113        41983 :                       if (ci < CONSTRUCTOR_NELTS (rhs))
    7114        41365 :                         new_rhs = CONSTRUCTOR_ELT (rhs, ci)->value;
    7115              :                       else
    7116          618 :                         new_rhs = build_zero_cst (elt_t);
    7117        41983 :                       tree new_lhs = build3 (BIT_FIELD_REF,
    7118              :                                              elt_t,
    7119              :                                              unshare_expr (use_lhs),
    7120        41983 :                                              bitsize_int (elt_w),
    7121        41983 :                                              bitsize_int (bi));
    7122        41983 :                       gimple *new_stmt = gimple_build_assign (new_lhs, new_rhs);
    7123        41983 :                       location_t loc = gimple_location (use_stmt);
    7124        41983 :                       gimple_set_location (new_stmt, loc);
    7125        83966 :                       gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
    7126        41983 :                       gimple_set_vdef (new_stmt,
    7127              :                                        make_ssa_name (gimple_vop (fun)));
    7128        83966 :                       SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
    7129        83966 :                       gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
    7130        41983 :                       gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    7131        41983 :                       gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
    7132              :                     }
    7133         1701 :                   gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
    7134         1701 :                   unlink_stmt_vdef (use_stmt);
    7135         1701 :                   release_defs (use_stmt);
    7136         1701 :                   gsi_remove (&gsi2, true);
    7137         1701 :                   release_defs (stmt);
    7138         1701 :                   gsi_remove (&gsi, true);
    7139              :                 }
    7140              :               else
    7141         2691 :                 gsi_next (&gsi);
    7142              :             }
    7143     62159798 :           else if (code == VEC_PERM_EXPR)
    7144              :             {
    7145              :               /* Find vectorized sequences, where we can reduce the lane
    7146              :                  utilization.  The narrowing will be donw later and only
    7147              :                  if we find a pair of sequences that can be blended.  */
    7148       190450 :               gassign *assign = dyn_cast <gassign *> (stmt);
    7149       190450 :               vec_perm_simplify_seq seq;
    7150       190450 :               if (recognise_vec_perm_simplify_seq (assign, &seq))
    7151          105 :                 append_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list,
    7152              :                                                    seq);
    7153              : 
    7154       190450 :               gsi_next (&gsi);
    7155              :           }
    7156              :           else
    7157     61969348 :             gsi_next (&gsi);
    7158              :         }
    7159              : 
    7160     46027915 :       process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
    7161              : 
    7162              :       /* Combine stmts with the stmts defining their operands.
    7163              :          Note we update GSI within the loop as necessary.  */
    7164    449740972 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    7165              :         {
    7166    357685142 :           gimple *stmt = gsi_stmt (gsi);
    7167              : 
    7168              :           /* Mark stmt as potentially needing revisiting.  */
    7169    357685142 :           gimple_set_plf (stmt, GF_PLF_1, false);
    7170              : 
    7171    357685142 :           bool can_make_abnormal_goto = (is_gimple_call (stmt)
    7172    357685142 :                                          && stmt_can_make_abnormal_goto (stmt));
    7173              : 
    7174              :           /* Substitute from our lattice.  We need to do so only once.  */
    7175    357685142 :           bool substituted_p = false;
    7176    357685142 :           use_operand_p usep;
    7177    357685142 :           ssa_op_iter iter;
    7178    524308322 :           FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
    7179              :             {
    7180    166623180 :               tree use = USE_FROM_PTR (usep);
    7181    166623180 :               tree val = fwprop_ssa_val (use);
    7182    166623180 :               if (val && val != use)
    7183              :                 {
    7184      1971858 :                   if (!is_gimple_debug (stmt))
    7185      1637733 :                     bitmap_set_bit (simple_dce_worklist, SSA_NAME_VERSION (use));
    7186      1971858 :                   if (may_propagate_copy (use, val))
    7187              :                     {
    7188      1968448 :                       propagate_value (usep, val);
    7189      1968448 :                       substituted_p = true;
    7190              :                     }
    7191              :                 }
    7192              :             }
    7193    357685142 :           if (substituted_p)
    7194      1911946 :             update_stmt (stmt);
    7195      1911946 :           if (substituted_p
    7196      1911946 :               && is_gimple_assign (stmt)
    7197      1159521 :               && gimple_assign_rhs_code (stmt) == ADDR_EXPR)
    7198        21058 :             recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
    7199    357685142 :           if (substituted_p
    7200    357685142 :               && can_make_abnormal_goto
    7201    357685142 :               && !stmt_can_make_abnormal_goto (stmt))
    7202            3 :             bitmap_set_bit (need_ab_cleanup, bb->index);
    7203              : 
    7204    360612202 :           bool changed;
    7205    721224404 :           do
    7206              :             {
    7207    360612202 :               gimple *orig_stmt = stmt = gsi_stmt (gsi);
    7208    360612202 :               bool was_call = is_gimple_call (stmt);
    7209    360612202 :               bool was_noreturn = (was_call
    7210    360612202 :                                    && gimple_call_noreturn_p (stmt));
    7211    360612202 :               changed = false;
    7212              : 
    7213    360612202 :               auto_vec<tree, 8> uses;
    7214    530272370 :               FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
    7215    338925978 :                 if (uses.space (1))
    7216    169265810 :                   uses.quick_push (USE_FROM_PTR (usep));
    7217              : 
    7218    360612202 :               if (fold_stmt (&gsi, fwprop_ssa_val, simple_dce_worklist))
    7219              :                 {
    7220      2612725 :                   changed = true;
    7221              :                   /* There is no updating of the address
    7222              :                      taken after the last forwprop so update
    7223              :                      the addresses when a folding happened to a call.
    7224              :                      The va_* builtins can remove taking of the address so
    7225              :                      can the sincos->cexpi transformation.  See PR 39643 and PR 20983. */
    7226      2612725 :                   if (was_call && last_p)
    7227      2612725 :                     todoflags |= TODO_update_address_taken;
    7228      2612725 :                   stmt = gsi_stmt (gsi);
    7229              :                   /* Cleanup the CFG if we simplified a condition to
    7230              :                      true or false.  */
    7231      2612725 :                   if (gcond *cond = dyn_cast <gcond *> (stmt))
    7232      1048216 :                     if (gimple_cond_true_p (cond)
    7233      1048216 :                         || gimple_cond_false_p (cond))
    7234        30895 :                       cfg_changed = true;
    7235              :                   /* Queue old uses for simple DCE if not debug statement.  */
    7236      2612725 :                   if (!is_gimple_debug (stmt))
    7237     11009465 :                     for (tree use : uses)
    7238      3192929 :                       if (TREE_CODE (use) == SSA_NAME
    7239      3192929 :                           && !SSA_NAME_IS_DEFAULT_DEF (use))
    7240      2994498 :                         bitmap_set_bit (simple_dce_worklist,
    7241      2994498 :                                         SSA_NAME_VERSION (use));
    7242      2612725 :                   update_stmt (stmt);
    7243              :                 }
    7244              : 
    7245    360612202 :               switch (gimple_code (stmt))
    7246              :                 {
    7247    109632307 :                 case GIMPLE_ASSIGN:
    7248    109632307 :                   {
    7249    109632307 :                     tree rhs1 = gimple_assign_rhs1 (stmt);
    7250    109632307 :                     enum tree_code code = gimple_assign_rhs_code (stmt);
    7251    109632307 :                     if (gimple_clobber_p (stmt))
    7252      7511610 :                       do_simple_agr_dse (as_a<gassign*>(stmt), full_walk);
    7253    102120697 :                     else if (gimple_store_p (stmt))
    7254              :                       {
    7255     31862179 :                         optimize_aggr_zeroprop (stmt, full_walk);
    7256     31862179 :                         if (gimple_assign_load_p (stmt))
    7257      3925834 :                           optimize_agr_copyprop (stmt);
    7258              :                       }
    7259     70258518 :                     else if (TREE_CODE_CLASS (code) == tcc_comparison)
    7260      2640354 :                       changed |= forward_propagate_into_comparison (&gsi);
    7261     67618164 :                     else if ((code == PLUS_EXPR || code == BIT_IOR_EXPR))
    7262              :                       {
    7263     10235596 :                         bool folded = match_long_mul (as_a <gassign *> (stmt));
    7264     10235596 :                         if (!folded)
    7265     10233650 :                           folded = simplify_rotate (&gsi);
    7266     10235596 :                         changed |= folded;
    7267              :                       }
    7268              :                     else if (code == BIT_XOR_EXPR)
    7269       149382 :                       changed |= simplify_rotate (&gsi);
    7270              :                     else if (code == VEC_PERM_EXPR)
    7271       193226 :                       changed |= simplify_permutation (&gsi);
    7272              :                     else if (code == CONSTRUCTOR
    7273       168113 :                              && TREE_CODE (TREE_TYPE (rhs1)) == VECTOR_TYPE)
    7274       168113 :                       changed |= simplify_vector_constructor (&gsi);
    7275     56871847 :                     else if (code == ARRAY_REF)
    7276      2020771 :                       changed |= simplify_count_zeroes (&gsi);
    7277     54851076 :                     else if (get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS
    7278     12694182 :                              && TREE_CODE (
    7279              :                                     gimple_assign_rhs2 (stmt)) == SSA_NAME
    7280      6723490 :                              && has_single_use (gimple_assign_rhs2 (stmt))
    7281     58989305 :                              && is_a<gphi*> (SSA_NAME_DEF_STMT (
    7282              :                                                 gimple_assign_rhs2 (stmt))))
    7283       134686 :                       changed |= simplify_phi_result_movdiv (stmt, code);
    7284              :                     break;
    7285              :                   }
    7286              : 
    7287       111600 :                 case GIMPLE_SWITCH:
    7288       111600 :                   changed |= simplify_gimple_switch (as_a <gswitch *> (stmt),
    7289              :                                                      edges_to_remove,
    7290              :                                                      simple_dce_worklist);
    7291       111600 :                   break;
    7292              : 
    7293     19792533 :                 case GIMPLE_COND:
    7294     19792533 :                   {
    7295     19792533 :                     int did_something = forward_propagate_into_gimple_cond
    7296     19792533 :                                                         (as_a <gcond *> (stmt));
    7297     19792533 :                     if (did_something == 2)
    7298         1618 :                       cfg_changed = true;
    7299     19792533 :                     changed |= did_something != 0;
    7300     19792533 :                     break;
    7301              :                   }
    7302              : 
    7303     23968931 :                 case GIMPLE_CALL:
    7304     23968931 :                   {
    7305     23968931 :                     tree callee = gimple_call_fndecl (stmt);
    7306     23968931 :                     if (callee != NULL_TREE
    7307     23968931 :                         && fndecl_built_in_p (callee, BUILT_IN_NORMAL))
    7308      6319213 :                       changed |= simplify_builtin_call (&gsi, callee, full_walk);
    7309              :                     break;
    7310              :                   }
    7311              : 
    7312    360612202 :                 default:;
    7313              :                 }
    7314              : 
    7315    360612202 :               if (changed || substituted_p)
    7316              :                 {
    7317      4267822 :                   substituted_p = false;
    7318      4267822 :                   stmt = gsi_stmt (gsi);
    7319      4267822 :                   if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
    7320           70 :                     bitmap_set_bit (to_purge, bb->index);
    7321      4267822 :                   if (!was_noreturn
    7322      4267822 :                       && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
    7323           12 :                     to_fixup.safe_push (stmt);
    7324              :                 }
    7325      4267822 :               if (changed)
    7326              :                 {
    7327              :                   /* If the stmt changed then re-visit it and the statements
    7328              :                      inserted before it.  */
    7329      6301255 :                   for (; !gsi_end_p (gsi); gsi_prev (&gsi))
    7330      5834843 :                     if (gimple_plf (gsi_stmt (gsi), GF_PLF_1))
    7331              :                       break;
    7332      2927060 :                   if (gsi_end_p (gsi))
    7333       466412 :                     gsi = gsi_start_bb (bb);
    7334              :                   else
    7335      2693854 :                     gsi_next (&gsi);
    7336              :                 }
    7337    360612202 :             }
    7338              :           while (changed);
    7339              : 
    7340              :           /* Stmt no longer needs to be revisited.  */
    7341    357685142 :           stmt = gsi_stmt (gsi);
    7342    357685142 :           gcc_checking_assert (!gimple_plf (stmt, GF_PLF_1));
    7343    357685142 :           gimple_set_plf (stmt, GF_PLF_1, true);
    7344              : 
    7345              :           /* Fill up the lattice.  */
    7346    357685142 :           if (gimple_assign_single_p (stmt))
    7347              :             {
    7348     72639799 :               tree lhs = gimple_assign_lhs (stmt);
    7349     72639799 :               tree rhs = gimple_assign_rhs1 (stmt);
    7350     72639799 :               if (TREE_CODE (lhs) == SSA_NAME)
    7351              :                 {
    7352     33279013 :                   tree val = lhs;
    7353     33279013 :                   if (TREE_CODE (rhs) == SSA_NAME)
    7354       785302 :                     val = fwprop_ssa_val (rhs);
    7355     32493711 :                   else if (is_gimple_min_invariant (rhs))
    7356       472932 :                     val = rhs;
    7357              :                   /* If we can propagate the lattice-value mark the
    7358              :                      stmt for removal.  */
    7359     33279013 :                   if (val != lhs
    7360     33279013 :                       && may_propagate_copy (lhs, val))
    7361      1254522 :                     to_remove_defs.safe_push (SSA_NAME_VERSION (lhs));
    7362     33279013 :                   fwprop_set_lattice_val (lhs, val);
    7363              :                 }
    7364              :             }
    7365    285045343 :           else if (gimple_nop_p (stmt))
    7366       102685 :             to_remove.safe_push (stmt);
    7367              :         }
    7368              : 
    7369              :       /* Substitute in destination PHI arguments.  */
    7370    110816285 :       FOR_EACH_EDGE (e, ei, bb->succs)
    7371     64788370 :         for (gphi_iterator gsi = gsi_start_phis (e->dest);
    7372    107616178 :              !gsi_end_p (gsi); gsi_next (&gsi))
    7373              :           {
    7374     42827808 :             gphi *phi = gsi.phi ();
    7375     42827808 :             use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
    7376     42827808 :             tree arg = USE_FROM_PTR (use_p);
    7377     70656276 :             if (TREE_CODE (arg) != SSA_NAME
    7378     42827808 :                 || virtual_operand_p (arg))
    7379     27828468 :               continue;
    7380     14999340 :             tree val = fwprop_ssa_val (arg);
    7381     14999340 :             if (val != arg
    7382     14999340 :                 && may_propagate_copy (arg, val, !(e->flags & EDGE_ABNORMAL)))
    7383       259609 :               propagate_value (use_p, val);
    7384              :           }
    7385              : 
    7386              :       /* Mark outgoing executable edges.  */
    7387     46027915 :       if (edge e = find_taken_edge (bb, NULL))
    7388              :         {
    7389     19641894 :           e->flags |= EDGE_EXECUTABLE;
    7390     46064732 :           if (EDGE_COUNT (bb->succs) > 1)
    7391        36817 :             cfg_changed = true;
    7392              :         }
    7393              :       else
    7394              :         {
    7395     71495679 :           FOR_EACH_EDGE (e, ei, bb->succs)
    7396     45109658 :             e->flags |= EDGE_EXECUTABLE;
    7397              :         }
    7398              :     }
    7399      5713356 :   free (postorder);
    7400      5713356 :   free (bb_to_rpo);
    7401      5713356 :   lattice.release ();
    7402              : 
    7403              :   /* First remove chains of stmts where we check no uses remain.  */
    7404      5713356 :   simple_dce_from_worklist (simple_dce_worklist, to_purge);
    7405              : 
    7406      6078383 :   auto remove = [](gimple *stmt)
    7407              :     {
    7408       365027 :       if (dump_file && (dump_flags & TDF_DETAILS))
    7409              :         {
    7410            1 :           fprintf (dump_file, "Removing dead stmt ");
    7411            1 :           print_gimple_stmt (dump_file, stmt, 0);
    7412            1 :           fprintf (dump_file, "\n");
    7413              :         }
    7414       365027 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    7415       365027 :       if (gimple_code (stmt) == GIMPLE_PHI)
    7416        95299 :         remove_phi_node (&gsi, true);
    7417              :       else
    7418              :         {
    7419       269728 :           unlink_stmt_vdef (stmt);
    7420       269728 :           gsi_remove (&gsi, true);
    7421       269728 :           release_defs (stmt);
    7422              :         }
    7423       365027 :     };
    7424              : 
    7425              :   /* Then remove stmts we know we can remove even though we did not
    7426              :      substitute in dead code regions, so uses can remain.  Do so in reverse
    7427              :      order to make debug stmt creation possible.  */
    7428     12928524 :   while (!to_remove_defs.is_empty())
    7429              :     {
    7430      1501812 :       tree def = ssa_name (to_remove_defs.pop ());
    7431              :       /* For example remove_prop_source_from_use can remove stmts queued
    7432              :          for removal.  Deal with this gracefully.  */
    7433      1501812 :       if (!def)
    7434      1239470 :         continue;
    7435       262342 :       gimple *stmt = SSA_NAME_DEF_STMT (def);
    7436       262342 :       remove (stmt);
    7437              :     }
    7438              : 
    7439              :   /* Wipe other queued stmts that do not have SSA defs.  */
    7440      5816041 :   while (!to_remove.is_empty())
    7441              :     {
    7442       102685 :       gimple *stmt = to_remove.pop ();
    7443       102685 :       remove (stmt);
    7444              :     }
    7445              : 
    7446              :   /* Fixup stmts that became noreturn calls.  This may require splitting
    7447              :      blocks and thus isn't possible during the walk.  Do this
    7448              :      in reverse order so we don't inadvertently remove a stmt we want to
    7449              :      fixup by visiting a dominating now noreturn call first.  */
    7450      5713368 :   while (!to_fixup.is_empty ())
    7451              :     {
    7452           12 :       gimple *stmt = to_fixup.pop ();
    7453           12 :       if (dump_file && dump_flags & TDF_DETAILS)
    7454              :         {
    7455            0 :           fprintf (dump_file, "Fixing up noreturn call ");
    7456            0 :           print_gimple_stmt (dump_file, stmt, 0);
    7457            0 :           fprintf (dump_file, "\n");
    7458              :         }
    7459           12 :       cfg_changed |= fixup_noreturn_call (stmt);
    7460              :     }
    7461              : 
    7462      5713356 :   cfg_changed |= gimple_purge_all_dead_eh_edges (to_purge);
    7463      5713356 :   cfg_changed |= gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
    7464      5713356 :   BITMAP_FREE (to_purge);
    7465              : 
    7466              :   /* Remove edges queued from switch stmt simplification.  */
    7467     17140068 :   for (auto ep : edges_to_remove)
    7468              :     {
    7469            0 :       basic_block src = BASIC_BLOCK_FOR_FN (fun, ep.first);
    7470            0 :       basic_block dest = BASIC_BLOCK_FOR_FN (fun, ep.second);
    7471            0 :       edge e;
    7472            0 :       if (src && dest && (e = find_edge (src, dest)))
    7473              :         {
    7474            0 :           free_dominance_info (CDI_DOMINATORS);
    7475            0 :           remove_edge (e);
    7476            0 :           cfg_changed = true;
    7477              :         }
    7478              :     }
    7479              : 
    7480      5713359 :   if (get_range_query (fun) != get_global_range_query ())
    7481         1616 :     disable_ranger (fun);
    7482              : 
    7483      5713356 :   if (cfg_changed)
    7484        13063 :     todoflags |= TODO_cleanup_cfg;
    7485              : 
    7486      5713356 :   return todoflags;
    7487      5713356 : }
    7488              : 
    7489              : } // anon namespace
    7490              : 
    7491              : gimple_opt_pass *
    7492       297180 : make_pass_forwprop (gcc::context *ctxt)
    7493              : {
    7494       297180 :   return new pass_forwprop (ctxt);
    7495              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.