LCOV - code coverage report
Current view: top level - gcc - tree-vect-slp.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 93.2 % 6197 5774
Test Date: 2026-11-14 16:39:50 Functions: 95.2 % 186 177
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* SLP - Basic Block Vectorization
       2              :    Copyright (C) 2007-2026 Free Software Foundation, Inc.
       3              :    Contributed by Dorit Naishlos <dorit@il.ibm.com>
       4              :    and Ira Rosen <irar@il.ibm.com>
       5              : 
       6              : This file is part of GCC.
       7              : 
       8              : GCC is free software; you can redistribute it and/or modify it under
       9              : the terms of the GNU General Public License as published by the Free
      10              : Software Foundation; either version 3, or (at your option) any later
      11              : version.
      12              : 
      13              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      14              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      15              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      16              : for more details.
      17              : 
      18              : You should have received a copy of the GNU General Public License
      19              : along with GCC; see the file COPYING3.  If not see
      20              : <http://www.gnu.org/licenses/>.  */
      21              : 
      22              : #include "config.h"
      23              : #define INCLUDE_ALGORITHM
      24              : #include "system.h"
      25              : #include "coretypes.h"
      26              : #include "backend.h"
      27              : #include "target.h"
      28              : #include "rtl.h"
      29              : #include "tree.h"
      30              : #include "gimple.h"
      31              : #include "tree-pass.h"
      32              : #include "ssa.h"
      33              : #include "optabs-tree.h"
      34              : #include "insn-config.h"
      35              : #include "recog.h"            /* FIXME: for insn_data */
      36              : #include "fold-const.h"
      37              : #include "stor-layout.h"
      38              : #include "gimple-iterator.h"
      39              : #include "cfgloop.h"
      40              : #include "tree-vectorizer.h"
      41              : #include "langhooks.h"
      42              : #include "gimple-walk.h"
      43              : #include "dbgcnt.h"
      44              : #include "tree-vector-builder.h"
      45              : #include "vec-perm-indices.h"
      46              : #include "gimple-fold.h"
      47              : #include "internal-fn.h"
      48              : #include "dump-context.h"
      49              : #include "cfganal.h"
      50              : #include "tree-eh.h"
      51              : #include "tree-cfg.h"
      52              : #include "alloc-pool.h"
      53              : #include "sreal.h"
      54              : #include "predict.h"
      55              : 
      56              : #define REDUC_GROUP_FIRST_ELEMENT(S) \
      57              :   (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
      58              : 
      59              : static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree,
      60              :                                             load_permutation_t &,
      61              :                                             const vec<tree> &,
      62              :                                             gimple_stmt_iterator *,
      63              :                                             poly_uint64, bool, bool,
      64              :                                             unsigned *,
      65              :                                             unsigned * = nullptr,
      66              :                                             bool = false);
      67              : static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *,
      68              :                                            slp_tree, lane_permutation_t &,
      69              :                                            vec<slp_tree> &, bool);
      70              : static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree);
      71              : static bool vect_slp_can_convert_to_external (const vec<stmt_vec_info> &);
      72              : 
      73              : static object_allocator<_slp_tree> *slp_tree_pool;
      74              : static slp_tree slp_first_node;
      75              : 
      76              : void
      77      1140025 : vect_slp_init (void)
      78              : {
      79      1140025 :   slp_tree_pool = new object_allocator<_slp_tree> ("SLP nodes");
      80      1140025 : }
      81              : 
      82              : void
      83      1140025 : vect_slp_fini (void)
      84              : {
      85      1835342 :   while (slp_first_node)
      86       695317 :     delete slp_first_node;
      87      2280050 :   delete slp_tree_pool;
      88      1140025 :   slp_tree_pool = NULL;
      89      1140025 : }
      90              : 
      91              : void *
      92      8391890 : _slp_tree::operator new (size_t n)
      93              : {
      94      8391890 :   gcc_assert (n == sizeof (_slp_tree));
      95      8391890 :   return slp_tree_pool->allocate_raw ();
      96              : }
      97              : 
      98              : void
      99      8391890 : _slp_tree::operator delete (void *node, size_t n)
     100              : {
     101      8391890 :   gcc_assert (n == sizeof (_slp_tree));
     102      8391890 :   slp_tree_pool->remove_raw (node);
     103      8391890 : }
     104              : 
     105              : 
     106              : /* Initialize a SLP node.  */
     107              : 
     108      8391890 : _slp_tree::_slp_tree ()
     109              : {
     110      8391890 :   this->prev_node = NULL;
     111      8391890 :   if (slp_first_node)
     112      7335874 :     slp_first_node->prev_node = this;
     113      8391890 :   this->next_node = slp_first_node;
     114      8391890 :   slp_first_node = this;
     115      8391890 :   SLP_TREE_SCALAR_STMTS (this) = vNULL;
     116      8391890 :   SLP_TREE_SCALAR_OPS (this) = vNULL;
     117      8391890 :   SLP_TREE_LIVE_LANES (this) = vNULL;
     118      8391890 :   SLP_TREE_VEC_DEFS (this) = vNULL;
     119      8391890 :   SLP_TREE_CHILDREN (this) = vNULL;
     120      8391890 :   SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
     121      8391890 :   SLP_TREE_LANE_PERMUTATION (this) = vNULL;
     122      8391890 :   SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
     123      8391890 :   SLP_TREE_CODE (this) = ERROR_MARK;
     124      8391890 :   SLP_TREE_GS_SCALE (this) = 0;
     125      8391890 :   SLP_TREE_GS_BASE (this) = NULL_TREE;
     126      8391890 :   this->ldst_lanes = false;
     127      8391890 :   this->avoid_stlf_fail = false;
     128      8391890 :   SLP_TREE_VECTYPE (this) = NULL_TREE;
     129      8391890 :   SLP_TREE_REPRESENTATIVE (this) = NULL;
     130      8391890 :   this->cycle_info.id = -1;
     131      8391890 :   this->cycle_info.reduc_idx = -1;
     132      8391890 :   SLP_TREE_REF_COUNT (this) = 1;
     133      8391890 :   this->failed = NULL;
     134      8391890 :   this->lanes = 0;
     135      8391890 :   SLP_TREE_TYPE (this) = undef_vec_info_type;
     136      8391890 :   this->data = NULL;
     137      8391890 :   this->si = NULL;
     138      8391890 : }
     139              : 
     140              : /* Tear down a SLP node.  */
     141              : 
     142      8391890 : _slp_tree::~_slp_tree ()
     143              : {
     144      8391890 :   if (this->prev_node)
     145      5127097 :     this->prev_node->next_node = this->next_node;
     146              :   else
     147      3264793 :     slp_first_node = this->next_node;
     148      8391890 :   if (this->next_node)
     149      6329361 :     this->next_node->prev_node = this->prev_node;
     150      8391890 :   SLP_TREE_CHILDREN (this).release ();
     151      8391890 :   SLP_TREE_SCALAR_STMTS (this).release ();
     152      8391890 :   SLP_TREE_SCALAR_OPS (this).release ();
     153      8391890 :   SLP_TREE_LIVE_LANES (this).release ();
     154      8391890 :   SLP_TREE_VEC_DEFS (this).release ();
     155      8391890 :   SLP_TREE_LOAD_PERMUTATION (this).release ();
     156      8391890 :   SLP_TREE_LANE_PERMUTATION (this).release ();
     157      8391890 :   if (this->failed)
     158      2188564 :     free (failed);
     159      8391890 :   if (this->data)
     160      1293306 :     delete this->data;
     161      8391890 : }
     162              : 
     163              : /* Push the single SSA definition in DEF to the vector of vector defs.  */
     164              : 
     165              : void
     166       518811 : _slp_tree::push_vec_def (gimple *def)
     167              : {
     168       518811 :   if (gphi *phi = dyn_cast <gphi *> (def))
     169        57696 :     vec_defs.quick_push (gimple_phi_result (phi));
     170              :   else
     171              :     {
     172       461115 :       def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS);
     173       461115 :       vec_defs.quick_push (get_def_from_ptr (defop));
     174              :     }
     175       518811 : }
     176              : 
     177              : /* Recursively free the memory allocated for the SLP tree rooted at NODE.  */
     178              : 
     179              : void
     180     15926858 : vect_free_slp_tree (slp_tree node)
     181              : {
     182     15926858 :   int i;
     183     15926858 :   slp_tree child;
     184              : 
     185     15926858 :   if (--SLP_TREE_REF_COUNT (node) != 0)
     186     15926858 :     return;
     187              : 
     188     12008787 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
     189      4312214 :     if (child)
     190      3911332 :       vect_free_slp_tree (child);
     191              : 
     192      7696573 :   delete node;
     193              : }
     194              : 
     195              : /* Return a location suitable for dumpings related to the SLP instance.  */
     196              : 
     197              : dump_user_location_t
     198      3560101 : _slp_instance::location () const
     199              : {
     200      3560101 :   if (!root_stmts.is_empty ())
     201       421891 :     return root_stmts[0]->stmt;
     202              :   else
     203      3138210 :     return SLP_TREE_SCALAR_STMTS (root)[0]->stmt;
     204              : }
     205              : 
     206              : 
     207              : /* Free the memory allocated for the SLP instance.  */
     208              : 
     209              : void
     210      1662560 : vect_free_slp_instance (slp_instance instance)
     211              : {
     212      1662560 :   vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
     213      1662560 :   SLP_INSTANCE_LOADS (instance).release ();
     214      1662560 :   SLP_INSTANCE_ROOT_STMTS (instance).release ();
     215      1662560 :   SLP_INSTANCE_REMAIN_DEFS (instance).release ();
     216      1662560 :   instance->subgraph_entries.release ();
     217      1662560 :   instance->cost_vec.release ();
     218      1662560 :   free (instance);
     219      1662560 : }
     220              : 
     221              : 
     222              : /* Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR
     223              :    for a permute node or else ERROR_MARK.  */
     224              : 
     225              : slp_tree
     226       101611 : vect_create_new_slp_node (unsigned nops, tree_code code)
     227              : {
     228       101611 :   gcc_assert (code == ERROR_MARK || code == VEC_PERM_EXPR);
     229       101611 :   slp_tree node = new _slp_tree;
     230       101611 :   SLP_TREE_SCALAR_STMTS (node) = vNULL;
     231       101611 :   SLP_TREE_CHILDREN (node).create (nops);
     232       101611 :   SLP_TREE_DEF_TYPE (node) = vect_internal_def;
     233       101611 :   SLP_TREE_CODE (node) = code;
     234       101611 :   return node;
     235              : }
     236              : 
     237              : /* Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children.  */
     238              : 
     239              : static slp_tree
     240      4083342 : vect_create_new_slp_node (slp_tree node,
     241              :                           vec<stmt_vec_info> scalar_stmts, unsigned nops)
     242              : {
     243      4083342 :   SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
     244      4083342 :   SLP_TREE_CHILDREN (node).create (nops);
     245      4083342 :   SLP_TREE_DEF_TYPE (node) = vect_internal_def;
     246      4083342 :   SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
     247      4083342 :   SLP_TREE_LANES (node) = scalar_stmts.length ();
     248      4083342 :   return node;
     249              : }
     250              : 
     251              : /* Create an SLP node for SCALAR_STMTS and NOPS children.  */
     252              : 
     253              : static slp_tree
     254        11988 : vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
     255              : {
     256        11988 :   return vect_create_new_slp_node (new _slp_tree, scalar_stmts, nops);
     257              : }
     258              : 
     259              : /* Create a vect_external_def SLP node inplace at NODE for scalar
     260              :    operands OPS.  */
     261              : 
     262              : static slp_tree
     263      2002734 : vect_create_new_slp_node (slp_tree node, vec<tree> ops)
     264              : {
     265      2002734 :   SLP_TREE_SCALAR_OPS (node) = ops;
     266      2002734 :   SLP_TREE_DEF_TYPE (node) = vect_external_def;
     267            0 :   SLP_TREE_LANES (node) = ops.length ();
     268      2002734 :   return node;
     269              : }
     270              : 
     271              : /* Create a vect_external_def SLP node for scalar operands OPS.  */
     272              : 
     273              : static slp_tree
     274      2002734 : vect_create_new_slp_node (vec<tree> ops)
     275              : {
     276      2002734 :   return vect_create_new_slp_node (new _slp_tree, ops);
     277              : }
     278              : 
     279              : 
     280              : /* This structure is used in creation of an SLP tree.  Each instance
     281              :    corresponds to the same operand in a group of scalar stmts in an SLP
     282              :    node.  */
     283              : typedef struct _slp_oprnd_info
     284              : {
     285              :   /* Def-stmts for the operands.  */
     286              :   vec<stmt_vec_info> def_stmts;
     287              :   /* Operands.  */
     288              :   vec<tree> ops;
     289              :   /* Information about the first statement, its vector def-type, type, the
     290              :      operand itself in case it's constant, and an indication if it's a pattern
     291              :      stmt and gather/scatter info.  */
     292              :   tree first_op_type;
     293              :   enum vect_def_type first_dt;
     294              :   bool any_pattern;
     295              :   bool first_gs_p;
     296              :   gather_scatter_info first_gs_info;
     297              : } *slp_oprnd_info;
     298              : 
     299              : 
     300              : /* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
     301              :    operand.  */
     302              : static vec<slp_oprnd_info>
     303      3741087 : vect_create_oprnd_info (int nops, int group_size)
     304              : {
     305      3741087 :   int i;
     306      3741087 :   slp_oprnd_info oprnd_info;
     307      3741087 :   vec<slp_oprnd_info> oprnds_info;
     308              : 
     309      3741087 :   oprnds_info.create (nops);
     310     13376821 :   for (i = 0; i < nops; i++)
     311              :     {
     312      5894647 :       oprnd_info = XNEW (struct _slp_oprnd_info);
     313      5894647 :       oprnd_info->def_stmts.create (group_size);
     314      5894647 :       oprnd_info->ops.create (group_size);
     315      5894647 :       oprnd_info->first_dt = vect_uninitialized_def;
     316      5894647 :       oprnd_info->first_op_type = NULL_TREE;
     317      5894647 :       oprnd_info->any_pattern = false;
     318      5894647 :       oprnd_info->first_gs_p = false;
     319      5894647 :       oprnds_info.quick_push (oprnd_info);
     320              :     }
     321              : 
     322      3741087 :   return oprnds_info;
     323              : }
     324              : 
     325              : 
     326              : /* Free operands info.  */
     327              : 
     328              : static void
     329      3618734 : vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
     330              : {
     331      3618734 :   int i;
     332      3618734 :   slp_oprnd_info oprnd_info;
     333              : 
     334      9382546 :   FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
     335              :     {
     336      5763812 :       oprnd_info->def_stmts.release ();
     337      5763812 :       oprnd_info->ops.release ();
     338      5763812 :       XDELETE (oprnd_info);
     339              :     }
     340              : 
     341      3618734 :   oprnds_info.release ();
     342      3618734 : }
     343              : 
     344              : /* Return the execution frequency of NODE (so that a higher value indicates
     345              :    a "more important" node when optimizing for speed).  */
     346              : 
     347              : static sreal
     348      3772972 : vect_slp_node_weight (vec_info *vinfo, slp_tree node)
     349              : {
     350      3772972 :   stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
     351      3772972 :   basic_block bb;
     352              :   /* ???  This is imprecise, VEC_PERM nodes do not have a representative
     353              :      but are laid out close to their children.  */
     354      3772972 :   if (!stmt_info)
     355       148959 :     bb = vinfo->bbs[0];
     356              :   else
     357      4121596 :     bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
     358      3772972 :   return bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
     359              : }
     360              : 
     361              : /* Return true if STMTS contains a pattern statement.  */
     362              : 
     363              : static bool
     364        25010 : vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
     365              : {
     366        25010 :   stmt_vec_info stmt_info;
     367        25010 :   unsigned int i;
     368        83484 :   FOR_EACH_VEC_ELT (stmts, i, stmt_info)
     369        60531 :     if (stmt_info && is_pattern_stmt_p (stmt_info))
     370              :       return true;
     371              :   return false;
     372              : }
     373              : 
     374              : /* Return true when all lanes in the external or constant NODE have
     375              :    the same value.  */
     376              : 
     377              : static bool
     378       650049 : vect_slp_tree_uniform_p (slp_tree node)
     379              : {
     380       650049 :   gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
     381              :               || SLP_TREE_DEF_TYPE (node) == vect_external_def);
     382              : 
     383              :   /* Pre-existing vectors.  */
     384       650049 :   if (SLP_TREE_SCALAR_OPS (node).is_empty ())
     385              :     return false;
     386              : 
     387              :   unsigned i;
     388              :   tree op, first = NULL_TREE;
     389      1489936 :   FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
     390      1330854 :     if (!first)
     391              :       first = op;
     392       680805 :     else if (!operand_equal_p (first, op, 0))
     393              :       return false;
     394              : 
     395              :   return true;
     396              : }
     397              : 
     398              : /* Find the place of the data-ref in STMT_INFO in the interleaving chain
     399              :    that starts from FIRST_STMT_INFO.  Return -1 if the data-ref is not a part
     400              :    of the chain.  */
     401              : 
     402              : int
     403       754762 : vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
     404              :                                       stmt_vec_info first_stmt_info)
     405              : {
     406       754762 :   stmt_vec_info next_stmt_info = first_stmt_info;
     407       754762 :   int result = 0;
     408              : 
     409       754762 :   if (first_stmt_info != DR_GROUP_FIRST_ELEMENT (stmt_info))
     410              :     return -1;
     411              : 
     412      2137967 :   do
     413              :     {
     414      2137967 :       if (next_stmt_info == stmt_info)
     415              :         return result;
     416      1383205 :       next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
     417      1383205 :       if (next_stmt_info)
     418      1383205 :         result += DR_GROUP_GAP (next_stmt_info);
     419              :     }
     420      1383205 :   while (next_stmt_info);
     421              : 
     422              :   return -1;
     423              : }
     424              : 
     425              : /* Check whether it is possible to load COUNT elements of type ELT_TYPE
     426              :    using the method implemented by duplicate_and_interleave.  Return true
     427              :    if so, returning the number of intermediate vectors in *NVECTORS_OUT
     428              :    (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
     429              :    (if nonnull).  */
     430              : 
     431              : bool
     432            0 : can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
     433              :                                 tree elt_type, unsigned int *nvectors_out,
     434              :                                 tree *vector_type_out,
     435              :                                 tree *permutes)
     436              : {
     437            0 :   tree base_vector_type = get_vectype_for_scalar_type (vinfo, elt_type, count);
     438            0 :   if (!base_vector_type || !VECTOR_MODE_P (TYPE_MODE (base_vector_type)))
     439              :     return false;
     440              : 
     441            0 :   machine_mode base_vector_mode = TYPE_MODE (base_vector_type);
     442            0 :   poly_int64 elt_bytes = count * GET_MODE_UNIT_SIZE (base_vector_mode);
     443            0 :   unsigned int nvectors = 1;
     444            0 :   for (;;)
     445              :     {
     446            0 :       scalar_int_mode int_mode;
     447            0 :       poly_int64 elt_bits = elt_bytes * BITS_PER_UNIT;
     448            0 :       if (int_mode_for_size (elt_bits, 1).exists (&int_mode))
     449              :         {
     450              :           /* Get the natural vector type for this SLP group size.  */
     451            0 :           tree int_type = build_nonstandard_integer_type
     452            0 :             (GET_MODE_BITSIZE (int_mode), 1);
     453            0 :           tree vector_type
     454            0 :             = get_vectype_for_scalar_type (vinfo, int_type, count);
     455            0 :           poly_int64 half_nelts;
     456            0 :           if (vector_type
     457            0 :               && VECTOR_MODE_P (TYPE_MODE (vector_type))
     458            0 :               && known_eq (GET_MODE_SIZE (TYPE_MODE (vector_type)),
     459              :                            GET_MODE_SIZE (base_vector_mode))
     460            0 :               && multiple_p (GET_MODE_NUNITS (TYPE_MODE (vector_type)),
     461              :                              2, &half_nelts))
     462              :             {
     463              :               /* Try fusing consecutive sequences of COUNT / NVECTORS elements
     464              :                  together into elements of type INT_TYPE and using the result
     465              :                  to build NVECTORS vectors.  */
     466            0 :               poly_uint64 nelts = GET_MODE_NUNITS (TYPE_MODE (vector_type));
     467            0 :               vec_perm_builder sel1 (nelts, 2, 3);
     468            0 :               vec_perm_builder sel2 (nelts, 2, 3);
     469              : 
     470            0 :               for (unsigned int i = 0; i < 3; ++i)
     471              :                 {
     472            0 :                   sel1.quick_push (i);
     473            0 :                   sel1.quick_push (i + nelts);
     474            0 :                   sel2.quick_push (half_nelts + i);
     475            0 :                   sel2.quick_push (half_nelts + i + nelts);
     476              :                 }
     477            0 :               vec_perm_indices indices1 (sel1, 2, nelts);
     478            0 :               vec_perm_indices indices2 (sel2, 2, nelts);
     479            0 :               machine_mode vmode = TYPE_MODE (vector_type);
     480            0 :               if (can_vec_perm_const_p (vmode, vmode, indices1)
     481            0 :                   && can_vec_perm_const_p (vmode, vmode, indices2))
     482              :                 {
     483            0 :                   if (nvectors_out)
     484            0 :                     *nvectors_out = nvectors;
     485            0 :                   if (vector_type_out)
     486            0 :                     *vector_type_out = vector_type;
     487            0 :                   if (permutes)
     488              :                     {
     489            0 :                       permutes[0] = vect_gen_perm_mask_checked (vector_type,
     490              :                                                                 indices1);
     491            0 :                       permutes[1] = vect_gen_perm_mask_checked (vector_type,
     492              :                                                                 indices2);
     493              :                     }
     494            0 :                   return true;
     495              :                 }
     496            0 :             }
     497              :         }
     498            0 :       if (!multiple_p (elt_bytes, 2, &elt_bytes))
     499              :         return false;
     500            0 :       nvectors *= 2;
     501              :       /* We need to be able to fuse COUNT / NVECTORS elements together.  */
     502            0 :       if (!multiple_p (count, nvectors))
     503              :         return false;
     504              :     }
     505              : }
     506              : 
     507              : /* Return true if DTA and DTB match.  */
     508              : 
     509              : static bool
     510     17502462 : vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
     511              : {
     512     17502462 :   return (dta == dtb
     513       416170 :           || ((dta == vect_external_def || dta == vect_constant_def)
     514       266638 :               && (dtb == vect_external_def || dtb == vect_constant_def)));
     515              : }
     516              : 
     517              : #define GATHER_SCATTER_OFFSET (-3)
     518              : 
     519              : /* For most SLP statements, there is a one-to-one mapping between
     520              :    gimple arguments and child nodes.  If that is not true for STMT,
     521              :    return an array that contains:
     522              : 
     523              :    - the number of child nodes, followed by
     524              :    - for each child node, the index of the argument associated with that node.
     525              :      The special index -1 is the first operand of an embedded comparison and
     526              :      the special index -2 is the second operand of an embedded comparison.
     527              :      The special index -3 is the offset of a gather as analyzed by
     528              :      vect_check_gather_scatter.
     529              : 
     530              :    SWAP is as for vect_get_and_check_slp_defs.  */
     531              : 
     532              : static const int *
     533     26014995 : vect_get_operand_map (const gimple *stmt, bool gather_scatter_p,
     534              :                       unsigned char swap)
     535              : {
     536     26014995 :   static const int no_arg_map[] = { 0 };
     537     26014995 :   static const int arg0_map[] = { 1, 0 };
     538     26014995 :   static const int arg2_map[] = { 1, 2 };
     539     26014995 :   static const int arg2_arg3_map[] = { 2, 2, 3 };
     540     26014995 :   static const int arg2_arg4_map[] = { 2, 2, 4 };
     541     26014995 :   static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
     542     26014995 :   static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
     543     26014995 :   static const int arg3_arg2_map[] = { 2, 3, 2 };
     544     26014995 :   static const int op00_map[] = { 1, -1 };
     545     26014995 :   static const int op1_op0_map[] = { 2, 1, 0 };
     546     26014995 :   static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
     547     26014995 :   static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
     548     26014995 :   static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
     549     26014995 :   static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
     550     26014995 :   static const int mask_call_maps[6][7] = {
     551              :         { 1, 1, },
     552              :         { 2, 1, 2, },
     553              :         { 3, 1, 2, 3, },
     554              :         { 4, 1, 2, 3, 4, },
     555              :         { 5, 1, 2, 3, 4, 5, },
     556              :         { 6, 1, 2, 3, 4, 5, 6 },
     557              :   };
     558              : 
     559     26014995 :   gcc_checking_assert (!swap
     560              :                        || !is_gimple_assign (stmt)
     561              :                        || TREE_CODE_CLASS
     562              :                             (gimple_assign_rhs_code (stmt)) == tcc_comparison
     563              :                        || commutative_tree_code
     564              :                             (gimple_assign_rhs_code (stmt)));
     565              : 
     566     26014995 :   if (auto assign = dyn_cast<const gassign *> (stmt))
     567              :     {
     568     24464987 :       tree_code code = gimple_assign_rhs_code (assign);
     569     24464987 :       if (code == COND_EXPR
     570     24464987 :           && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
     571            0 :         gcc_unreachable ();
     572     24464987 :       else if ((TREE_CODE_CLASS (code) == tcc_comparison
     573     22917696 :                 || commutative_tree_code (code))
     574     33525614 :                && swap)
     575              :         return op1_op0_map;
     576     24422705 :       else if (code == VIEW_CONVERT_EXPR)
     577              :         return op00_map;
     578     24413217 :       else if (gather_scatter_p)
     579        54752 :         return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
     580        54752 :                 ? off_op0_map : off_map);
     581              :     }
     582      1550008 :   else if (auto call = dyn_cast<const gcall *> (stmt))
     583              :     {
     584       174117 :       if (gimple_call_internal_p (call))
     585        95098 :         switch (gimple_call_internal_fn (call))
     586              :           {
     587        13996 :           case IFN_MASK_LOAD:
     588        13996 :             return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
     589              : 
     590            0 :           case IFN_GATHER_LOAD:
     591            0 :             return arg2_map;
     592              : 
     593            0 :           case IFN_MASK_GATHER_LOAD:
     594            0 :           case IFN_MASK_LEN_GATHER_LOAD:
     595            0 :             return arg2_arg5_arg6_map;
     596              : 
     597            0 :           case IFN_SCATTER_STORE:
     598            0 :             return arg2_arg4_map;
     599              : 
     600            0 :           case IFN_MASK_SCATTER_STORE:
     601            0 :           case IFN_MASK_LEN_SCATTER_STORE:
     602            0 :             return arg2_arg4_arg5_map;
     603              : 
     604         9326 :           case IFN_MASK_STORE:
     605         9326 :             return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
     606              : 
     607         1090 :           case IFN_MASK_CALL:
     608         1090 :             {
     609         1090 :               unsigned nargs = gimple_call_num_args (call);
     610         1090 :               if (nargs >= 2 && nargs <= 7)
     611         1090 :                 return mask_call_maps[nargs-2];
     612              :               else
     613              :                 return nullptr;
     614              :             }
     615              : 
     616          437 :           case IFN_CLZ:
     617          437 :           case IFN_CTZ:
     618          437 :             return arg0_map;
     619              : 
     620         7308 :           case IFN_GOMP_SIMD_LANE:
     621         7308 :             return no_arg_map;
     622              : 
     623              :           default:
     624              :             break;
     625              :           }
     626              :     }
     627              :   return nullptr;
     628              : }
     629              : 
     630              : static const int *
     631     25995787 : vect_get_operand_map (const stmt_vec_info stmt, unsigned char swap = 0)
     632              : {
     633            0 :   return vect_get_operand_map (stmt->stmt, STMT_VINFO_GATHER_SCATTER_P (stmt),
     634            0 :                                swap);
     635              : }
     636              : 
     637              : /* Return the SLP node child index for operand OP of STMT.  */
     638              : 
     639              : int
     640      1398238 : vect_slp_child_index_for_operand (const stmt_vec_info stmt, int op)
     641              : {
     642      1398238 :   const int *opmap = vect_get_operand_map (stmt);
     643      1398238 :   if (!opmap)
     644              :     return op;
     645        20109 :   for (int i = 1; i < 1 + opmap[0]; ++i)
     646        20109 :     if (opmap[i] == op)
     647        11495 :       return i - 1;
     648            0 :   gcc_unreachable ();
     649              : }
     650              : 
     651              : /* Helper class for mapping of GIMPLE operands to SLP children.  */
     652              : /* ???  Add vect_slp_child_index_for_operand here and amend opmaps
     653              :    with the full reverse mapping and indicating the position of the
     654              :    first commutative operand index, eliding the swap_p argument from
     655              :    vect_get_operand_map.  Adjust all consumers.  */
     656              : 
     657              : struct slp_oprnds {
     658              :   slp_oprnds (stmt_vec_info);
     659              :   tree get_op_for_slp_child (stmt_vec_info, unsigned);
     660              :   const int *opmap;
     661              :   const unsigned int num_slp_children;
     662              : };
     663              : 
     664      4677206 : slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
     665      4677206 :   : opmap (vect_get_operand_map (stmt_info)),
     666      4677206 :     num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
     667              : {
     668      4677206 : }
     669              : 
     670              : /* For SLP child number N get the corresponding tree operand from GIMPLE
     671              :    statement described by STMT_INFO.  */
     672              : 
     673              : tree
     674      5236105 : slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
     675              : {
     676      5236105 :   gcc_assert (n < num_slp_children);
     677      5236105 :   int opno = opmap ? opmap[n + 1] : (int) n;
     678      5236105 :   if (opno == GATHER_SCATTER_OFFSET)
     679            0 :     gcc_unreachable (); // TODO
     680      5236105 :   else if (opno < 0)
     681         2602 :     return TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
     682              :   else
     683      5233503 :     return gimple_arg (stmt_info->stmt, opno);
     684              : }
     685              : 
     686              : /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
     687              :    they are of a valid type and that they match the defs of the first stmt of
     688              :    the SLP group (stored in OPRNDS_INFO).  This function tries to match stmts
     689              :    by swapping operands of STMTS[STMT_NUM] when possible.  Non-zero SWAP
     690              :    indicates swap is required for cond_expr stmts.  Specifically, SWAP
     691              :    is 1 if STMT is cond and operands of comparison need to be swapped;
     692              :    SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
     693              : 
     694              :    If there was a fatal error return -1; if the error could be corrected by
     695              :    swapping operands of father node of this one, return 1; if everything is
     696              :    ok return 0.  */
     697              : static int
     698     13663956 : vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
     699              :                              bool *skip_args,
     700              :                              vec<stmt_vec_info> stmts, unsigned stmt_num,
     701              :                              vec<slp_oprnd_info> *oprnds_info,
     702              :                              bool soft_fail)
     703              : {
     704     13663956 :   stmt_vec_info stmt_info = stmts[stmt_num];
     705     13663956 :   tree oprnd;
     706     13663956 :   unsigned int i, number_of_oprnds;
     707     13663956 :   enum vect_def_type dt = vect_uninitialized_def;
     708     13663956 :   slp_oprnd_info oprnd_info;
     709     13663956 :   gather_scatter_info gs_info;
     710     13663956 :   unsigned int gs_op = -1u;
     711     13663956 :   unsigned int commutative_op = -1U;
     712     13663956 :   bool first = stmt_num == 0;
     713              : 
     714     13663956 :   if (!stmt_info)
     715              :     {
     716            0 :       for (auto oi : *oprnds_info)
     717              :         {
     718            0 :           oi->def_stmts.quick_push (NULL);
     719            0 :           oi->ops.quick_push (NULL_TREE);
     720              :         }
     721              :       return 0;
     722              :     }
     723              : 
     724     13663956 :   if (!is_a<gcall *> (stmt_info->stmt)
     725              :       && !is_a<gassign *> (stmt_info->stmt)
     726              :       && !is_a<gphi *> (stmt_info->stmt))
     727              :     return -1;
     728              : 
     729     13663956 :   number_of_oprnds = gimple_num_args (stmt_info->stmt);
     730     13663956 :   const int *map = vect_get_operand_map (stmt_info, swap);
     731     13663956 :   if (map)
     732        84035 :     number_of_oprnds = *map++;
     733     13663956 :   if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
     734              :     {
     735        56035 :       if (gimple_call_internal_p (stmt))
     736              :         {
     737        34324 :           internal_fn ifn = gimple_call_internal_fn (stmt);
     738        34324 :           commutative_op = first_commutative_argument (ifn);
     739        34324 :           if (internal_gather_scatter_fn_p (ifn))
     740              :             {
     741            0 :               vect_describe_gather_scatter_call
     742            0 :                 (stmt_info,
     743            0 :                  first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
     744            0 :               if (first)
     745            0 :                 (*oprnds_info)[0]->first_gs_p = true;
     746              :               gs_op = 0;
     747              :             }
     748              :         }
     749              :     }
     750     13607921 :   else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
     751              :     {
     752     16379941 :       if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
     753      8459527 :         commutative_op = 0;
     754              :     }
     755              : 
     756     13663956 :   bool swapped = (swap != 0);
     757     13663956 :   bool backedge = false;
     758     13663956 :   enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
     759     37255039 :   for (i = 0; i < number_of_oprnds; i++)
     760              :     {
     761     23592471 :       oprnd_info = (*oprnds_info)[i];
     762     23592471 :       int opno = map ? map[i] : int (i);
     763     23592471 :       if (opno == GATHER_SCATTER_OFFSET)
     764              :         {
     765        28351 :           gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
     766        28351 :           if (!is_a <loop_vec_info> (vinfo)
     767        28351 :               || !vect_check_gather_scatter (stmt_info, vectype,
     768              :                                              as_a <loop_vec_info> (vinfo),
     769              :                                              first ? &oprnd_info->first_gs_info
     770              :                                              : &gs_info))
     771         1388 :             return -1;
     772              : 
     773        28351 :           if (first)
     774              :             {
     775        27170 :               oprnd_info->first_gs_p = true;
     776        27170 :               oprnd = oprnd_info->first_gs_info.offset;
     777              :             }
     778              :           else
     779              :             {
     780         1181 :               gs_op = i;
     781         1181 :               oprnd = gs_info.offset;
     782              :             }
     783              :         }
     784     23564120 :       else if (opno < 0)
     785         3267 :         oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
     786              :       else
     787              :         {
     788     23560853 :           oprnd = gimple_arg (stmt_info->stmt, opno);
     789     23560853 :           if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
     790              :             {
     791      1263511 :               edge e = gimple_phi_arg_edge (stmt, opno);
     792      2527022 :               backedge = (is_a <bb_vec_info> (vinfo)
     793      1950793 :                           ? e->flags & EDGE_DFS_BACK
     794       687282 :                           : dominated_by_p (CDI_DOMINATORS, e->src,
     795       687282 :                                             gimple_bb (stmt_info->stmt)));
     796              :             }
     797              :         }
     798              : 
     799     23592471 :       stmt_vec_info def_stmt_info;
     800     23592471 :       if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
     801              :         {
     802         1064 :           if (dump_enabled_p ())
     803            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     804              :                              "Build SLP failed: can't analyze def for %T\n",
     805              :                              oprnd);
     806              : 
     807              :           return -1;
     808              :         }
     809              : 
     810     23591407 :       if (skip_args[i])
     811              :         {
     812       561462 :           oprnd_info->def_stmts.quick_push (NULL);
     813       561462 :           oprnd_info->ops.quick_push (NULL_TREE);
     814       561462 :           oprnd_info->first_dt = vect_uninitialized_def;
     815       561462 :           continue;
     816              :         }
     817              : 
     818     23029945 :       oprnd_info->def_stmts.quick_push (def_stmt_info);
     819     23029945 :       oprnd_info->ops.quick_push (oprnd);
     820              : 
     821     23029945 :       if (def_stmt_info
     822     23029945 :           && is_pattern_stmt_p (def_stmt_info))
     823              :         {
     824       473498 :           if (STMT_VINFO_RELATED_STMT (vect_orig_stmt (def_stmt_info))
     825              :               != def_stmt_info)
     826       326473 :             oprnd_info->any_pattern = true;
     827              :           else
     828              :             /* If we promote this to external use the original stmt def.  */
     829       147025 :             oprnd_info->ops.last ()
     830       294050 :               = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
     831              :         }
     832              : 
     833              :       /* If there's a extern def on a backedge make sure we can
     834              :          code-generate at the region start.
     835              :          ???  This is another case that could be fixed by adjusting
     836              :          how we split the function but at the moment we'd have conflicting
     837              :          goals there.  */
     838     23029945 :       if (backedge
     839       163836 :           && dts[i] == vect_external_def
     840          345 :           && is_a <bb_vec_info> (vinfo)
     841          345 :           && TREE_CODE (oprnd) == SSA_NAME
     842          324 :           && !SSA_NAME_IS_DEFAULT_DEF (oprnd)
     843     23030269 :           && !dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
     844          324 :                               gimple_bb (SSA_NAME_DEF_STMT (oprnd))))
     845              :         {
     846          324 :           if (dump_enabled_p ())
     847            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     848              :                              "Build SLP failed: extern def %T only defined "
     849              :                              "on backedge\n", oprnd);
     850              :           return -1;
     851              :         }
     852              : 
     853     23029621 :       if (first)
     854              :         {
     855      5358901 :           tree type = TREE_TYPE (oprnd);
     856      5358901 :           dt = dts[i];
     857              : 
     858              :           /* For the swapping logic below force vect_reduction_def
     859              :              for the reduction op in a SLP reduction group.  */
     860      5358901 :           if (!STMT_VINFO_DATA_REF (stmt_info)
     861      4026653 :               && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
     862         1804 :               && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
     863      5359760 :               && def_stmt_info)
     864          859 :             dts[i] = dt = vect_reduction_def;
     865              : 
     866              :           /* Check the types of the definition.  */
     867      5358901 :           switch (dt)
     868              :             {
     869      5358901 :             case vect_external_def:
     870      5358901 :             case vect_constant_def:
     871      5358901 :             case vect_internal_def:
     872      5358901 :             case vect_reduction_def:
     873      5358901 :             case vect_double_reduction_def:
     874      5358901 :             case vect_induction_def:
     875      5358901 :             case vect_nested_cycle:
     876      5358901 :             case vect_first_order_recurrence:
     877      5358901 :               break;
     878              : 
     879            0 :             default:
     880              :               /* FORNOW: Not supported.  */
     881            0 :               if (dump_enabled_p ())
     882            0 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     883              :                                  "Build SLP failed: illegal type of def %T\n",
     884              :                                  oprnd);
     885              :               return -1;
     886              :             }
     887              : 
     888      5358901 :           oprnd_info->first_dt = dt;
     889      5358901 :           oprnd_info->first_op_type = type;
     890              :         }
     891              :     }
     892     13662568 :   if (first)
     893              :     return 0;
     894              : 
     895              :   /* Now match the operand definition types to that of the first stmt.  */
     896     27231155 :   for (i = 0; i < number_of_oprnds;)
     897              :     {
     898     17646202 :       if (skip_args[i])
     899              :         {
     900        30337 :           ++i;
     901        30337 :           continue;
     902              :         }
     903              : 
     904     17615865 :       oprnd_info = (*oprnds_info)[i];
     905     17615865 :       dt = dts[i];
     906     17615865 :       stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
     907     17615865 :       oprnd = oprnd_info->ops[stmt_num];
     908     17615865 :       tree type = TREE_TYPE (oprnd);
     909              : 
     910     17615865 :       if (!types_compatible_p (oprnd_info->first_op_type, type))
     911              :         {
     912       120735 :           if (dump_enabled_p ())
     913           93 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     914              :                              "Build SLP failed: different operand types\n");
     915              :           return 1;
     916              :         }
     917              : 
     918     17495130 :       if ((gs_op == i) != oprnd_info->first_gs_p)
     919              :         {
     920            0 :           if (dump_enabled_p ())
     921            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     922              :                              "Build SLP failed: mixed gather and non-gather\n");
     923              :           return 1;
     924              :         }
     925     17495130 :       else if (gs_op == i)
     926              :         {
     927         1149 :           if (!operand_equal_p (oprnd_info->first_gs_info.base,
     928         1149 :                                 gs_info.base))
     929              :             {
     930           16 :               if (dump_enabled_p ())
     931            6 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     932              :                                  "Build SLP failed: different gather base\n");
     933              :               return 1;
     934              :             }
     935         1133 :           if (oprnd_info->first_gs_info.scale != gs_info.scale)
     936              :             {
     937            8 :               if (dump_enabled_p ())
     938            2 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     939              :                                  "Build SLP failed: different gather scale\n");
     940              :               return 1;
     941              :             }
     942              :         }
     943              : 
     944              :       /* Not first stmt of the group, check that the def-stmt/s match
     945              :          the def-stmt/s of the first stmt.  Allow different definition
     946              :          types for reduction chains: the first stmt must be a
     947              :          vect_reduction_def (a phi node), and the rest
     948              :          end in the reduction chain.  */
     949     17495106 :       if ((!vect_def_types_match (oprnd_info->first_dt, dt)
     950       334013 :            && !(oprnd_info->first_dt == vect_reduction_def
     951         3269 :                 && !STMT_VINFO_DATA_REF (stmt_info)
     952         3269 :                 && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
     953         3243 :                 && def_stmt_info
     954         3241 :                 && !STMT_VINFO_DATA_REF (def_stmt_info)
     955         3241 :                 && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
     956              :                     == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
     957     17164334 :           || (!STMT_VINFO_DATA_REF (stmt_info)
     958     15376091 :               && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
     959         6927 :               && ((!def_stmt_info
     960         6691 :                    || STMT_VINFO_DATA_REF (def_stmt_info)
     961        11656 :                    || (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
     962              :                        != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
     963         6927 :                   != (oprnd_info->first_dt != vect_reduction_def))))
     964              :         {
     965              :           /* Try swapping operands if we got a mismatch.  For BB
     966              :              vectorization only in case it will clearly improve things.  */
     967       334371 :           if (i == commutative_op && !swapped
     968       330772 :               && (!is_a <bb_vec_info> (vinfo)
     969         5655 :                   || (!vect_def_types_match ((*oprnds_info)[i+1]->first_dt,
     970         5655 :                                              dts[i+1])
     971         1468 :                       && (vect_def_types_match (oprnd_info->first_dt, dts[i+1])
     972              :                           || vect_def_types_match
     973          233 :                                ((*oprnds_info)[i+1]->first_dt, dts[i])))))
     974              :             {
     975         3599 :               if (dump_enabled_p ())
     976          194 :                 dump_printf_loc (MSG_NOTE, vect_location,
     977              :                                  "trying swapped operands\n");
     978         3599 :               std::swap (dts[i], dts[i+1]);
     979         3599 :               std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
     980         3599 :                          (*oprnds_info)[i+1]->def_stmts[stmt_num]);
     981         3599 :               std::swap ((*oprnds_info)[i]->ops[stmt_num],
     982         3599 :                          (*oprnds_info)[i+1]->ops[stmt_num]);
     983              :               /* After swapping some operands we lost track whether an
     984              :                  operand has any pattern defs so be conservative here.  */
     985         3599 :               if ((*oprnds_info)[i]->any_pattern
     986         3599 :                   || (*oprnds_info)[i+1]->any_pattern)
     987           36 :                 (*oprnds_info)[i]->any_pattern
     988           18 :                   = (*oprnds_info)[i+1]->any_pattern = true;
     989         3599 :               swapped = true;
     990         3599 :               continue;
     991              :             }
     992              : 
     993       327173 :           if (is_a <bb_vec_info> (vinfo)
     994       308935 :               && !oprnd_info->any_pattern
     995       308667 :               && number_of_oprnds > 1
     996       635840 :               && !soft_fail)
     997              :             {
     998              :               /* Now for commutative ops we should see whether we can
     999              :                  make the other operand matching.  */
    1000       107227 :               if (dump_enabled_p ())
    1001          336 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1002              :                                  "treating operand as external\n");
    1003              :               oprnd_info->first_dt = dt = vect_external_def;
    1004              :             }
    1005              :           else
    1006              :             {
    1007       219946 :               if (dump_enabled_p ())
    1008          412 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1009              :                                  "Build SLP failed: different types\n");
    1010              :               return 1;
    1011              :             }
    1012              :         }
    1013              : 
    1014              :       /* Make sure to demote the overall operand to external.  */
    1015     17164334 :       if (dt == vect_external_def)
    1016       422267 :         oprnd_info->first_dt = vect_external_def;
    1017              :       /* For a SLP reduction chain we want to duplicate the reduction to
    1018              :          each of the chain members.  That gets us a sane SLP graph (still
    1019              :          the stmts are not 100% correct wrt the initial values).  */
    1020     16849294 :       else if ((dt == vect_internal_def
    1021     16849294 :                 || dt == vect_reduction_def)
    1022     15589967 :                && oprnd_info->first_dt == vect_reduction_def
    1023        49214 :                && !STMT_VINFO_DATA_REF (stmt_info)
    1024        49214 :                && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
    1025         3241 :                && !STMT_VINFO_DATA_REF (def_stmt_info)
    1026     16852535 :                && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
    1027              :                    == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
    1028              :         {
    1029         3241 :           oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
    1030         3241 :           oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
    1031              :         }
    1032              : 
    1033     17271561 :       ++i;
    1034              :     }
    1035              : 
    1036              :   /* Swap operands.  */
    1037      9584953 :   if (swapped)
    1038              :     {
    1039        42001 :       if (dump_enabled_p ())
    1040          526 :         dump_printf_loc (MSG_NOTE, vect_location,
    1041              :                          "swapped operands to match def types in %G",
    1042              :                          stmt_info->stmt);
    1043              :     }
    1044              : 
    1045              :   return 0;
    1046              : }
    1047              : 
    1048              : /* Return true if call statements CALL1 and CALL2 are similar enough
    1049              :    to be combined into the same SLP group.  */
    1050              : 
    1051              : bool
    1052        65765 : compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators)
    1053              : {
    1054        65765 :   unsigned int nargs = gimple_call_num_args (call1);
    1055        65765 :   if (nargs != gimple_call_num_args (call2))
    1056              :     return false;
    1057              : 
    1058        48666 :   auto cfn1 = gimple_call_combined_fn (call1);
    1059        48666 :   auto cfn2 = gimple_call_combined_fn (call2);
    1060        48666 :   if (cfn1 != cfn2
    1061           39 :       && (!allow_two_operators
    1062           39 :           || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS)
    1063            2 :                && (cfn2 == CFN_FMA || cfn2 == CFN_FMS))))
    1064              :     return false;
    1065              : 
    1066        48629 :   if (gimple_call_internal_p (call1))
    1067              :     {
    1068         7777 :       if (!types_compatible_p (TREE_TYPE (gimple_call_lhs (call1)),
    1069         7777 :                                TREE_TYPE (gimple_call_lhs (call2))))
    1070              :         return false;
    1071        16495 :       for (unsigned int i = 0; i < nargs; ++i)
    1072         8718 :         if (!types_compatible_p (TREE_TYPE (gimple_call_arg (call1, i)),
    1073         8718 :                                  TREE_TYPE (gimple_call_arg (call2, i))))
    1074              :           return false;
    1075              :     }
    1076              :   else
    1077              :     {
    1078        40852 :       if (!operand_equal_p (gimple_call_fn (call1),
    1079        40852 :                             gimple_call_fn (call2), 0))
    1080              :         return false;
    1081              : 
    1082        34293 :       if (gimple_call_fntype (call1) != gimple_call_fntype (call2))
    1083              :         return false;
    1084              :     }
    1085              : 
    1086              :   /* Check that any unvectorized arguments are equal.  */
    1087        19208 :   if (const int *map = vect_get_operand_map (call1, false, false))
    1088              :     {
    1089           65 :       unsigned int nkept = *map++;
    1090           65 :       unsigned int mapi = 0;
    1091          207 :       for (unsigned int i = 0; i < nargs; ++i)
    1092          142 :         if (mapi < nkept && map[mapi] == int (i))
    1093           77 :           mapi += 1;
    1094           65 :         else if (!operand_equal_p (gimple_call_arg (call1, i),
    1095           65 :                                    gimple_call_arg (call2, i)))
    1096              :           return false;
    1097              :     }
    1098              : 
    1099              :   return true;
    1100              : }
    1101              : 
    1102              : /* Verify if the scalar stmts STMTS are isomorphic, require data
    1103              :    permutation or are of unsupported types of operation.
    1104              :    Return false if at least one (or all in case of BB vectorization)
    1105              :    stmt is unvectorizable or the comparison could not be carried out.
    1106              :    Return true if they all (or at least one in case of BB vectorization)
    1107              :    are and indicate in MATCHES[] which stmts are not isomorphic to the
    1108              :    stmt at *START_I.
    1109              : 
    1110              :    This function is designed to be invoked repeatedly on the same
    1111              :    set of STMTS with increasing *START_I.
    1112              : 
    1113              :    When *START_I is zero and the function returns true MATCHES[] will be
    1114              :    initialized with, in case of BB vectorization, unvectorizable stmts
    1115              :    marked with -2, stmts isomorphic to *START_I as *START_I and
    1116              :    other vectorizable stmts with -1 (not isomorphic to *START_I).
    1117              :    *START_I, when initially zero, is updated to the first vectorizable
    1118              :    statement, so MATCHES[] can have a prefix with entries valued -2.
    1119              : 
    1120              :    When *START_I is not zero MATCHES[] is expected to be pre-initialized
    1121              :    by a former call with *START_I zero and MATCHES[*START_I] should be -1.
    1122              :    The function will return true and have the stmts isomorphic to
    1123              :    MATCHES[*START_I] marked with *START_I.
    1124              : 
    1125              :    *TWO_OPERATORS indicates whether the group of isomorphic statements
    1126              :    uses two related operations like PLUS_EXPR and MINUS_EXPR.  If
    1127              :    TWO_OPERATORS is NULL such case is not considered isomorphic.
    1128              : 
    1129              :    SWAP[] indicates whether for a stmt to be isomorphic to its group
    1130              :    leader, has to have its operands swapped (1) or its predicate inverted (2).
    1131              :    COND_EXPR is possibly isomorphic to another one after swapping its
    1132              :    operands.  Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
    1133              :    the first stmt by swapping the two operands of comparison; set SWAP[i]
    1134              :    to 2 if stmt I is isormorphic to the first stmt by inverting the code
    1135              :    of comparison.  Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
    1136              :    to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1.  */
    1137              : 
    1138              : static bool
    1139      6346506 : vect_build_slp_tree_3 (vec_info *vinfo, vec<stmt_vec_info> stmts,
    1140              :                        match_elt_t *matches, unsigned char *swap,
    1141              :                        bool *two_operators, tree vectype, unsigned *start_i)
    1142              : {
    1143      6346506 :   stmt_vec_info first_stmt_info = NULL;
    1144      6346506 :   code_helper first_stmt_code = ERROR_MARK;
    1145      6346506 :   code_helper alt_stmt_code = ERROR_MARK;
    1146      6346506 :   code_helper first_cond_code = ERROR_MARK;
    1147      6346506 :   bool need_same_oprnds = false;
    1148      6346506 :   tree first_lhs = NULL_TREE;
    1149      6346506 :   tree first_op1 = NULL_TREE;
    1150      6346506 :   stmt_vec_info first_load = NULL, prev_first_load = NULL;
    1151      6346506 :   bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
    1152      6346506 :   bool first_stmt_phi_p = false;
    1153      6346506 :   int first_reduc_idx = -1;
    1154              : 
    1155      6346506 :   basic_block common_bb = NULL;
    1156      6346506 :   gimple *trapping_stmt = NULL;
    1157      6346506 :   int first_match = -1;
    1158              : 
    1159     26163365 :   for (unsigned i = *start_i; i < stmts.length (); ++i)
    1160              :     {
    1161     19817130 :       stmt_vec_info stmt_info = stmts[i];
    1162     19817130 :       bool ldst_p = false;
    1163     19817130 :       bool ldst_masklen_p = false;
    1164     19817130 :       bool phi_p = false;
    1165     19817130 :       code_helper rhs_code = ERROR_MARK;
    1166              : 
    1167     19817130 :       if (*start_i == 0)
    1168              :         {
    1169     17960904 :           swap[i] = 0;
    1170     17960904 :           matches[i] = -1;
    1171              :         }
    1172      1856226 :       else if (matches[i] != -1)
    1173     19816859 :         continue;
    1174              : 
    1175     19758099 :       if (!stmt_info)
    1176              :         {
    1177              :           /* ???  We shouldn't run into this.  */
    1178        41367 :           gcc_assert (first_match != -1);
    1179        41367 :           matches[i] = first_match;
    1180        41367 :           continue;
    1181              :         }
    1182              : 
    1183     19716732 :       gimple *stmt = stmt_info->stmt;
    1184     19716732 :       if (dump_enabled_p ())
    1185       244575 :         dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for %G", stmt);
    1186              : 
    1187              :       /* Fail to vectorize statements marked as unvectorizable, throw
    1188              :          or are volatile.  */
    1189     19716732 :       if (!STMT_VINFO_VECTORIZABLE (stmt_info)
    1190     19328531 :           || stmt_can_throw_internal (cfun, stmt)
    1191     38038536 :           || gimple_has_volatile_ops (stmt))
    1192              :         {
    1193       405548 :           if (dump_enabled_p ())
    1194          474 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1195              :                              "Build SLP failed: unvectorizable statement %G",
    1196              :                              stmt);
    1197       405548 :           if (is_a <bb_vec_info> (vinfo))
    1198              :             {
    1199       405548 :               matches[i] = -2;
    1200       405548 :               continue;
    1201              :             }
    1202      6346506 :           return false;
    1203              :         }
    1204              : 
    1205     19311184 :       gcall *call_stmt = dyn_cast <gcall *> (stmt);
    1206     19311184 :       tree lhs = gimple_get_lhs (stmt);
    1207     19311184 :       if (lhs == NULL_TREE && !call_stmt)
    1208              :         {
    1209           34 :           if (dump_enabled_p ())
    1210            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1211              :                              "Build SLP failed: not GIMPLE_ASSIGN nor "
    1212              :                              "GIMPLE_CALL %G", stmt);
    1213           34 :           if (is_a <bb_vec_info> (vinfo))
    1214              :             {
    1215           34 :               matches[i] = -2;
    1216           34 :               continue;
    1217              :             }
    1218              :           return false;
    1219              :         }
    1220              : 
    1221     19311150 :       if (call_stmt)
    1222              :         {
    1223       207594 :           combined_fn cfn = gimple_call_combined_fn (call_stmt);
    1224       207594 :           if (cfn != CFN_LAST && cfn != CFN_MASK_CALL)
    1225        65358 :             rhs_code = cfn;
    1226              :           else
    1227              :             rhs_code = CALL_EXPR;
    1228              : 
    1229       207594 :           if (cfn == CFN_GATHER_LOAD
    1230       207594 :               || cfn == CFN_SCATTER_STORE)
    1231              :             ldst_p = true;
    1232              :           else if (cfn == CFN_MASK_LOAD
    1233              :                    || cfn == CFN_MASK_GATHER_LOAD
    1234              :                    || cfn == CFN_MASK_LEN_GATHER_LOAD
    1235              :                    || cfn == CFN_MASK_SCATTER_STORE
    1236              :                    || cfn == CFN_MASK_LEN_SCATTER_STORE)
    1237              :             {
    1238              :               ldst_p = true;
    1239              :               ldst_masklen_p = true;
    1240              :             }
    1241              :           else if (cfn == CFN_MASK_STORE)
    1242              :             {
    1243              :               ldst_p = true;
    1244              :               ldst_masklen_p = true;
    1245              :               rhs_code = CFN_MASK_STORE;
    1246              :             }
    1247              :           else if (cfn == CFN_GOMP_SIMD_LANE)
    1248              :             ;
    1249       196435 :           else if ((cfn != CFN_LAST
    1250              :                     && cfn != CFN_MASK_CALL
    1251        54199 :                     && internal_fn_p (cfn)
    1252        40347 :                     && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
    1253       196324 :                    || gimple_call_tail_p (call_stmt)
    1254       196324 :                    || gimple_call_noreturn_p (call_stmt)
    1255       392759 :                    || gimple_call_chain (call_stmt))
    1256              :             {
    1257         1394 :               if (dump_enabled_p ())
    1258            8 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1259              :                                  "Build SLP failed: unsupported call type %G",
    1260              :                                  (gimple *) call_stmt);
    1261         1394 :               if (is_a <bb_vec_info> (vinfo))
    1262              :                 {
    1263         1394 :                   matches[i] = -2;
    1264         1394 :                   continue;
    1265              :                 }
    1266              :               return false;
    1267              :             }
    1268              :         }
    1269     19103556 :       else if (gimple_code (stmt) == GIMPLE_PHI)
    1270              :         {
    1271     19261690 :           rhs_code = ERROR_MARK;
    1272              :           phi_p = true;
    1273              :         }
    1274              :       else
    1275              :         {
    1276     18096829 :           rhs_code = gimple_assign_rhs_code (stmt);
    1277     18096829 :           ldst_p = STMT_VINFO_DATA_REF (stmt_info) != nullptr;
    1278              :         }
    1279              : 
    1280     18096829 :       if (!ldst_p
    1281              :           && !phi_p
    1282     13226157 :           && rhs_code.is_tree_code ()
    1283     13168415 :           && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_binary
    1284              :           && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_unary
    1285              :           && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_expression
    1286              :           && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_comparison
    1287       241049 :           && rhs_code != VIEW_CONVERT_EXPR
    1288              :           && rhs_code != CALL_EXPR
    1289              :           && rhs_code != BIT_FIELD_REF
    1290     18096829 :           && rhs_code != SSA_NAME)
    1291              :         {
    1292        48066 :           if (dump_enabled_p ())
    1293           98 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1294              :                              "Build SLP failed: operation unsupported %G",
    1295              :                              stmt);
    1296        48066 :           if (is_a <bb_vec_info> (vinfo))
    1297              :             {
    1298        47921 :               matches[i] = -2;
    1299        47921 :               continue;
    1300              :             }
    1301              :           return false;
    1302              :         }
    1303              : 
    1304              :       /* Non-grouped store or load.  */
    1305     19261690 :       if (ldst_p
    1306      5076872 :           && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
    1307       960454 :           && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
    1308       677161 :           && rhs_code != CFN_GATHER_LOAD
    1309       677161 :           && rhs_code != CFN_MASK_GATHER_LOAD
    1310       677161 :           && rhs_code != CFN_MASK_LEN_GATHER_LOAD
    1311       677161 :           && rhs_code != CFN_SCATTER_STORE
    1312       677161 :           && rhs_code != CFN_MASK_SCATTER_STORE
    1313       677161 :           && rhs_code != CFN_MASK_LEN_SCATTER_STORE
    1314       677161 :           && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
    1315              :           /* Not grouped loads are handled as externals for BB
    1316              :              vectorization.  Treat them as not vectorizable.  */
    1317     19914396 :           && is_a <bb_vec_info> (vinfo))
    1318              :         {
    1319              :           /* Not grouped load.  */
    1320            0 :           if (dump_enabled_p ())
    1321            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1322              :                              "Build SLP failed: not grouped load %G", stmt);
    1323            0 :           matches[i] = -2;
    1324            0 :           continue;
    1325              :         }
    1326              : 
    1327     19261690 :       if (!ldst_p && rhs_code == BIT_FIELD_REF)
    1328              :         {
    1329        45285 :           tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
    1330        45285 :           if (!is_a <bb_vec_info> (vinfo)
    1331        45159 :               || TREE_CODE (vec) != SSA_NAME
    1332        45135 :               || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (vec)
    1333              :               /* When the element types are not compatible we pun the
    1334              :                  source to the target vectype which requires equal size.  */
    1335        90364 :               || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
    1336        42345 :                    || !types_compatible_p (TREE_TYPE (vectype),
    1337        42345 :                                            TREE_TYPE (TREE_TYPE (vec))))
    1338         4101 :                   && !operand_equal_p (TYPE_SIZE (vectype),
    1339         4101 :                                        TYPE_SIZE (TREE_TYPE (vec)))))
    1340              :             {
    1341         2440 :               if (dump_enabled_p ())
    1342           32 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1343              :                                  "Build SLP failed: "
    1344              :                                  "BIT_FIELD_REF not supported\n");
    1345         2440 :               if (is_a <bb_vec_info> (vinfo))
    1346              :                 {
    1347         2314 :                   matches[i] = -2;
    1348         2314 :                   continue;
    1349              :                 }
    1350              :               return false;
    1351              :             }
    1352              :         }
    1353              : 
    1354              :       /* Check the operation.  */
    1355     19259250 :       if (first_match == -1)
    1356              :         {
    1357      6245045 :           first_match = i;
    1358      6245045 :           first_stmt_info = stmt_info;
    1359      6245045 :           common_bb = gimple_bb (stmt_info->stmt);
    1360      6245045 :           first_lhs = lhs;
    1361      6245045 :           first_stmt_code = rhs_code;
    1362      6245045 :           first_stmt_ldst_p = ldst_p;
    1363      6245045 :           first_stmt_ldst_masklen_p = ldst_masklen_p;
    1364      6245045 :           first_stmt_phi_p = phi_p;
    1365      6245045 :           first_reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
    1366              : 
    1367              :           /* Shift arguments should be equal in all the packed stmts for a
    1368              :              vector shift with scalar shift operand.  */
    1369      6245045 :           if (rhs_code == LSHIFT_EXPR
    1370      6144771 :               || rhs_code == RSHIFT_EXPR
    1371      6042617 :               || rhs_code == LROTATE_EXPR
    1372     12287548 :               || rhs_code == RROTATE_EXPR)
    1373              :             {
    1374              :               /* First see if we have a vector/vector shift.  */
    1375       203027 :               if (!directly_supported_p (rhs_code, vectype, optab_vector))
    1376              :                 {
    1377              :                   /* No vector/vector shift, arrange for a vector/scalar
    1378              :                      SLP layout.  */
    1379       189573 :                   need_same_oprnds = true;
    1380       189573 :                   first_op1 = gimple_assign_rhs2 (stmt);
    1381              :                 }
    1382              :             }
    1383      6042018 :           else if (rhs_code == WIDEN_LSHIFT_EXPR)
    1384              :             {
    1385            0 :               need_same_oprnds = true;
    1386            0 :               first_op1 = gimple_assign_rhs2 (stmt);
    1387              :             }
    1388      6042018 :           else if (rhs_code == CFN_DIV_POW2)
    1389              :             {
    1390            0 :               need_same_oprnds = true;
    1391            0 :               first_op1 = gimple_call_arg (call_stmt, 1);
    1392              :             }
    1393      6042018 :           else if (rhs_code == CFN_GOMP_SIMD_LANE)
    1394              :             {
    1395         3654 :               need_same_oprnds = true;
    1396         3654 :               first_op1 = gimple_call_arg (call_stmt, 1);
    1397              :             }
    1398              :         }
    1399              :       else
    1400              :         {
    1401     13014205 :           int comm_arg;
    1402     13014582 :           if (first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
    1403              :               /* For SLP reduction groups the index isn't necessarily
    1404              :                  uniform but only that of the first stmt matters.  */
    1405         2468 :               && !(first_reduc_idx != -1
    1406         2468 :                    && STMT_VINFO_REDUC_IDX (stmt_info) != -1
    1407         2468 :                    && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
    1408     13014205 :               && !(first_reduc_idx != -1
    1409         1121 :                    && STMT_VINFO_REDUC_IDX (stmt_info) != -1
    1410         1121 :                    && (comm_arg = first_commutative_argument
    1411         1121 :                                     (rhs_code, TREE_TYPE (lhs))) >= 0
    1412              :                    && (first_reduc_idx
    1413          890 :                        == 2 * comm_arg + 1 - STMT_VINFO_REDUC_IDX (stmt_info))))
    1414              :             {
    1415          377 :               if (dump_enabled_p ())
    1416              :                 {
    1417           12 :                   dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1418              :                                    "Build SLP failed: different reduc_idx "
    1419              :                                    "%d instead of %d in %G",
    1420              :                                    STMT_VINFO_REDUC_IDX (stmt_info),
    1421              :                                    first_reduc_idx, stmt);
    1422              :                 }
    1423              :               /* Mismatch.  */
    1424          377 :               continue;
    1425              :             }
    1426     13013828 :           if (!ldst_p
    1427     13013828 :               && two_operators
    1428     10379838 :               && first_stmt_code != rhs_code
    1429     14531577 :               && alt_stmt_code == ERROR_MARK)
    1430              :             alt_stmt_code = rhs_code;
    1431     14518330 :           if ((!ldst_p
    1432     10379838 :                && first_stmt_code != rhs_code
    1433      1517749 :                && (first_stmt_code != IMAGPART_EXPR
    1434          202 :                    || rhs_code != REALPART_EXPR)
    1435      1517749 :                && (first_stmt_code != REALPART_EXPR
    1436          735 :                    || rhs_code != IMAGPART_EXPR)
    1437              :                /* Handle mismatches in plus/minus by computing both
    1438              :                   and merging the results.  */
    1439      1517749 :                && !((((first_stmt_code == PLUS_EXPR
    1440      1415605 :                        || first_stmt_code == MINUS_EXPR)
    1441       126441 :                       && (alt_stmt_code == PLUS_EXPR
    1442       116496 :                           || alt_stmt_code == MINUS_EXPR))
    1443      1491089 :                      || ((first_stmt_code == CFN_FMA
    1444      1491087 :                           || first_stmt_code == CFN_FMS)
    1445            2 :                          && (alt_stmt_code == CFN_FMA
    1446            2 :                              || alt_stmt_code == CFN_FMS)))
    1447        26662 :                     && rhs_code == alt_stmt_code)
    1448      1533180 :                && !(first_stmt_code.is_tree_code ()
    1449      1401842 :                     && rhs_code.is_tree_code ()
    1450      1287085 :                     && (TREE_CODE_CLASS (tree_code (first_stmt_code))
    1451              :                         == tcc_comparison)
    1452       141564 :                     && (swap_tree_comparison (tree_code (first_stmt_code))
    1453       141564 :                         == tree_code (rhs_code))
    1454              :                     && (first_reduc_idx == -1
    1455            0 :                         || REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
    1456              :               || (ldst_p
    1457      5267980 :                   && (STMT_VINFO_GROUPED_ACCESS (stmt_info)
    1458      2633990 :                       != STMT_VINFO_GROUPED_ACCESS (first_stmt_info)))
    1459              :               || (ldst_p
    1460      2579733 :                   && (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
    1461      2579733 :                       != STMT_VINFO_GATHER_SCATTER_P (first_stmt_info)))
    1462     11509472 :               || first_stmt_ldst_p != ldst_p
    1463     11509334 :               || (ldst_p && first_stmt_ldst_masklen_p != ldst_masklen_p)
    1464     24523154 :               || first_stmt_phi_p != phi_p)
    1465              :             {
    1466      1504502 :               if (dump_enabled_p ())
    1467              :                 {
    1468         3672 :                   dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1469              :                                    "Build SLP failed: different operation "
    1470              :                                    "in stmt %G", stmt);
    1471         3672 :                   dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1472              :                                    "original stmt %G", first_stmt_info->stmt);
    1473              :                 }
    1474              :               /* Mismatch.  */
    1475      1504502 :               continue;
    1476              :             }
    1477              : 
    1478     11528509 :           if (!ldst_p
    1479      8929754 :               && first_stmt_code == BIT_FIELD_REF
    1480     11539455 :               && (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
    1481        30129 :                   != TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
    1482              :             {
    1483        19183 :               if (dump_enabled_p ())
    1484          116 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1485              :                                  "Build SLP failed: different BIT_FIELD_REF "
    1486              :                                  "arguments in %G", stmt);
    1487              :               /* Mismatch.  */
    1488        19183 :               continue;
    1489              :             }
    1490              : 
    1491     11490143 :           if (call_stmt
    1492        89121 :               && first_stmt_code != CFN_MASK_LOAD
    1493     11578862 :               && first_stmt_code != CFN_MASK_STORE)
    1494              :             {
    1495        88408 :               if (!is_a <gcall *> (stmts[0]->stmt)
    1496        88408 :                   || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt),
    1497              :                                           call_stmt, true))
    1498              :                 {
    1499        69200 :                   if (dump_enabled_p ())
    1500           13 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1501              :                                      "Build SLP failed: different calls in %G",
    1502              :                                      stmt);
    1503              :                   /* Mismatch.  */
    1504        69200 :                   continue;
    1505              :                 }
    1506              :             }
    1507              : 
    1508     11463566 :           if (phi_p
    1509     11420943 :               && (gimple_bb (first_stmt_info->stmt)
    1510       200574 :                   != gimple_bb (stmt_info->stmt)))
    1511              :             {
    1512        42623 :               if (dump_enabled_p ())
    1513            8 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1514              :                                  "Build SLP failed: different BB for PHI %G",
    1515              :                                  stmt);
    1516              :               /* Mismatch.  */
    1517        42623 :               continue;
    1518              :             }
    1519              : 
    1520     11378320 :           if (need_same_oprnds)
    1521              :             {
    1522        63773 :               tree other_op1 = gimple_arg (stmt, 1);
    1523        63773 :               if (!operand_equal_p (first_op1, other_op1, 0))
    1524              :                 {
    1525        11349 :                   if (dump_enabled_p ())
    1526          195 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1527              :                                      "Build SLP failed: different shift "
    1528              :                                      "arguments in %G", stmt);
    1529              :                   /* Mismatch.  */
    1530        11349 :                   continue;
    1531              :                 }
    1532              :             }
    1533              : 
    1534     11367713 :           if (first_lhs
    1535     11366971 :               && lhs
    1536     11366971 :               && !types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (first_lhs)))
    1537              :             {
    1538          742 :               if (dump_enabled_p ())
    1539            6 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1540              :                                  "Build SLP failed: different vector type "
    1541              :                                  "in %G", stmt);
    1542              :               /* Mismatch.  */
    1543          742 :               continue;
    1544              :             }
    1545              : 
    1546              :           /* We need to ensure all stmts are in the same BB when one stmt could
    1547              :              trap.  */
    1548     11366229 :           if (trapping_stmt || gimple_could_trap_p (stmt))
    1549              :             {
    1550       953963 :               gcc_assert (!trapping_stmt || common_bb);
    1551       953963 :               if (gimple_bb (stmt) != common_bb)
    1552              :                 {
    1553         6259 :                   if (dump_enabled_p ())
    1554           13 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1555              :                                      "Build SLP failed: not all stmts in same "
    1556              :                                      "BB but possibly trapping operation in %G",
    1557              :                                      trapping_stmt ? trapping_stmt : stmt);
    1558         6259 :                   continue;
    1559              :                 }
    1560              :             }
    1561              :         }
    1562              : 
    1563              :       /* Grouped store or load.  */
    1564     17605015 :       if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    1565              :         {
    1566      4056548 :           gcc_assert (ldst_p);
    1567      4056548 :           if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
    1568              :             {
    1569              :               /* Store.  */
    1570      3036375 :               gcc_assert (rhs_code == CFN_MASK_STORE
    1571              :                           || REFERENCE_CLASS_P (lhs)
    1572              :                           || DECL_P (lhs));
    1573              :             }
    1574              :           else
    1575              :             {
    1576              :               /* Load.  */
    1577      1020173 :               first_load = DR_GROUP_FIRST_ELEMENT (stmt_info);
    1578      1020173 :               if (prev_first_load)
    1579              :                 {
    1580              :                   /* Check that there are no loads from different interleaving
    1581              :                      chains in the same node.  */
    1582       450507 :                   if (prev_first_load != first_load)
    1583              :                     {
    1584        68110 :                       if (dump_enabled_p ())
    1585         2296 :                         dump_printf_loc (MSG_MISSED_OPTIMIZATION,
    1586              :                                          vect_location,
    1587              :                                          "Build SLP failed: different "
    1588              :                                          "interleaving chains in one node %G",
    1589              :                                          stmt);
    1590              :                       /* Mismatch.  */
    1591        68110 :                       continue;
    1592              :                     }
    1593              :                 }
    1594              :               else
    1595              :                 prev_first_load = first_load;
    1596              :            }
    1597              :         }
    1598              :       /* Non-grouped store or load.  */
    1599     13548467 :       else if (ldst_p)
    1600              :         {
    1601       960135 :           if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
    1602       676842 :               && rhs_code != CFN_GATHER_LOAD
    1603              :               && rhs_code != CFN_MASK_GATHER_LOAD
    1604              :               && rhs_code != CFN_MASK_LEN_GATHER_LOAD
    1605              :               && rhs_code != CFN_SCATTER_STORE
    1606              :               && rhs_code != CFN_MASK_SCATTER_STORE
    1607              :               && rhs_code != CFN_MASK_LEN_SCATTER_STORE
    1608      1636977 :               && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    1609              :             {
    1610       652406 :               gcc_assert (!is_a <bb_vec_info> (vinfo));
    1611              :               /* For loop vectorization we can handle splats the same we
    1612              :                  handle single element interleaving.  Likewise we can handle
    1613              :                  a collection of invariant refs.  */
    1614       727256 :               if (stmt_info != first_stmt_info
    1615       652692 :                   && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
    1616          286 :                        && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
    1617              :                                                         (first_stmt_info)))))
    1618              :                 {
    1619              :                   /* Not grouped load.  */
    1620        74850 :                   if (dump_enabled_p ())
    1621          160 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1622              :                                      "Build SLP failed: not grouped load %G",
    1623              :                                      stmt);
    1624        74850 :                   continue;
    1625              :                 }
    1626              :             }
    1627              :         }
    1628              :       /* Not memory operation.  */
    1629              :       else
    1630              :         {
    1631     12588332 :           if (rhs_code == COND_EXPR)
    1632              :             {
    1633        65751 :               tree cond_expr = gimple_assign_rhs1 (stmt);
    1634        65751 :               enum tree_code cond_code = TREE_CODE (cond_expr);
    1635        65751 :               enum tree_code swap_code = ERROR_MARK;
    1636        65751 :               enum tree_code invert_code = ERROR_MARK;
    1637              : 
    1638        65751 :               if (i == (unsigned)first_match)
    1639        55382 :                 first_cond_code = TREE_CODE (cond_expr);
    1640        10369 :               else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
    1641              :                 {
    1642            0 :                   bool honor_nans = HONOR_NANS (TREE_OPERAND (cond_expr, 0));
    1643            0 :                   swap_code = swap_tree_comparison (cond_code);
    1644            0 :                   invert_code = invert_tree_comparison (cond_code, honor_nans);
    1645              :                 }
    1646              : 
    1647        65751 :               if (first_cond_code == cond_code)
    1648              :                 ;
    1649              :               /* Isomorphic can be achieved by swapping.  */
    1650            0 :               else if (first_cond_code == swap_code)
    1651            0 :                 swap[i] = 1;
    1652              :               /* Isomorphic can be achieved by inverting.  */
    1653            0 :               else if (first_cond_code == invert_code)
    1654            0 :                 swap[i] = 2;
    1655              :               else
    1656              :                 {
    1657            0 :                   if (dump_enabled_p ())
    1658            0 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1659              :                                      "Build SLP failed: different"
    1660              :                                      " operation %G", stmt);
    1661              :                   /* Mismatch.  */
    1662            0 :                   continue;
    1663              :                 }
    1664              :             }
    1665              : 
    1666     12588332 :           if (i != (unsigned) first_match
    1667      8786169 :               && first_stmt_code != rhs_code
    1668        67663 :               && first_stmt_code.is_tree_code ()
    1669        67661 :               && rhs_code.is_tree_code ()
    1670        67661 :               && TREE_CODE_CLASS ((tree_code)first_stmt_code) == tcc_comparison
    1671     12629882 :               && (swap_tree_comparison ((tree_code)first_stmt_code)
    1672        41550 :                   == (tree_code)rhs_code))
    1673        41550 :             swap[i] = 1;
    1674              : 
    1675     12588332 :           if (i != (unsigned) first_match
    1676      8786169 :               && first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
    1677         1791 :               && first_reduc_idx != -1
    1678         1791 :               && STMT_VINFO_REDUC_IDX (stmt_info) != -1
    1679         1791 :               && rhs_code.is_tree_code ()
    1680         1787 :               && commutative_tree_code (tree_code (rhs_code))
    1681     12590117 :               && first_reduc_idx == 1 - STMT_VINFO_REDUC_IDX (stmt_info))
    1682         1785 :             swap[i] = 1;
    1683              :         }
    1684              : 
    1685              :       /* We need to ensure all stmts are in the same BB when one stmt could
    1686              :          trap.  Not matching stmts are not relevant, so exclude those.  */
    1687     17462055 :       if (!trapping_stmt && gimple_could_trap_p (stmt))
    1688              :         trapping_stmt = stmt;
    1689     17462055 :       if (common_bb != gimple_bb (stmt))
    1690              :         {
    1691        63423 :           common_bb = NULL;
    1692        63423 :           gcc_assert (!trapping_stmt);
    1693              :         }
    1694              : 
    1695     17462055 :       matches[i] = first_match;
    1696              :     }
    1697              : 
    1698              :   /* Record if we allowed two distinct operations for the SLP node.  */
    1699      6346235 :   if (((first_stmt_code == PLUS_EXPR
    1700      5829095 :         || first_stmt_code == MINUS_EXPR)
    1701       626770 :        && (alt_stmt_code == PLUS_EXPR
    1702       619468 :            || alt_stmt_code == MINUS_EXPR))
    1703     12674876 :       || ((first_stmt_code == CFN_FMA
    1704      6328592 :            || first_stmt_code == CFN_FMS)
    1705           49 :           && (alt_stmt_code == CFN_FMA
    1706           49 :               || alt_stmt_code == CFN_FMS)))
    1707        17596 :     *two_operators = true;
    1708              : 
    1709      6346235 :   if (first_match == -1)
    1710              :     {
    1711              :       /* If there was no useful stmt, fail.  Can only happen during
    1712              :          the first sweep.  */
    1713       101190 :       gcc_assert (*start_i == 0);
    1714       101190 :       *start_i = stmts.length () - 1;
    1715       101190 :       return false;
    1716              :     }
    1717              : 
    1718      6245045 :   *start_i = first_match;
    1719      6245045 :   return true;
    1720              : }
    1721              : 
    1722              : /* Verify if the scalar stmts STMTS are isomorphic, require data
    1723              :    permutation or are of unsupported types of operation.  Return
    1724              :    true if they are, otherwise return false and indicate in *MATCHES
    1725              :    the groups of isomorphic stmts.  See vect_build_slp_tree_3 for
    1726              :    details.  *TWO_OPERATORS is for the first isomorphic group,
    1727              :    knowledge whether following isomorphic groups have one or two operators
    1728              :    is not retained.  */
    1729              : 
    1730              : static bool
    1731      6252099 : vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
    1732              :                        vec<stmt_vec_info> stmts, match_elt_t *matches,
    1733              :                        bool *two_operators, tree *node_vectype)
    1734              : {
    1735      6252099 :   stmt_vec_info first_stmt_info = stmts[0];
    1736      6252099 :   unsigned int group_size = stmts.length ();
    1737      6252099 :   tree vectype;
    1738      6252099 :   if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
    1739              :                                        group_size))
    1740              :     {
    1741              :       /* Fatal mismatch.  */
    1742       257784 :       matches[0] = -1;
    1743       257784 :       return false;
    1744              :     }
    1745      5994315 :   if (is_a <bb_vec_info> (vinfo)
    1746      5994315 :       && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
    1747              :     {
    1748       375474 :       if (dump_enabled_p ())
    1749          310 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1750              :                          "Build SLP failed: not using single lane "
    1751              :                          "vector type %T\n", vectype);
    1752       375474 :       matches[0] = -1;
    1753       375474 :       return false;
    1754              :     }
    1755              :   /* Check nunits required but continue analysis, producing matches[]
    1756              :      as if nunits was not an issue.  This allows splitting of groups
    1757              :      to happen.  */
    1758      5618841 :   unsigned HOST_WIDE_INT const_nunits = 0;
    1759      5618841 :   if (vectype
    1760      5618811 :       && is_a <bb_vec_info> (vinfo)
    1761     11237652 :       && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
    1762              :     {
    1763       210390 :       if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
    1764       210390 :           || const_nunits > group_size)
    1765              :         {
    1766            0 :           if (dump_enabled_p ())
    1767            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1768              :                              "Build SLP failed: unrolling required "
    1769              :                              "in basic block SLP\n");
    1770              :           /* Fatal mismatch.  */
    1771            0 :           matches[0] = -1;
    1772            0 :           return false;
    1773              :         }
    1774              :     }
    1775              : 
    1776      5618841 :   gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
    1777      5618841 :   *node_vectype = vectype;
    1778              : 
    1779      5618841 :   unsigned start_i = 0;
    1780      5618841 :   if (!vect_build_slp_tree_3 (vinfo, stmts, matches, swap, two_operators,
    1781              :                               vectype, &start_i))
    1782              :     {
    1783              :       /* Fatal mismatch.  */
    1784       101461 :       matches[0] = -1;
    1785       101461 :       return false;
    1786              :     }
    1787      5517380 :   gcc_assert (matches[start_i] == (int)start_i);
    1788              :   /* Discover further isomorphic groups.  */
    1789     17615511 :   for (start_i = start_i + 1; start_i < group_size; ++start_i)
    1790     12098131 :     if (matches[start_i] == -1)
    1791              :       {
    1792       727665 :         unsigned prev_start_i = start_i;
    1793       727665 :         bool tem_two_operators;
    1794       727665 :         bool res = vect_build_slp_tree_3 (vinfo, stmts, matches, swap,
    1795              :                                           &tem_two_operators,
    1796              :                                           vectype, &start_i);
    1797       727665 :         gcc_assert (res && matches[start_i] == (int)prev_start_i);
    1798              :       }
    1799              : 
    1800              :   bool res = true;
    1801     23210581 :   for (unsigned i = 0; i < group_size; ++i)
    1802     17693201 :     if (matches[i] != 0)
    1803      1845102 :       res = false;
    1804              :   if (!res)
    1805              :     return false;
    1806              : 
    1807              :   return true;
    1808              : }
    1809              : 
    1810              : /* Traits for the hash_set to record failed SLP builds for a stmt set.
    1811              :    Note we never remove apart from at destruction time so we do not
    1812              :    need a special value for deleted that differs from empty.  */
    1813              : struct bst_traits
    1814              : {
    1815              :   typedef vec <stmt_vec_info> value_type;
    1816              :   typedef vec <stmt_vec_info> compare_type;
    1817              :   static inline hashval_t hash (value_type);
    1818              :   static inline bool equal (value_type existing, value_type candidate);
    1819    517810852 :   static inline bool is_empty (value_type x) { return !x.exists (); }
    1820    115822826 :   static inline bool is_deleted (value_type x) { return !x.exists (); }
    1821              :   static const bool empty_zero_p = true;
    1822            0 :   static inline void mark_empty (value_type &x) { x.release (); }
    1823              :   static inline void mark_deleted (value_type &x) { x.release (); }
    1824      9988706 :   static inline void remove (value_type &x) { x.release (); }
    1825              : };
    1826              : inline hashval_t
    1827    100959004 : bst_traits::hash (value_type x)
    1828              : {
    1829    100959004 :   inchash::hash h;
    1830    439013620 :   for (unsigned i = 0; i < x.length (); ++i)
    1831    338054616 :     h.add_int (x[i] ? gimple_uid (x[i]->stmt) : -1);
    1832    100959004 :   return h.end ();
    1833              : }
    1834              : inline bool
    1835     88305405 : bst_traits::equal (value_type existing, value_type candidate)
    1836              : {
    1837    264916215 :   if (existing.length () != candidate.length ())
    1838              :     return false;
    1839     89572227 :   for (unsigned i = 0; i < existing.length (); ++i)
    1840     84900639 :     if (existing[i] != candidate[i])
    1841              :       return false;
    1842              :   return true;
    1843              : }
    1844              : 
    1845              : typedef hash_map <vec <stmt_vec_info>, slp_tree,
    1846              :                   simple_hashmap_traits <bst_traits, slp_tree> >
    1847              :   scalar_stmts_to_slp_tree_map_t;
    1848              : 
    1849              : /* Release BST_MAP.  */
    1850              : 
    1851              : static void
    1852      1947323 : release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
    1853              : {
    1854              :   /* The map keeps a reference on SLP nodes built, release that.  */
    1855     11936029 :   for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
    1856     11936029 :        it != bst_map->end (); ++it)
    1857      9988706 :     if ((*it).second)
    1858      9988706 :       vect_free_slp_tree ((*it).second);
    1859      3894646 :   delete bst_map;
    1860      1947323 : }
    1861              : 
    1862              : /* ???  This was std::pair<std::pair<tree_code, vect_def_type>, tree>
    1863              :    but then vec::insert does memmove and that's not compatible with
    1864              :    std::pair.  */
    1865              : struct chain_op_t
    1866              : {
    1867      4391948 :   chain_op_t (tree_code code_, vect_def_type dt_, tree op_)
    1868      4391948 :       : code (code_), dt (dt_), op (op_) {}
    1869              :   tree_code code;
    1870              :   vect_def_type dt;
    1871              :   tree op;
    1872              : };
    1873              : 
    1874              : /* Comparator for sorting associatable chains.  */
    1875              : 
    1876              : static int
    1877     12883570 : dt_sort_cmp (const void *op1_, const void *op2_, void *)
    1878              : {
    1879     12883570 :   auto *op1 = (const chain_op_t *) op1_;
    1880     12883570 :   auto *op2 = (const chain_op_t *) op2_;
    1881     12883570 :   if (op1->dt != op2->dt)
    1882      2032039 :     return (int)op1->dt - (int)op2->dt;
    1883     10851531 :   return (int)op1->code - (int)op2->code;
    1884              : }
    1885              : 
    1886              : /* Linearize the associatable expression chain at START with the
    1887              :    associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
    1888              :    filling CHAIN with the result and using WORKLIST as intermediate storage.
    1889              :    CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
    1890              :    or MINUS_EXPR.  *CHAIN_STMTS if not NULL is filled with all computation
    1891              :    stmts, starting with START.  When ALLOW_ALT_CODE is false, do not
    1892              :    follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf).  */
    1893              : 
    1894              : static void
    1895      1891233 : vect_slp_linearize_chain (vec_info *vinfo,
    1896              :                           vec<std::pair<tree_code, gimple *> > &worklist,
    1897              :                           vec<chain_op_t> &chain,
    1898              :                           enum tree_code code, gimple *start,
    1899              :                           gimple *&code_stmt, gimple *&alt_code_stmt,
    1900              :                           vec<gimple *> *chain_stmts,
    1901              :                           bool allow_alt_code = true)
    1902              : {
    1903              :   /* For each lane linearize the addition/subtraction (or other
    1904              :      uniform associatable operation) expression tree.  */
    1905      1891233 :   worklist.safe_push (std::make_pair (code, start));
    1906      4391948 :   while (!worklist.is_empty ())
    1907              :     {
    1908      2500715 :       auto entry = worklist.pop ();
    1909      2500715 :       gassign *stmt = as_a <gassign *> (entry.second);
    1910      2500715 :       enum tree_code in_code = entry.first;
    1911      5001430 :       enum tree_code this_code = gimple_assign_rhs_code (stmt);
    1912              :       /* Pick some stmts suitable for SLP_TREE_REPRESENTATIVE.  */
    1913      2500715 :       if (!code_stmt
    1914      2500715 :           && gimple_assign_rhs_code (stmt) == code)
    1915      1584679 :         code_stmt = stmt;
    1916       916036 :       else if (!alt_code_stmt
    1917       916036 :                && gimple_assign_rhs_code (stmt) == MINUS_EXPR)
    1918       363297 :         alt_code_stmt = stmt;
    1919      2500715 :       if (chain_stmts)
    1920      2420293 :         chain_stmts->safe_push (stmt);
    1921      7502145 :       for (unsigned opnum = 1; opnum <= 2; ++opnum)
    1922              :         {
    1923      5001430 :           tree op = gimple_op (stmt, opnum);
    1924      5001430 :           vect_def_type dt;
    1925      5001430 :           stmt_vec_info def_stmt_info;
    1926      5001430 :           bool res = vect_is_simple_use (op, vinfo, &dt, &def_stmt_info);
    1927      5001430 :           gcc_assert (res);
    1928      5001430 :           if (dt == vect_internal_def
    1929      5001430 :               && is_pattern_stmt_p (def_stmt_info))
    1930        11045 :             op = gimple_get_lhs (def_stmt_info->stmt);
    1931      5001430 :           gimple *use_stmt;
    1932      5001430 :           use_operand_p use_p;
    1933      5001430 :           if (dt == vect_internal_def
    1934      4545693 :               && single_imm_use (op, &use_p, &use_stmt)
    1935      2898485 :               && is_gimple_assign (def_stmt_info->stmt)
    1936      7704238 :               && (gimple_assign_rhs_code (def_stmt_info->stmt) == code
    1937      2093663 :                   || (allow_alt_code
    1938        56815 :                       && code == PLUS_EXPR
    1939        36956 :                       && (gimple_assign_rhs_code (def_stmt_info->stmt)
    1940              :                           == MINUS_EXPR))))
    1941              :             {
    1942       609482 :               tree_code op_def_code = this_code;
    1943       609482 :               if (op_def_code == MINUS_EXPR && opnum == 1)
    1944        54870 :                 op_def_code = PLUS_EXPR;
    1945       609482 :               if (in_code == MINUS_EXPR)
    1946          222 :                 op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
    1947       609482 :               worklist.safe_push (std::make_pair (op_def_code,
    1948       609482 :                                                   def_stmt_info->stmt));
    1949              :             }
    1950              :           else
    1951              :             {
    1952      4391948 :               tree_code op_def_code = this_code;
    1953      4391948 :               if (op_def_code == MINUS_EXPR && opnum == 1)
    1954       308546 :                 op_def_code = PLUS_EXPR;
    1955      4391948 :               if (in_code == MINUS_EXPR)
    1956         4410 :                 op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
    1957      4391948 :               chain.safe_push (chain_op_t (op_def_code, dt, op));
    1958              :             }
    1959              :         }
    1960              :     }
    1961      1891233 : }
    1962              : 
    1963              : /* Distance from the node currently being discovered to the closest upthread
    1964              :    commutative operation whose operand-zero discovery may still be fixed by
    1965              :    retrying with swapped operands, or -1U if there is none.  */
    1966              : 
    1967              : static unsigned least_upthread_swappable_op_distance = -1U;
    1968              : 
    1969              : static slp_tree
    1970              : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
    1971              :                        vec<stmt_vec_info> stmts,
    1972              :                        match_elt_t *matches, unsigned *limit,
    1973              :                        unsigned *tree_size,
    1974              :                        scalar_stmts_to_slp_tree_map_t *bst_map);
    1975              : 
    1976              : static slp_tree
    1977      6766436 : vect_build_slp_tree (vec_info *vinfo,
    1978              :                      vec<stmt_vec_info> stmts,
    1979              :                      match_elt_t *matches, unsigned *limit, unsigned *tree_size,
    1980              :                      scalar_stmts_to_slp_tree_map_t *bst_map)
    1981              : {
    1982      6766436 :   unsigned int group_size = stmts.length ();
    1983      6766436 :   if (slp_tree *leader = bst_map->get (stmts))
    1984              :     {
    1985       508524 :       if (dump_enabled_p ())
    1986        18853 :         dump_printf_loc (MSG_NOTE, vect_location, "re-using %sSLP tree %p\n",
    1987        18853 :                          !(*leader)->failed ? "" : "failed ",
    1988              :                          (void *) *leader);
    1989       508524 :       if (!(*leader)->failed)
    1990              :         {
    1991       459853 :           SLP_TREE_REF_COUNT (*leader)++;
    1992       459853 :           stmts.release ();
    1993       459853 :           return *leader;
    1994              :         }
    1995        48671 :       memcpy (matches, (*leader)->failed, sizeof (match_elt_t) * group_size);
    1996        48671 :       return NULL;
    1997              :     }
    1998              : 
    1999              :   /* Single-lane SLP doesn't have the chance of run-away, do not account
    2000              :      it to the limit.  */
    2001      6257912 :   if (stmts.length () > 1)
    2002              :     {
    2003      3450749 :       if (*limit == 0)
    2004              :         {
    2005         1355 :           if (dump_enabled_p ())
    2006           15 :             dump_printf_loc (MSG_NOTE, vect_location,
    2007              :                              "SLP discovery limit exceeded\n");
    2008         1355 :           memset (matches, -1, sizeof (match_elt_t) * group_size);
    2009         1355 :           return NULL;
    2010              :         }
    2011      3449394 :       --*limit;
    2012              :     }
    2013              : 
    2014              :   /* Seed the bst_map with a stub node to be filled by vect_build_slp_tree_2
    2015              :      so we can pick up backedge destinations during discovery.  */
    2016      6256557 :   slp_tree res = new _slp_tree;
    2017      6256557 :   SLP_TREE_DEF_TYPE (res) = vect_internal_def;
    2018      6256557 :   SLP_TREE_SCALAR_STMTS (res) = stmts;
    2019      6256557 :   bst_map->put (stmts.copy (), res);
    2020              : 
    2021      6256557 :   if (dump_enabled_p ())
    2022       160196 :     dump_printf_loc (MSG_NOTE, vect_location,
    2023              :                      "starting SLP discovery for node %p\n", (void *) res);
    2024              : 
    2025      6256557 :   slp_tree res_ = vect_build_slp_tree_2 (vinfo, res, stmts,
    2026              :                                         matches, limit, tree_size, bst_map);
    2027      6256557 :   if (!res_)
    2028              :     {
    2029      2188564 :       if (dump_enabled_p ())
    2030         8267 :         dump_printf_loc (MSG_NOTE, vect_location,
    2031              :                          "SLP discovery for node %p failed\n", (void *) res);
    2032              :       /* Mark the node invalid so we can detect those when still in use
    2033              :          as backedge destinations.  */
    2034      2188564 :       SLP_TREE_SCALAR_STMTS (res) = vNULL;
    2035      2188564 :       SLP_TREE_DEF_TYPE (res) = vect_uninitialized_def;
    2036      2188564 :       res->failed = XNEWVEC (match_elt_t, group_size);
    2037      2188564 :       if (flag_checking)
    2038              :         {
    2039              :           unsigned i;
    2040      3821239 :           for (i = 0; i < group_size; ++i)
    2041      3821239 :             if (matches[i] != 0)
    2042              :               break;
    2043      2188564 :           gcc_assert (i < group_size);
    2044              :         }
    2045      2188564 :       memcpy (res->failed, matches, sizeof (match_elt_t) * group_size);
    2046              :     }
    2047              :   else
    2048              :     {
    2049      4067993 :       if (dump_enabled_p ())
    2050       151929 :         dump_printf_loc (MSG_NOTE, vect_location,
    2051              :                          "SLP discovery for node %p succeeded\n",
    2052              :                          (void *) res);
    2053      4067993 :       gcc_assert (res_ == res);
    2054              :       /* Keep a reference for the bst_map use.  */
    2055      4067993 :       SLP_TREE_REF_COUNT (res)++;
    2056              :       /* For BB vectorization nodes that need splitting should be failed.  */
    2057      4067993 :       gcc_assert (!is_a <bb_vec_info> (vinfo)
    2058              :                   || multiple_p (group_size,
    2059              :                                  TYPE_VECTOR_SUBPARTS (res->vectype)));
    2060              :     }
    2061              :   return res_;
    2062              : }
    2063              : 
    2064              : /* Helper for building an associated SLP node chain.  */
    2065              : 
    2066              : static void
    2067          162 : vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype,
    2068              :                                    slp_tree op0, slp_tree op1,
    2069              :                                    stmt_vec_info oper1, stmt_vec_info oper2,
    2070              :                                    vec<std::pair<unsigned, unsigned> > lperm)
    2071              : {
    2072          162 :   unsigned group_size = SLP_TREE_LANES (op1);
    2073              : 
    2074          162 :   slp_tree child1 = new _slp_tree;
    2075          162 :   SLP_TREE_DEF_TYPE (child1) = vect_internal_def;
    2076          162 :   SLP_TREE_VECTYPE (child1) = vectype;
    2077          162 :   SLP_TREE_LANES (child1) = group_size;
    2078          162 :   SLP_TREE_CHILDREN (child1).create (2);
    2079          162 :   SLP_TREE_CHILDREN (child1).quick_push (op0);
    2080          162 :   SLP_TREE_CHILDREN (child1).quick_push (op1);
    2081          162 :   SLP_TREE_REPRESENTATIVE (child1) = oper1;
    2082              : 
    2083          162 :   slp_tree child2 = new _slp_tree;
    2084          162 :   SLP_TREE_DEF_TYPE (child2) = vect_internal_def;
    2085          162 :   SLP_TREE_VECTYPE (child2) = vectype;
    2086          162 :   SLP_TREE_LANES (child2) = group_size;
    2087          162 :   SLP_TREE_CHILDREN (child2).create (2);
    2088          162 :   SLP_TREE_CHILDREN (child2).quick_push (op0);
    2089          162 :   SLP_TREE_REF_COUNT (op0)++;
    2090          162 :   SLP_TREE_CHILDREN (child2).quick_push (op1);
    2091          162 :   SLP_TREE_REF_COUNT (op1)++;
    2092          162 :   SLP_TREE_REPRESENTATIVE (child2) = oper2;
    2093              : 
    2094          162 :   SLP_TREE_DEF_TYPE (perm) = vect_internal_def;
    2095          162 :   SLP_TREE_CODE (perm) = VEC_PERM_EXPR;
    2096          162 :   SLP_TREE_VECTYPE (perm) = vectype;
    2097          162 :   SLP_TREE_LANES (perm) = group_size;
    2098          162 :   SLP_TREE_REPRESENTATIVE (perm) = NULL;
    2099          162 :   SLP_TREE_LANE_PERMUTATION (perm) = lperm;
    2100          162 :   SLP_TREE_CHILDREN (perm).quick_push (child1);
    2101          162 :   SLP_TREE_CHILDREN (perm).quick_push (child2);
    2102          162 : }
    2103              : 
    2104              : /* For isomorphic matches[], indicate a splitting point according to
    2105              :    CONST_NUNITS.  */
    2106              : 
    2107              : static void
    2108       123354 : force_split_matches (match_elt_t *matches, unsigned group_size,
    2109              :                      unsigned const_nunits)
    2110              : {
    2111       123354 :   if (dump_enabled_p ())
    2112           54 :     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2113              :                      "Build SLP failed: unrolling required "
    2114              :                      "in basic block SLP\n");
    2115              : 
    2116              :   /* With constant vector elements simulate a mismatch at the
    2117              :      point we need to split.  But indicate the tail is isomorphic.  */
    2118       123354 :   unsigned tail = group_size & (const_nunits - 1);
    2119       271730 :   for (unsigned i = group_size - tail; i < group_size; ++i)
    2120       148376 :     matches[i] = (int)(group_size - tail);
    2121       123354 : }
    2122              : 
    2123              : /* Recursively build an SLP tree starting from NODE.
    2124              :    Fail (and return a value not equal to zero) if def-stmts are not
    2125              :    isomorphic, require data permutation or are of unsupported types of
    2126              :    operation.  Otherwise, return 0.
    2127              :    The value returned is the depth in the SLP tree where a mismatch
    2128              :    was found.  */
    2129              : 
    2130              : static slp_tree
    2131      6256557 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
    2132              :                        vec<stmt_vec_info> stmts,
    2133              :                        match_elt_t *matches, unsigned *limit,
    2134              :                        unsigned *tree_size,
    2135              :                        scalar_stmts_to_slp_tree_map_t *bst_map)
    2136              : {
    2137      6256557 :   unsigned int group_size = stmts.length ();
    2138      6256557 :   unsigned nops, i, this_tree_size = 0;
    2139              : 
    2140      6256557 :   matches[0] = -1;
    2141              : 
    2142      6256557 :   stmt_vec_info stmt_info = stmts[0];
    2143      6256557 :   if (!is_a<gcall *> (stmt_info->stmt)
    2144              :       && !is_a<gassign *> (stmt_info->stmt)
    2145              :       && !is_a<gphi *> (stmt_info->stmt))
    2146              :     return NULL;
    2147              : 
    2148      6256387 :   nops = gimple_num_args (stmt_info->stmt);
    2149      6256387 :   if (const int *map = vect_get_operand_map (stmt_info))
    2150        40466 :     nops = map[0];
    2151              : 
    2152              :   /* If the SLP node is a PHI (induction or reduction), terminate
    2153              :      the recursion.  */
    2154      6256387 :   bool *skip_args = XALLOCAVEC (bool, nops);
    2155      6256387 :   memset (skip_args, 0, sizeof (bool) * nops);
    2156      6256387 :   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
    2157      2988535 :     if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
    2158              :       {
    2159       325537 :         tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
    2160       325537 :         tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
    2161              :                                                     group_size);
    2162       325537 :         if (!vectype)
    2163              :           {
    2164         4282 :             if (dump_enabled_p ())
    2165            7 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2166              :                                "Build SLP failed: unsupported data-type in %G\n",
    2167              :                                stmt_info->stmt);
    2168              :             return NULL;
    2169              :           }
    2170              : 
    2171       321255 :         vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
    2172       321255 :         if (def_type == vect_induction_def)
    2173              :           {
    2174              :             /* Induction PHIs are not cycles but walk the initial
    2175              :                value.  Only for inner loops through, for outer loops
    2176              :                we need to pick up the value from the actual PHIs
    2177              :                to more easily support peeling and epilogue vectorization.  */
    2178       214606 :             class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    2179       214606 :             if (!nested_in_vect_loop_p (loop, stmt_info))
    2180       213764 :               skip_args[loop_preheader_edge (loop)->dest_idx] = true;
    2181              :             else
    2182              :               loop = loop->inner;
    2183       214606 :             skip_args[loop_latch_edge (loop)->dest_idx] = true;
    2184              :           }
    2185       106649 :         else if (def_type == vect_reduction_def
    2186              :                  || def_type == vect_double_reduction_def
    2187              :                  || def_type == vect_nested_cycle
    2188       106649 :                  || def_type == vect_first_order_recurrence)
    2189              :           {
    2190              :             /* Else def types have to match.  */
    2191              :             stmt_vec_info other_info;
    2192              :             bool all_same = true;
    2193       228896 :             FOR_EACH_VEC_ELT (stmts, i, other_info)
    2194              :               {
    2195       123557 :                 if (STMT_VINFO_DEF_TYPE (other_info) != def_type)
    2196      6256557 :                   return NULL;
    2197       123551 :                 if (other_info != stmt_info)
    2198        15110 :                   all_same = false;
    2199              :               }
    2200       105339 :             class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    2201              :             /* Reduction initial values are not explicitly represented.  */
    2202       105339 :             if (def_type != vect_first_order_recurrence
    2203       105339 :                 && gimple_bb (stmt_info->stmt) == loop->header)
    2204       102095 :               skip_args[loop_preheader_edge (loop)->dest_idx] = true;
    2205              :             /* Reduction chain backedge defs are filled manually.
    2206              :                ???  Need a better way to identify a SLP reduction chain PHI.
    2207              :                Or a better overall way to SLP match those.  */
    2208       105339 :             if (stmts.length () > 1
    2209       105339 :                 && all_same && def_type == vect_reduction_def)
    2210          660 :               skip_args[loop_latch_edge (loop)->dest_idx] = true;
    2211              :           }
    2212         1304 :         else if (def_type != vect_internal_def)
    2213              :           return NULL;
    2214              :       }
    2215              : 
    2216              : 
    2217      6252099 :   bool two_operators = false;
    2218      6252099 :   unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
    2219      6252099 :   tree vectype = NULL_TREE;
    2220      6252099 :   if (!vect_build_slp_tree_1 (vinfo, swap, stmts, matches, &two_operators,
    2221              :                               &vectype))
    2222              :     return NULL;
    2223              : 
    2224      4687785 :   bool soft_fail
    2225      4687785 :     = (is_a <bb_vec_info> (vinfo)
    2226      4687785 :        && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)));
    2227              : 
    2228              :   /* If the SLP node is a load, terminate the recursion unless masked.  */
    2229      4687785 :   if (STMT_VINFO_DATA_REF (stmt_info)
    2230      2266668 :       && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
    2231              :     {
    2232       972208 :       if (soft_fail)
    2233              :         {
    2234          979 :           force_split_matches (matches, group_size,
    2235          979 :                                TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
    2236          979 :           return NULL;
    2237              :         }
    2238       971229 :       if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    2239              :         gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
    2240              :       else
    2241              :         {
    2242       947993 :           (*tree_size)++;
    2243       947993 :           node = vect_create_new_slp_node (node, stmts, 0);
    2244       947993 :           SLP_TREE_VECTYPE (node) = vectype;
    2245              :           /* And compute the load permutation.  Whether it is actually
    2246              :              a permutation depends on the unrolling factor which is
    2247              :              decided later.  */
    2248       947993 :           vec<unsigned> load_permutation;
    2249       947993 :           int j;
    2250       947993 :           stmt_vec_info load_info;
    2251       947993 :           load_permutation.create (group_size);
    2252       947993 :           stmt_vec_info first_stmt_info
    2253       947993 :             = STMT_VINFO_GROUPED_ACCESS (stmt_info)
    2254       947993 :               ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
    2255       947993 :           bool any_permute = false;
    2256      2285873 :           FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
    2257              :             {
    2258      1337880 :               int load_place;
    2259      1337880 :               if (! load_info)
    2260              :                 {
    2261        41127 :                   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2262              :                     load_place = j;
    2263              :                   else
    2264              :                     load_place = 0;
    2265              :                 }
    2266      1296753 :               else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2267       754762 :                 load_place = vect_get_place_in_interleaving_chain
    2268       754762 :                     (load_info, first_stmt_info);
    2269              :               else
    2270              :                 /* Recognize the splat case as { 0, 0, ... } but make
    2271              :                    sure to use the appropriate refs for collections
    2272              :                    of invariant refs.  */
    2273       541991 :                 load_place = (load_info == stmt_info) ? 0 : j;
    2274       796166 :               gcc_assert (load_place != -1);
    2275      1337880 :               any_permute |= load_place != j;
    2276      1337880 :               load_permutation.quick_push (load_place);
    2277              :             }
    2278              : 
    2279       947993 :           if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
    2280              :             {
    2281         3377 :               gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
    2282         3377 :               bool has_gaps = false;
    2283         3377 :               if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2284          189 :                 for (stmt_vec_info si = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
    2285          846 :                      si; si = DR_GROUP_NEXT_ELEMENT (si))
    2286          657 :                   if (DR_GROUP_GAP (si) != 1)
    2287           80 :                     has_gaps = true;
    2288              :               /* We cannot handle permuted masked loads directly, see
    2289              :                  PR114375.  We cannot handle strided masked loads or masked
    2290              :                  loads with gaps unless the mask is uniform.  */
    2291         3377 :               if ((STMT_VINFO_GROUPED_ACCESS (stmt_info)
    2292          189 :                    && (DR_GROUP_GAP (first_stmt_info) != 0
    2293          129 :                        || (has_gaps
    2294           35 :                            && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info))))
    2295         6674 :                   || STMT_VINFO_STRIDED_P (stmt_info))
    2296              :                 {
    2297          108 :                   load_permutation.release ();
    2298          108 :                   matches[0] = -1;
    2299       944763 :                   return NULL;
    2300              :                 }
    2301              : 
    2302              :               /* For permuted masked loads do an unpermuted masked load of
    2303              :                  the whole group followed by a SLP permute node.  */
    2304         3269 :               if (any_permute
    2305         3269 :                   || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
    2306           83 :                       && DR_GROUP_SIZE (first_stmt_info) != group_size))
    2307              :                 {
    2308              :                   /* Discover the whole unpermuted load.  */
    2309           39 :                   vec<stmt_vec_info> stmts2;
    2310           39 :                   unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
    2311           68 :                       ? DR_GROUP_SIZE (first_stmt_info) : 1;
    2312           39 :                   stmts2.create (dr_group_size);
    2313           39 :                   stmts2.quick_grow_cleared (dr_group_size);
    2314           39 :                   unsigned i = 0;
    2315           39 :                   for (stmt_vec_info si = first_stmt_info;
    2316          464 :                        si; si = DR_GROUP_NEXT_ELEMENT (si))
    2317              :                     {
    2318          425 :                       if (si != first_stmt_info)
    2319         1586 :                         for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
    2320         1200 :                           stmts2[i++] = NULL;
    2321          425 :                       stmts2[i++] = si;
    2322              :                     }
    2323           39 :                   match_elt_t *matches2
    2324           39 :                     = XALLOCAVEC (match_elt_t, dr_group_size);
    2325           39 :                   slp_tree unperm_load
    2326           39 :                     = vect_build_slp_tree (vinfo, stmts2, matches2, limit,
    2327           39 :                                            &this_tree_size, bst_map);
    2328              :                   /* When we are able to do the full masked load emit that
    2329              :                      followed by 'node' being the desired final permutation.  */
    2330           39 :                   if (unperm_load)
    2331              :                     {
    2332           16 :                       gcc_assert
    2333              :                         (!SLP_TREE_LOAD_PERMUTATION (unperm_load).exists ());
    2334           16 :                       lane_permutation_t lperm;
    2335           16 :                       lperm.create (group_size);
    2336           72 :                       for (unsigned j = 0; j < load_permutation.length (); ++j)
    2337           40 :                         lperm.quick_push
    2338           40 :                           (std::make_pair (0, load_permutation[j]));
    2339           16 :                       SLP_TREE_CODE (node) = VEC_PERM_EXPR;
    2340           16 :                       SLP_TREE_CHILDREN (node).safe_push (unperm_load);
    2341           16 :                       SLP_TREE_LANE_PERMUTATION (node) = lperm;
    2342           16 :                       SLP_TREE_REPRESENTATIVE (node) = NULL;
    2343           16 :                       load_permutation.release ();
    2344           16 :                       return node;
    2345              :                     }
    2346           23 :                   stmts2.release ();
    2347           23 :                   load_permutation.release ();
    2348           23 :                   matches[0] = -1;
    2349           23 :                   return NULL;
    2350              :                 }
    2351         3230 :               load_permutation.release ();
    2352              :             }
    2353              :           else
    2354              :             {
    2355       944616 :               if (!any_permute
    2356       816582 :                   && STMT_VINFO_GROUPED_ACCESS (stmt_info)
    2357      1238756 :                   && group_size == DR_GROUP_SIZE (first_stmt_info))
    2358       129616 :                 load_permutation.release ();
    2359       944616 :               SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
    2360       944616 :               return node;
    2361              :             }
    2362              :         }
    2363              :     }
    2364      3715577 :   else if (gimple_assign_single_p (stmt_info->stmt)
    2365      2596008 :            && !gimple_vuse (stmt_info->stmt)
    2366      3724686 :            && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
    2367              :     {
    2368         3427 :       if (soft_fail)
    2369              :         {
    2370           22 :           force_split_matches (matches, group_size,
    2371           22 :                                TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
    2372           22 :           return NULL;
    2373              :         }
    2374              :       /* vect_build_slp_tree_2 determined all BIT_FIELD_REFs reference
    2375              :          the same SSA name vector of a compatible type to vectype.  */
    2376         3405 :       vec<std::pair<unsigned, unsigned> > lperm = vNULL;
    2377         3405 :       tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0);
    2378         3405 :       stmt_vec_info estmt_info;
    2379        11867 :       FOR_EACH_VEC_ELT (stmts, i, estmt_info)
    2380              :         {
    2381         8609 :           gassign *estmt = as_a <gassign *> (estmt_info->stmt);
    2382         8609 :           tree bfref = gimple_assign_rhs1 (estmt);
    2383         8609 :           HOST_WIDE_INT lane;
    2384         8609 :           if (!known_eq (bit_field_size (bfref),
    2385              :                          tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (vectype))))
    2386        17071 :               || !constant_multiple_p (bit_field_offset (bfref),
    2387         8609 :                                        bit_field_size (bfref), &lane))
    2388              :             {
    2389          147 :               lperm.release ();
    2390          147 :               matches[0] = -1;
    2391          147 :               return NULL;
    2392              :             }
    2393         8462 :           lperm.safe_push (std::make_pair (0, (unsigned)lane));
    2394              :         }
    2395         3258 :       slp_tree vnode = vect_create_new_slp_node (vNULL);
    2396         3258 :       if (operand_equal_p (TYPE_SIZE (vectype), TYPE_SIZE (TREE_TYPE (vec))))
    2397              :         /* ???  We record vectype here but we hide eventually necessary
    2398              :            punning and instead rely on code generation to materialize
    2399              :            VIEW_CONVERT_EXPRs as necessary.  We instead should make
    2400              :            this explicit somehow.  */
    2401         1456 :         SLP_TREE_VECTYPE (vnode) = vectype;
    2402              :       else
    2403              :         {
    2404              :           /* For different size but compatible elements we can still
    2405              :              use VEC_PERM_EXPR without punning.  */
    2406         1802 :           gcc_assert (VECTOR_TYPE_P (TREE_TYPE (vec))
    2407              :                       && types_compatible_p (TREE_TYPE (vectype),
    2408              :                                              TREE_TYPE (TREE_TYPE (vec))));
    2409         1802 :           SLP_TREE_VECTYPE (vnode) = TREE_TYPE (vec);
    2410              :         }
    2411         3258 :       auto nunits = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (vnode));
    2412         3258 :       unsigned HOST_WIDE_INT const_nunits;
    2413         3258 :       if (nunits.is_constant (&const_nunits))
    2414         3258 :         SLP_TREE_LANES (vnode) = const_nunits;
    2415         3258 :       SLP_TREE_VEC_DEFS (vnode).safe_push (vec);
    2416              :       /* We are always building a permutation node even if it is an identity
    2417              :          permute to shield the rest of the vectorizer from the odd node
    2418              :          representing an actual vector without any scalar ops.
    2419              :          ???  We could hide it completely with making the permute node
    2420              :          external?  */
    2421         3258 :       node = vect_create_new_slp_node (node, stmts, 1);
    2422         3258 :       SLP_TREE_CODE (node) = VEC_PERM_EXPR;
    2423         3258 :       SLP_TREE_LANE_PERMUTATION (node) = lperm;
    2424         3258 :       SLP_TREE_VECTYPE (node) = vectype;
    2425         3258 :       SLP_TREE_CHILDREN (node).quick_push (vnode);
    2426         3258 :       SLP_TREE_REPRESENTATIVE (node) = NULL;
    2427         3258 :       return node;
    2428              :     }
    2429              :   /* When discovery reaches an associatable operation see whether we can
    2430              :      improve that to match up lanes in a way superior to the operand
    2431              :      swapping code which at most looks at two defs.
    2432              :      ???  For BB vectorization we cannot do the brute-force search
    2433              :      for matching as we can succeed by means of builds from scalars
    2434              :      and have no good way to "cost" one build against another.  */
    2435      3712150 :   else if (is_a <loop_vec_info> (vinfo)
    2436              :            /* Do not bother for single-lane SLP.  */
    2437      2138242 :            && group_size > 1
    2438              :            /* ???  We don't handle !vect_internal_def defs below.  */
    2439       108683 :            && STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
    2440              :            /* ???  Do not associate a reduction, this will wreck REDUC_IDX
    2441              :               mapping as long as that exists on the stmt_info level.  */
    2442        89452 :            && STMT_VINFO_REDUC_IDX (stmt_info) == -1
    2443        82617 :            && is_gimple_assign (stmt_info->stmt)
    2444        82290 :            && (associative_tree_code (gimple_assign_rhs_code (stmt_info->stmt))
    2445        56721 :                || gimple_assign_rhs_code (stmt_info->stmt) == MINUS_EXPR)
    2446      3739682 :            && ((FLOAT_TYPE_P (vectype) && flag_associative_math)
    2447        16121 :                || (INTEGRAL_TYPE_P (TREE_TYPE (vectype))
    2448        13582 :                    && TYPE_OVERFLOW_WRAPS (TREE_TYPE (vectype)))))
    2449              :     {
    2450              :       /* See if we have a chain of (mixed) adds or subtracts or other
    2451              :          associatable ops.  */
    2452        20664 :       enum tree_code code = gimple_assign_rhs_code (stmt_info->stmt);
    2453        20664 :       if (code == MINUS_EXPR)
    2454          845 :         code = PLUS_EXPR;
    2455        20664 :       stmt_vec_info other_op_stmt_info = NULL;
    2456        20664 :       stmt_vec_info op_stmt_info = NULL;
    2457        20664 :       unsigned chain_len = 0;
    2458        20664 :       auto_vec<chain_op_t> chain;
    2459        20664 :       auto_vec<std::pair<tree_code, gimple *> > worklist;
    2460        20664 :       auto_vec<vec<chain_op_t> > chains (group_size);
    2461        20664 :       auto_vec<slp_tree, 4> children;
    2462        20664 :       bool hard_fail = true;
    2463        21667 :       for (unsigned lane = 0; lane < group_size; ++lane)
    2464              :         {
    2465        21336 :           if (!stmts[lane])
    2466              :             {
    2467              :               /* ???  Below we require lane zero is present.  */
    2468            0 :               if (lane == 0)
    2469              :                 {
    2470              :                   hard_fail = false;
    2471        20333 :                   break;
    2472              :                 }
    2473            0 :               chains.quick_push (vNULL);
    2474            0 :               continue;
    2475              :             }
    2476              :           /* For each lane linearize the addition/subtraction (or other
    2477              :              uniform associatable operation) expression tree.  */
    2478        21336 :           gimple *op_stmt = NULL, *other_op_stmt = NULL;
    2479        21336 :           vect_slp_linearize_chain (vinfo, worklist, chain, code,
    2480        21336 :                                     stmts[lane]->stmt, op_stmt, other_op_stmt,
    2481              :                                     NULL);
    2482        21336 :           if (!op_stmt_info && op_stmt)
    2483        20010 :             op_stmt_info = vinfo->lookup_stmt (op_stmt);
    2484        21336 :           if (!other_op_stmt_info && other_op_stmt)
    2485          886 :             other_op_stmt_info = vinfo->lookup_stmt (other_op_stmt);
    2486        21336 :           if (chain.length () == 2)
    2487              :             {
    2488              :               /* In a chain of just two elements resort to the regular
    2489              :                  operand swapping scheme.  Likewise if we run into a
    2490              :                  length mismatch process regularly as well as we did not
    2491              :                  process the other lanes we cannot report a good hint what
    2492              :                  lanes to try swapping in the parent.  */
    2493              :               hard_fail = false;
    2494              :               break;
    2495              :             }
    2496         1006 :           else if (chain_len == 0)
    2497          371 :             chain_len = chain.length ();
    2498         1270 :           else if (chain.length () != chain_len)
    2499              :             {
    2500              :               /* ???  Here we could slip in magic to compensate with
    2501              :                  neutral operands.  */
    2502            3 :               matches[lane] = -1;
    2503            3 :               if (lane != group_size - 1)
    2504            3 :                 matches[0] = -1;
    2505              :               break;
    2506              :             }
    2507         1003 :           chains.quick_push (chain.copy ());
    2508         1003 :           chain.truncate (0);
    2509              :         }
    2510        41328 :       if (chains.length () == group_size)
    2511              :         {
    2512              :           /* We cannot yet use SLP_TREE_CODE to communicate the operation.  */
    2513          331 :           if (!op_stmt_info)
    2514              :             {
    2515            3 :               hard_fail = false;
    2516            3 :               goto out;
    2517              :             }
    2518              :           /* Now we have a set of chains with the same length.  */
    2519              :           /* 1. pre-sort according to def_type and operation.  */
    2520         1219 :           for (unsigned lane = 0; lane < group_size; ++lane)
    2521         1782 :             chains[lane].stablesort (dt_sort_cmp, vinfo);
    2522          328 :           if (dump_enabled_p ())
    2523              :             {
    2524          155 :               dump_printf_loc (MSG_NOTE, vect_location,
    2525              :                                "pre-sorted chains of %s\n",
    2526              :                                get_tree_code_name (code));
    2527          655 :               for (unsigned lane = 0; lane < group_size; ++lane)
    2528              :                 {
    2529          500 :                   if (!stmts[lane])
    2530            0 :                     dump_printf (MSG_NOTE, "--");
    2531              :                   else
    2532         2250 :                     for (unsigned opnum = 0; opnum < chain_len; ++opnum)
    2533         3500 :                       dump_printf (MSG_NOTE, "%s %T ",
    2534         1750 :                                    get_tree_code_name (chains[lane][opnum].code),
    2535         1750 :                                    chains[lane][opnum].op);
    2536          500 :                   dump_printf (MSG_NOTE, "\n");
    2537              :                 }
    2538              :             }
    2539              :           /* 2. try to build children nodes, associating as necessary.  */
    2540              :           /* 2a. prepare and perform early checks to avoid eating into
    2541              :              discovery limit unnecessarily.  */
    2542          328 :           vect_def_type *dts = XALLOCAVEC (vect_def_type, chain_len);
    2543         1383 :           for (unsigned n = 0; n < chain_len; ++n)
    2544              :             {
    2545         1055 :               vect_def_type dt = chains[0][n].dt;
    2546         1055 :               unsigned lane;
    2547         4034 :               for (lane = 0; lane < group_size; ++lane)
    2548         5958 :                 if (stmts[lane] && chains[lane][n].dt != dt)
    2549              :                   {
    2550            0 :                     if (dt == vect_constant_def
    2551            0 :                         && chains[lane][n].dt == vect_external_def)
    2552              :                       dt = vect_external_def;
    2553            0 :                     else if (dt == vect_external_def
    2554            0 :                              && chains[lane][n].dt == vect_constant_def)
    2555              :                       ;
    2556              :                     else
    2557              :                       break;
    2558              :                   }
    2559         1055 :               if (lane != group_size)
    2560              :                 {
    2561            0 :                   if (dump_enabled_p ())
    2562            0 :                     dump_printf_loc (MSG_NOTE, vect_location,
    2563              :                                      "giving up on chain due to mismatched "
    2564              :                                      "def types\n");
    2565            0 :                   matches[lane] = -1;
    2566            0 :                   if (lane != group_size - 1)
    2567            0 :                     matches[0] = -1;
    2568            0 :                   goto out;
    2569              :                 }
    2570         1055 :               dts[n] = dt;
    2571         1055 :               if (dt == vect_constant_def
    2572         1055 :                   || dt == vect_external_def)
    2573              :                 {
    2574              :                   /* Check whether we can build the invariant.  If we can't
    2575              :                      we never will be able to.  */
    2576           77 :                   tree type = TREE_TYPE (chains[0][n].op);
    2577         1055 :                   if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
    2578              :                       && (TREE_CODE (type) == BOOLEAN_TYPE
    2579              :                           || !can_duplicate_and_interleave_p (vinfo, group_size,
    2580              :                                                               type)))
    2581              :                     {
    2582              :                       matches[0] = -1;
    2583              :                       goto out;
    2584              :                     }
    2585              :                 }
    2586          978 :               else if (dt != vect_internal_def)
    2587              :                 {
    2588              :                   /* Not sure, we might need sth special.
    2589              :                      gcc.dg/vect/pr96854.c,
    2590              :                      gfortran.dg/vect/fast-math-pr37021.f90
    2591              :                      and gfortran.dg/vect/pr61171.f trigger.  */
    2592              :                   /* Soft-fail for now.  */
    2593            0 :                   hard_fail = false;
    2594            0 :                   goto out;
    2595              :                 }
    2596              :             }
    2597              :           /* 2b. do the actual build.  */
    2598         1328 :           for (unsigned n = 0; n < chain_len; ++n)
    2599              :             {
    2600         1020 :               vect_def_type dt = dts[n];
    2601         1020 :               unsigned lane;
    2602         1020 :               if (dt == vect_constant_def
    2603         1020 :                   || dt == vect_external_def)
    2604              :                 {
    2605           77 :                   vec<tree> ops;
    2606           77 :                   ops.create (group_size);
    2607          397 :                   for (lane = 0; lane < group_size; ++lane)
    2608          243 :                     if (stmts[lane])
    2609          243 :                       ops.quick_push (chains[lane][n].op);
    2610              :                     else
    2611            0 :                       ops.quick_push (NULL_TREE);
    2612           77 :                   slp_tree child = vect_create_new_slp_node (ops);
    2613           77 :                   SLP_TREE_DEF_TYPE (child) = dt;
    2614           77 :                   children.safe_push (child);
    2615              :                 }
    2616              :               else
    2617              :                 {
    2618          943 :                   vec<stmt_vec_info> op_stmts;
    2619          943 :                   op_stmts.create (group_size);
    2620          943 :                   slp_tree child = NULL;
    2621              :                   /* Brute-force our way.  We have to consider a lane
    2622              :                      failing after fixing an earlier fail up in the
    2623              :                      SLP discovery recursion.  So track the current
    2624              :                      permute per lane.  */
    2625          943 :                   unsigned *perms = XALLOCAVEC (unsigned, group_size);
    2626          943 :                   memset (perms, 0, sizeof (unsigned) * group_size);
    2627         1039 :                   do
    2628              :                     {
    2629         1039 :                       op_stmts.truncate (0);
    2630         5002 :                       for (lane = 0; lane < group_size; ++lane)
    2631         2924 :                         if (stmts[lane])
    2632         2924 :                           op_stmts.quick_push
    2633         2924 :                             (vinfo->lookup_def (chains[lane][n].op));
    2634              :                         else
    2635            0 :                           op_stmts.quick_push (NULL);
    2636         1039 :                       child = vect_build_slp_tree (vinfo, op_stmts,
    2637              :                                                    matches, limit,
    2638              :                                                    &this_tree_size, bst_map);
    2639              :                       /* ???  We're likely getting too many fatal mismatches
    2640              :                          here so maybe we want to ignore them (but then we
    2641              :                          have no idea which lanes fatally mismatched).  */
    2642              :                       /* ???  Revisit this with matches[] improvements.  */
    2643         1039 :                       if (child || matches[0] != 0)
    2644              :                         break;
    2645              :                       /* Swap another lane we have not yet matched up into
    2646              :                          lanes that did not match.  If we run out of
    2647              :                          permute possibilities for a lane terminate the
    2648              :                          search.  */
    2649          291 :                       bool term = false;
    2650          291 :                       for (lane = 1; lane < group_size; ++lane)
    2651          195 :                         if (matches[lane] != 0)
    2652              :                           {
    2653          167 :                             if (n + perms[lane] + 1 == chain_len)
    2654              :                               {
    2655              :                                 term = true;
    2656              :                                 break;
    2657              :                               }
    2658          148 :                             if (dump_enabled_p ())
    2659          115 :                               dump_printf_loc (MSG_NOTE, vect_location,
    2660              :                                                "swapping operand %d and %d "
    2661              :                                                "of lane %d\n",
    2662              :                                                n, n + perms[lane] + 1, lane);
    2663          296 :                             std::swap (chains[lane][n],
    2664          148 :                                        chains[lane][n + perms[lane] + 1]);
    2665          148 :                             perms[lane]++;
    2666              :                           }
    2667          115 :                       if (term)
    2668              :                         break;
    2669              :                     }
    2670              :                   while (1);
    2671          943 :                   if (!child)
    2672              :                     {
    2673           20 :                       if (dump_enabled_p ())
    2674           18 :                         dump_printf_loc (MSG_NOTE, vect_location,
    2675              :                                          "failed to match up op %d\n", n);
    2676           20 :                       op_stmts.release ();
    2677           20 :                       if (lane != group_size - 1)
    2678           10 :                         matches[0] = -1;
    2679              :                       else
    2680           10 :                         matches[lane] = -1;
    2681           20 :                       goto out;
    2682              :                     }
    2683          923 :                   if (dump_enabled_p ())
    2684              :                     {
    2685          409 :                       dump_printf_loc (MSG_NOTE, vect_location,
    2686              :                                        "matched up op %d to\n", n);
    2687          409 :                       vect_print_slp_tree (MSG_NOTE, vect_location, child);
    2688              :                     }
    2689          923 :                   children.safe_push (child);
    2690              :                 }
    2691              :             }
    2692              :           /* 3. build SLP nodes to combine the chain.  */
    2693         1122 :           for (unsigned lane = 0; lane < group_size; ++lane)
    2694         1640 :             if (stmts[lane] && chains[lane][0].code != code)
    2695              :               {
    2696              :                 /* See if there's any alternate all-PLUS entry.  */
    2697              :                 unsigned n;
    2698            6 :                 for (n = 1; n < chain_len; ++n)
    2699              :                   {
    2700           30 :                     for (lane = 0; lane < group_size; ++lane)
    2701           48 :                       if (stmts[lane] && chains[lane][n].code != code)
    2702              :                         break;
    2703            6 :                     if (lane == group_size)
    2704              :                       break;
    2705              :                   }
    2706            6 :                 if (n != chain_len)
    2707              :                   {
    2708              :                     /* Swap that in at first position.  */
    2709            6 :                     std::swap (children[0], children[n]);
    2710           30 :                     for (lane = 0; lane < group_size; ++lane)
    2711           24 :                       if (stmts[lane])
    2712           24 :                         std::swap (chains[lane][0], chains[lane][n]);
    2713              :                   }
    2714              :                 else
    2715              :                   {
    2716              :                     /* ???  When this triggers and we end up with two
    2717              :                        vect_constant/external_def up-front things break (ICE)
    2718              :                        spectacularly finding an insertion place for the
    2719              :                        all-constant op.  We should have a fully
    2720              :                        vect_internal_def operand though(?) so we can swap
    2721              :                        that into first place and then prepend the all-zero
    2722              :                        constant.  */
    2723            0 :                     if (dump_enabled_p ())
    2724            0 :                       dump_printf_loc (MSG_NOTE, vect_location,
    2725              :                                        "inserting constant zero to compensate "
    2726              :                                        "for (partially) negated first "
    2727              :                                        "operand\n");
    2728            0 :                     chain_len++;
    2729            0 :                     for (lane = 0; lane < group_size; ++lane)
    2730            0 :                       if (stmts[lane])
    2731            0 :                         chains[lane].safe_insert
    2732            0 :                           (0, chain_op_t (code, vect_constant_def, NULL_TREE));
    2733            0 :                     vec<tree> zero_ops;
    2734            0 :                     zero_ops.create (group_size);
    2735            0 :                     zero_ops.quick_push (build_zero_cst (TREE_TYPE (vectype)));
    2736            0 :                     for (lane = 1; lane < group_size; ++lane)
    2737            0 :                       if (stmts[lane])
    2738            0 :                         zero_ops.quick_push (zero_ops[0]);
    2739              :                       else
    2740            0 :                         zero_ops.quick_push (NULL_TREE);
    2741            0 :                     slp_tree zero = vect_create_new_slp_node (zero_ops);
    2742            0 :                     SLP_TREE_DEF_TYPE (zero) = vect_constant_def;
    2743            0 :                     children.safe_insert (0, zero);
    2744              :                   }
    2745              :                 break;
    2746              :               }
    2747          993 :           for (unsigned i = 1; i < children.length (); ++i)
    2748              :             {
    2749          685 :               slp_tree op0 = children[i - 1];
    2750          685 :               slp_tree op1 = children[i];
    2751          685 :               bool this_two_op = false;
    2752         2441 :               for (unsigned lane = 0; lane < group_size; ++lane)
    2753         3836 :                 if (stmts[lane] && chains[lane][i].code != chains[0][i].code)
    2754              :                   {
    2755              :                     this_two_op = true;
    2756              :                     break;
    2757              :                   }
    2758          685 :               slp_tree child;
    2759          685 :               if (i == children.length () - 1)
    2760          308 :                 child = vect_create_new_slp_node (node, stmts, 2);
    2761              :               else
    2762          377 :                 child = vect_create_new_slp_node (2, ERROR_MARK);
    2763          685 :               if (this_two_op)
    2764              :                 {
    2765          162 :                   vec<std::pair<unsigned, unsigned> > lperm;
    2766          162 :                   lperm.create (group_size);
    2767          744 :                   for (unsigned lane = 0; lane < group_size; ++lane)
    2768          840 :                     lperm.quick_push (std::make_pair
    2769          420 :                       (chains[lane][i].code != chains[0][i].code, lane));
    2770          324 :                   vect_slp_build_two_operator_nodes (child, vectype, op0, op1,
    2771          162 :                                                      (chains[0][i].code == code
    2772              :                                                       ? op_stmt_info
    2773              :                                                       : other_op_stmt_info),
    2774          162 :                                                      (chains[0][i].code == code
    2775              :                                                       ? other_op_stmt_info
    2776              :                                                       : op_stmt_info),
    2777              :                                                      lperm);
    2778              :                 }
    2779              :               else
    2780              :                 {
    2781          523 :                   SLP_TREE_DEF_TYPE (child) = vect_internal_def;
    2782          523 :                   SLP_TREE_VECTYPE (child) = vectype;
    2783          523 :                   SLP_TREE_LANES (child) = group_size;
    2784          523 :                   SLP_TREE_CHILDREN (child).quick_push (op0);
    2785          523 :                   SLP_TREE_CHILDREN (child).quick_push (op1);
    2786          523 :                   SLP_TREE_REPRESENTATIVE (child)
    2787         1046 :                     = (chains[0][i].code == code
    2788          523 :                        ? op_stmt_info : other_op_stmt_info);
    2789              :                 }
    2790          685 :               children[i] = child;
    2791              :             }
    2792          308 :           *tree_size += this_tree_size + 1;
    2793         1477 :           while (!chains.is_empty ())
    2794          838 :             chains.pop ().release ();
    2795              :           return node;
    2796              :         }
    2797        20333 : out:
    2798        20356 :       if (dump_enabled_p ())
    2799         2825 :         dump_printf_loc (MSG_NOTE, vect_location,
    2800              :                          "failed to line up SLP graph by re-associating "
    2801              :                          "operations in lanes%s\n",
    2802              :                          !hard_fail ? " trying regular discovery" : "");
    2803        20363 :       while (!children.is_empty ())
    2804            7 :         vect_free_slp_tree (children.pop ());
    2805        20521 :       while (!chains.is_empty ())
    2806          165 :         chains.pop ().release ();
    2807              :       /* Hard-fail, otherwise we might run into quadratic processing of the
    2808              :          chains starting one stmt into the chain again.  */
    2809        20356 :       if (hard_fail)
    2810              :         return NULL;
    2811              :       /* Fall thru to normal processing.  */
    2812        20664 :     }
    2813              : 
    2814              :   /* Get at the operands, verifying they are compatible.  */
    2815      3738285 :   vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
    2816      3738285 :   slp_oprnd_info oprnd_info;
    2817     21139138 :   FOR_EACH_VEC_ELT (stmts, i, stmt_info)
    2818              :     {
    2819     27327912 :       int res = vect_get_and_check_slp_defs (vinfo, vectype,
    2820     13663956 :                                              swap[i], skip_args,
    2821              :                                              stmts, i, &oprnds_info,
    2822              :                                              soft_fail);
    2823     13663956 :       if (res != 0)
    2824              :         /* ???  This puts -1 back into matches[] and the cache.  */
    2825       342093 :         matches[(res == -1) ? 0 : i] = -1;
    2826     13663956 :       if (matches[0] == -1)
    2827              :         break;
    2828              :     }
    2829     17006507 :   for (i = 0; i < group_size; ++i)
    2830     13520085 :     if (matches[i] != 0)
    2831              :       {
    2832       251863 :         vect_free_oprnd_info (oprnds_info);
    2833       251863 :         return NULL;
    2834              :       }
    2835      3486422 :   swap = NULL;
    2836              : 
    2837              :   /* Perform delayed soft-failing only here so we can factor in mismatches
    2838              :      determined by vect_get_and_check_slp_defs.  */
    2839      3486422 :   if (soft_fail)
    2840              :     {
    2841       122353 :       force_split_matches (matches, group_size,
    2842       122353 :                            TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
    2843       122353 :       return NULL;
    2844              :     }
    2845              : 
    2846      3364069 :   bool has_two_operators_perm = false;
    2847     16820345 :   auto_vec<unsigned> two_op_perm_indices[2];
    2848      3364069 :   vec<stmt_vec_info> two_op_scalar_stmts[2] = {vNULL, vNULL};
    2849              : 
    2850      3380844 :   if (two_operators && oprnds_info.length () == 2 && group_size > 2)
    2851              :     {
    2852         2802 :       unsigned idx = 0;
    2853         2802 :       hash_map<gimple *, unsigned> seen;
    2854         2802 :       vec<slp_oprnd_info> new_oprnds_info
    2855         2802 :         = vect_create_oprnd_info (1, group_size);
    2856         2802 :       bool success = true;
    2857              : 
    2858         2802 :       enum tree_code code = ERROR_MARK;
    2859         2802 :       if (oprnds_info[0]->def_stmts[0]
    2860         2802 :           && is_a<gassign *> (oprnds_info[0]->def_stmts[0]->stmt))
    2861         2742 :         code = gimple_assign_rhs_code (oprnds_info[0]->def_stmts[0]->stmt);
    2862         2802 :       basic_block bb = nullptr;
    2863              : 
    2864         6504 :       for (unsigned j = 0; j < group_size; ++j)
    2865              :         {
    2866        15572 :           FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
    2867              :             {
    2868        11870 :               stmt_vec_info stmt_info = oprnd_info->def_stmts[j];
    2869        11870 :               if (!stmt_info
    2870        11697 :                   || !is_a<gassign *> (stmt_info->stmt)
    2871        11697 :                   || gimple_assign_rhs_code (stmt_info->stmt) != code
    2872        21421 :                   || skip_args[i])
    2873              :                 {
    2874              :                   success = false;
    2875         2323 :                   break;
    2876              :                 }
    2877              :               /* Avoid mixing lanes with defs in different basic-blocks.  */
    2878         9551 :               if (!bb)
    2879         2962 :                 bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
    2880         8547 :               else if (gimple_bb (vect_orig_stmt (stmt_info)->stmt) != bb)
    2881              :                 {
    2882              :                   success = false;
    2883              :                   break;
    2884              :                 }
    2885              : 
    2886         9547 :               bool exists;
    2887         9547 :               unsigned &stmt_idx
    2888         9547 :                 = seen.get_or_insert (stmt_info->stmt, &exists);
    2889              : 
    2890         9547 :               if (!exists)
    2891              :                 {
    2892         8166 :                   new_oprnds_info[0]->def_stmts.safe_push (stmt_info);
    2893         8166 :                   new_oprnds_info[0]->ops.safe_push (oprnd_info->ops[j]);
    2894         8166 :                   stmt_idx = idx;
    2895         8166 :                   idx++;
    2896              :                 }
    2897              : 
    2898         9547 :               two_op_perm_indices[i].safe_push (stmt_idx);
    2899              :             }
    2900              : 
    2901         6025 :           if (!success)
    2902              :             break;
    2903              :         }
    2904              : 
    2905         2802 :       if (success && idx == group_size)
    2906              :         {
    2907           98 :           if (dump_enabled_p ())
    2908              :             {
    2909            0 :               dump_printf_loc (MSG_NOTE, vect_location,
    2910              :                                "Replace two_operators operands:\n");
    2911              : 
    2912            0 :               FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
    2913              :                 {
    2914            0 :                   dump_printf_loc (MSG_NOTE, vect_location,
    2915              :                                    "Operand %u:\n", i);
    2916            0 :                   for (unsigned j = 0; j < group_size; j++)
    2917            0 :                     dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
    2918            0 :                                      j, oprnd_info->def_stmts[j]->stmt);
    2919              :                 }
    2920              : 
    2921            0 :               dump_printf_loc (MSG_NOTE, vect_location,
    2922              :                                "With a single operand:\n");
    2923            0 :               for (unsigned j = 0; j < group_size; j++)
    2924            0 :                 dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
    2925            0 :                                  j, new_oprnds_info[0]->def_stmts[j]->stmt);
    2926              :             }
    2927              : 
    2928           98 :           two_op_scalar_stmts[0].safe_splice (oprnds_info[0]->def_stmts);
    2929           98 :           two_op_scalar_stmts[1].safe_splice (oprnds_info[1]->def_stmts);
    2930              : 
    2931           98 :           new_oprnds_info[0]->first_op_type = oprnds_info[0]->first_op_type;
    2932           98 :           new_oprnds_info[0]->first_dt = oprnds_info[0]->first_dt;
    2933           98 :           new_oprnds_info[0]->any_pattern = oprnds_info[0]->any_pattern;
    2934           98 :           new_oprnds_info[0]->first_gs_p = oprnds_info[0]->first_gs_p;
    2935           98 :           new_oprnds_info[0]->first_gs_info = oprnds_info[0]->first_gs_info;
    2936              : 
    2937           98 :           vect_free_oprnd_info (oprnds_info);
    2938           98 :           oprnds_info = new_oprnds_info;
    2939           98 :           nops = 1;
    2940           98 :           has_two_operators_perm = true;
    2941              :         }
    2942              :       else
    2943         2704 :         vect_free_oprnd_info (new_oprnds_info);
    2944         2802 :     }
    2945              : 
    2946      6728138 :   auto_vec<slp_tree, 4> children;
    2947              : 
    2948      3364069 :   stmt_info = stmts[0];
    2949              : 
    2950      3364069 :   int reduc_idx = -1;
    2951      3364069 :   int gs_scale = 0;
    2952      3364069 :   tree gs_base = NULL_TREE;
    2953              : 
    2954              :   /* Create SLP_TREE nodes for the definition node/s.  */
    2955      8659920 :   FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
    2956              :     {
    2957      5399741 :       slp_tree child = nullptr;
    2958      5399741 :       unsigned int j;
    2959      5399741 :       unsigned old_swap_distance;
    2960      5399741 :       bool can_swap;
    2961      5399741 :       bool can_swap_nonmatching;
    2962      5399741 :       bool *stmt_can_swap;
    2963              : 
    2964              :       /* We're skipping certain operands from processing, for example
    2965              :          outer loop reduction initial defs.  */
    2966      5399741 :       if (skip_args[i])
    2967              :         {
    2968       531125 :           children.safe_push (NULL);
    2969      5295851 :           continue;
    2970              :         }
    2971              : 
    2972      4868616 :       if (oprnd_info->first_dt == vect_uninitialized_def)
    2973              :         {
    2974              :           /* COND_EXPR have one too many eventually if the condition
    2975              :              is a SSA name.  */
    2976            0 :           gcc_assert (i == 3 && nops == 4);
    2977            0 :           continue;
    2978              :         }
    2979              : 
    2980      4868616 :       if (oprnd_info->first_gs_p)
    2981              :         {
    2982        27120 :           gs_scale = oprnd_info->first_gs_info.scale;
    2983        27120 :           gs_base = oprnd_info->first_gs_info.base;
    2984              :         }
    2985              : 
    2986      4868616 :       if (is_a <bb_vec_info> (vinfo)
    2987      1691433 :           && oprnd_info->first_dt == vect_internal_def
    2988      5772085 :           && !oprnd_info->any_pattern)
    2989              :         {
    2990              :           /* For BB vectorization, if all defs are the same do not
    2991              :              bother to continue the build along the single-lane
    2992              :              graph but use a splat of the scalar value.  */
    2993       857411 :           stmt_vec_info first_def = oprnd_info->def_stmts[0];
    2994       924133 :           for (j = 1; j < group_size; ++j)
    2995       873196 :             if (oprnd_info->def_stmts[j] != first_def)
    2996              :               break;
    2997       857411 :           if (j == group_size
    2998              :               /* But avoid doing this for loads where we may be
    2999              :                  able to CSE things, unless the stmt is not
    3000              :                  vectorizable.  */
    3001       857411 :               && (!STMT_VINFO_VECTORIZABLE (first_def)
    3002        61122 :                   || !gimple_vuse (first_def->stmt)))
    3003              :             {
    3004        41211 :               if (dump_enabled_p ())
    3005          119 :                 dump_printf_loc (MSG_NOTE, vect_location,
    3006              :                                  "Using a splat of the uniform operand %G",
    3007              :                                  first_def->stmt);
    3008        41211 :               oprnd_info->first_dt = vect_external_def;
    3009              :             }
    3010              :         }
    3011              : 
    3012      4868616 :       if (oprnd_info->first_dt == vect_external_def
    3013      4868616 :           || oprnd_info->first_dt == vect_constant_def)
    3014              :         {
    3015      1605919 :           if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ())
    3016              :             {
    3017              :               tree op0;
    3018              :               tree uniform_val = op0 = oprnd_info->ops[0];
    3019              :               for (j = 1; j < oprnd_info->ops.length (); ++j)
    3020              :                 if (oprnd_info->ops[j]
    3021              :                     && !operand_equal_p (uniform_val, oprnd_info->ops[j]))
    3022              :                   {
    3023              :                     uniform_val = NULL_TREE;
    3024              :                     break;
    3025              :                   }
    3026              :               if (!uniform_val
    3027              :                   && !can_duplicate_and_interleave_p (vinfo,
    3028              :                                                       oprnd_info->ops.length (),
    3029              :                                                       TREE_TYPE (op0)))
    3030              :                 {
    3031              :                   for (unsigned k = 0; k < j; ++k)
    3032              :                     matches[k] = 0;
    3033              :                   matches[j] = -1;
    3034              :                   for (unsigned k = j + 1; k < oprnd_info->ops.length (); ++k)
    3035              :                     if (oprnd_info->ops[k]
    3036              :                         && !operand_equal_p (op0, oprnd_info->ops[k]))
    3037              :                       matches[k] = -1;
    3038              :                     else
    3039              :                       matches[k] = 0;
    3040              :                   if (dump_enabled_p ())
    3041              :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3042              :                                      "Build SLP failed: invalid type of def "
    3043              :                                      "for variable-length SLP %T\n", op0);
    3044              :                   goto fail;
    3045              :                 }
    3046              :             }
    3047      1605919 :           slp_tree invnode = vect_create_new_slp_node (oprnd_info->ops);
    3048      1605919 :           SLP_TREE_DEF_TYPE (invnode) = oprnd_info->first_dt;
    3049      1605919 :           oprnd_info->ops = vNULL;
    3050      1605919 :           children.safe_push (invnode);
    3051      1605919 :           continue;
    3052      1605919 :         }
    3053              : 
    3054              :       /* See which SLP operand a reduction chain continues on.  We want
    3055              :          to chain even PHIs but not backedges.  */
    3056      3262697 :       if (STMT_VINFO_REDUC_DEF (oprnd_info->def_stmts[0])
    3057      3262697 :           || STMT_VINFO_REDUC_IDX (oprnd_info->def_stmts[0]) != -1)
    3058              :         {
    3059       234481 :           if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
    3060              :             {
    3061          760 :               if (oprnd_info->first_dt == vect_double_reduction_def)
    3062          380 :                 reduc_idx = i;
    3063              :             }
    3064       233721 :           else if (is_a <gphi *> (stmt_info->stmt)
    3065       233721 :                    && gimple_phi_num_args
    3066       101820 :                         (as_a <gphi *> (stmt_info->stmt)) != 1)
    3067              :             ;
    3068       132286 :           else if (STMT_VINFO_REDUC_IDX (stmt_info) == -1
    3069          385 :                    && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
    3070              :             ;
    3071       132286 :           else if (reduc_idx == -1)
    3072       123794 :             reduc_idx = i;
    3073              :           else
    3074              :             /* For .COND_* reduction operations the else value can be the
    3075              :                same as one of the operation operands.  The other def
    3076              :                stmts have been moved, so we can't check easily.  Check
    3077              :                it's a call at least.  */
    3078         8492 :             gcc_assert (is_a <gcall *> (stmt_info->stmt));
    3079              :         }
    3080              : 
    3081              :       /* When we have a masked load with uniform mask discover this
    3082              :          as a single-lane mask with a splat permute.  This way we can
    3083              :          recognize this as a masked load-lane by stripping the splat.  */
    3084      3262697 :       if (is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
    3085        60305 :           && gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
    3086              :                                      IFN_MASK_LOAD)
    3087         4763 :           && STMT_VINFO_GROUPED_ACCESS (stmt_info)
    3088      3262774 :           && ! STMT_VINFO_SLP_VECT_ONLY (DR_GROUP_FIRST_ELEMENT (stmt_info)))
    3089              :         {
    3090           35 :           vec<stmt_vec_info> def_stmts2;
    3091           35 :           def_stmts2.create (1);
    3092           35 :           def_stmts2.quick_push (oprnd_info->def_stmts[0]);
    3093           35 :           child = vect_build_slp_tree (vinfo, def_stmts2,
    3094              :                                        matches, limit,
    3095              :                                        &this_tree_size, bst_map);
    3096           35 :           if (child)
    3097              :             {
    3098           35 :               slp_tree pnode = vect_create_new_slp_node (1, VEC_PERM_EXPR);
    3099           35 :               SLP_TREE_VECTYPE (pnode) = SLP_TREE_VECTYPE (child);
    3100           35 :               SLP_TREE_LANES (pnode) = group_size;
    3101           35 :               SLP_TREE_SCALAR_STMTS (pnode).create (group_size);
    3102           35 :               SLP_TREE_LANE_PERMUTATION (pnode).create (group_size);
    3103          245 :               for (unsigned k = 0; k < group_size; ++k)
    3104              :                 {
    3105          175 :                   SLP_TREE_SCALAR_STMTS (pnode)
    3106          175 :                     .quick_push (oprnd_info->def_stmts[0]);
    3107          175 :                   SLP_TREE_LANE_PERMUTATION (pnode)
    3108          175 :                     .quick_push (std::make_pair (0u, 0u));
    3109              :                 }
    3110           35 :               SLP_TREE_CHILDREN (pnode).quick_push (child);
    3111           35 :               children.safe_push (pnode);
    3112           35 :               oprnd_info->def_stmts = vNULL;
    3113           35 :               continue;
    3114           35 :             }
    3115              :           else
    3116            0 :             def_stmts2.release ();
    3117              :         }
    3118              : 
    3119      6525324 :       can_swap = (i == 0
    3120      2424738 :                   && (nops == 2 || nops == 3)
    3121      1553109 :                   && oprnds_info.length () > 1
    3122      1553109 :                   && oprnds_info[1]->first_dt == vect_internal_def
    3123       617093 :                   && (is_gimple_assign (stmt_info->stmt)
    3124        51897 :                       || is_gimple_call (stmt_info->stmt))
    3125              :                   /* Swapping operands for reductions breaks assumptions
    3126              :                      later on.  */
    3127      3833277 :                   && STMT_VINFO_REDUC_IDX (stmt_info) == -1);
    3128      3262662 :       can_swap_nonmatching = can_swap;
    3129      3262662 :       stmt_can_swap = NULL;
    3130      3262662 :       if (can_swap)
    3131              :         {
    3132       519362 :           stmt_can_swap = XALLOCAVEC (bool, group_size);
    3133      8140022 :           for (j = 0; j < group_size; ++j)
    3134              :             {
    3135      7620660 :               stmt_can_swap[j] = false;
    3136      7620660 :               if (!stmts[j])
    3137              :                 /* NULL lanes are gaps and have no stmt to swap.  */
    3138            0 :                 stmt_can_swap[j] = true;
    3139      7620660 :               else if (gassign *stmt = dyn_cast <gassign *> (stmts[j]->stmt))
    3140              :                 {
    3141      7614379 :                   tree_code code = gimple_assign_rhs_code (stmt);
    3142     15228758 :                   stmt_can_swap[j] = (commutative_tree_code (code)
    3143      7614379 :                                       || commutative_ternary_tree_code (code));
    3144              :                 }
    3145         6281 :               else if (gcall *call = dyn_cast <gcall *> (stmts[j]->stmt))
    3146              :                 {
    3147         6281 :                   internal_fn fn = (gimple_call_internal_p (call)
    3148         6281 :                                     ? gimple_call_internal_fn (call) : IFN_LAST);
    3149        12562 :                   stmt_can_swap[j] = ((commutative_binary_fn_p (fn)
    3150         5026 :                                        || commutative_ternary_fn_p (fn))
    3151         6317 :                                       && first_commutative_argument (fn) == 0);
    3152              :                 }
    3153              : 
    3154      7620660 :               if (j != 0 && !stmt_can_swap[j])
    3155      7620660 :                 can_swap_nonmatching = false;
    3156              :             }
    3157              :         }
    3158              : 
    3159      3262662 :       old_swap_distance = least_upthread_swappable_op_distance;
    3160      3262662 :       if (can_swap_nonmatching)
    3161       477425 :         least_upthread_swappable_op_distance = 1;
    3162      2785237 :       else if (least_upthread_swappable_op_distance != -1U)
    3163       345832 :         least_upthread_swappable_op_distance++;
    3164      3262662 :       child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
    3165              :                                    matches, limit,
    3166              :                                    &this_tree_size, bst_map);
    3167      3262662 :       least_upthread_swappable_op_distance = old_swap_distance;
    3168      3262662 :       if (child != NULL)
    3169              :         {
    3170      2762692 :           oprnd_info->def_stmts = vNULL;
    3171      2762692 :           children.safe_push (child);
    3172      2762692 :           continue;
    3173              :         }
    3174              : 
    3175              :       /* If the SLP build for operand zero failed and operand zero
    3176              :          and one can be commuted try that for the scalar stmts
    3177              :          that failed the match.  */
    3178       499970 :       if (/* A first scalar stmt mismatch signals a fatal mismatch.  */
    3179       499970 :           matches[0] == 0
    3180       499970 :           && can_swap)
    3181              :         {
    3182              :           /* See whether we can swap the matching or the non-matching
    3183              :              stmt operands.  */
    3184              :           bool swap_not_matching = true;
    3185        69760 :           do
    3186              :             {
    3187      7072780 :               for (j = 0; j < group_size; ++j)
    3188              :                 {
    3189      7021225 :                   if ((matches[j] == 0) != !swap_not_matching)
    3190        76951 :                     continue;
    3191              :                   /* Verify if we can swap operands of this stmt.  */
    3192      6944274 :                   if (!stmt_can_swap[j])
    3193              :                     {
    3194        18205 :                       if (!swap_not_matching)
    3195         8020 :                         goto fail;
    3196              :                       swap_not_matching = false;
    3197              :                       break;
    3198              :                     }
    3199              :                 }
    3200              :             }
    3201        61740 :           while (j != group_size);
    3202              : 
    3203              :           /* Swap mismatched definition stmts.  */
    3204        51555 :           if (dump_enabled_p ())
    3205          432 :             dump_printf_loc (MSG_NOTE, vect_location,
    3206              :                              "Re-trying with swapped operands of stmts ");
    3207      7043333 :           for (j = 0; j < group_size; ++j)
    3208      6991778 :             if ((matches[j] == 0) == !swap_not_matching)
    3209              :               {
    3210     13851730 :                 std::swap (oprnds_info[0]->def_stmts[j],
    3211      6925865 :                            oprnds_info[1]->def_stmts[j]);
    3212     13851730 :                 std::swap (oprnds_info[0]->ops[j],
    3213      6925865 :                            oprnds_info[1]->ops[j]);
    3214      6925865 :                 if (dump_enabled_p ())
    3215         1101 :                   dump_printf (MSG_NOTE, "%d ", j);
    3216              :               }
    3217        51555 :           if (dump_enabled_p ())
    3218          432 :             dump_printf (MSG_NOTE, "\n");
    3219              :           /* After swapping some operands we lost track whether an
    3220              :              operand has any pattern defs so be conservative here.  */
    3221        99961 :           if (oprnds_info[0]->any_pattern || oprnds_info[1]->any_pattern)
    3222         3212 :             oprnds_info[0]->any_pattern = oprnds_info[1]->any_pattern = true;
    3223              :           /* And try again with scratch 'matches' ... */
    3224        51555 :           match_elt_t *tem = XALLOCAVEC (match_elt_t, group_size);
    3225        51555 :           if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
    3226              :                                             tem, limit,
    3227              :                                             &this_tree_size, bst_map)) != NULL)
    3228              :             {
    3229         6277 :               oprnd_info->def_stmts = vNULL;
    3230         6277 :               children.safe_push (child);
    3231         6277 :               continue;
    3232              :             }
    3233              :         }
    3234       493693 : fail:
    3235              : 
    3236              :       /* If the SLP build failed and we analyze a basic-block
    3237              :          simply treat nodes we fail to build as externally defined
    3238              :          (and thus build vectors from the scalar defs).
    3239              :          The cost model will reject outright expensive cases.
    3240              :          ???  This doesn't treat cases where permutation ultimatively
    3241              :          fails (or we don't try permutation below).  Ideally we'd
    3242              :          even compute a permutation that will end up with the maximum
    3243              :          SLP tree size...  */
    3244       493693 :       if (is_a <bb_vec_info> (vinfo)
    3245              :           /* ???  Rejecting patterns this way doesn't work.  We'd have to
    3246              :              do extra work to cancel the pattern so the uses see the
    3247              :              scalar version.  */
    3248              :           /* Skip building vector operands from scalars while operand
    3249              :              discovery may still be fixed by retrying with swapped operands.  */
    3250       440829 :           && (least_upthread_swappable_op_distance != 1
    3251              :               /* A first scalar stmt mismatch signals a fatal mismatch
    3252              :                  that the parent commutative retry cannot recover.
    3253              :                  ???  Possibly revisit this with the matches[] improvements
    3254              :                  as we can have matches[0] == -2 here.  */
    3255        26894 :               || matches[0] != 0)
    3256       422568 :           && !is_pattern_stmt_p (stmt_info)
    3257       890357 :           && !oprnd_info->any_pattern)
    3258              :         {
    3259              :           /* But if there's a leading vector sized set of matching stmts
    3260              :              fail here so we can split the group.  This matches the condition
    3261              :              vect_analyze_slp_instance uses.  */
    3262              :           /* ???  We might want to split here and combine the results to support
    3263              :              multiple vector sizes better.  */
    3264       616313 :           for (j = 0; j < group_size; ++j)
    3265       616313 :             if (matches[j] != 0)
    3266              :               break;
    3267       396291 :           if (!known_ge (j, TYPE_VECTOR_SUBPARTS (vectype))
    3268       396257 :               && vect_slp_can_convert_to_external (oprnd_info->def_stmts))
    3269              :             {
    3270       389803 :               if (dump_enabled_p ())
    3271          809 :                 dump_printf_loc (MSG_NOTE, vect_location,
    3272              :                                  "Building vector operands from scalars\n");
    3273       389803 :               this_tree_size++;
    3274       389803 :               child = vect_create_new_slp_node (oprnd_info->ops);
    3275       389803 :               children.safe_push (child);
    3276       389803 :               oprnd_info->ops = vNULL;
    3277       389803 :               continue;
    3278              :             }
    3279              :         }
    3280              : 
    3281       103890 :       gcc_assert (child == NULL);
    3282       223451 :       FOR_EACH_VEC_ELT (children, j, child)
    3283        15671 :         if (child)
    3284        15671 :           vect_free_slp_tree (child);
    3285       103890 :       vect_free_oprnd_info (oprnds_info);
    3286       103890 :       return NULL;
    3287              :     }
    3288              : 
    3289      3260179 :   vect_free_oprnd_info (oprnds_info);
    3290              : 
    3291              :   /* If we have all children of a child built up from uniform scalars
    3292              :      or does more than one possibly expensive vector construction then
    3293              :      just throw that away, causing it built up from scalars.
    3294              :      The exception is the SLP node for the vector store.  */
    3295      3260179 :   if (is_a <bb_vec_info> (vinfo)
    3296      1156609 :       && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
    3297              :       /* ???  Rejecting patterns this way doesn't work.  We'd have to
    3298              :          do extra work to cancel the pattern so the uses see the
    3299              :          scalar version.  */
    3300      3740091 :       && !is_pattern_stmt_p (stmt_info))
    3301              :     {
    3302              :       slp_tree child;
    3303              :       unsigned j;
    3304              :       bool all_uniform_p = true;
    3305              :       unsigned n_vector_builds = 0;
    3306      1351992 :       FOR_EACH_VEC_ELT (children, j, child)
    3307              :         {
    3308       901236 :           if (!child)
    3309              :             ;
    3310       901236 :           else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
    3311              :             all_uniform_p = false;
    3312       643549 :           else if (!vect_slp_tree_uniform_p (child))
    3313              :             {
    3314       487290 :               all_uniform_p = false;
    3315       487290 :               if (SLP_TREE_DEF_TYPE (child) == vect_external_def)
    3316       446609 :                 n_vector_builds++;
    3317              :             }
    3318              :         }
    3319       450756 :       if (all_uniform_p
    3320       450756 :           || n_vector_builds > 1
    3321       765616 :           || (n_vector_builds == children.length ()
    3322        35452 :               && is_a <gphi *> (stmt_info->stmt)))
    3323              :         {
    3324              :           /* Roll back.  */
    3325       140384 :           matches[0] = -1;
    3326       444730 :           FOR_EACH_VEC_ELT (children, j, child)
    3327       304346 :             if (child)
    3328       304346 :               vect_free_slp_tree (child);
    3329              : 
    3330       140384 :           if (dump_enabled_p ())
    3331          262 :             dump_printf_loc (MSG_NOTE, vect_location,
    3332              :                              "Building parent vector operands from "
    3333              :                              "scalars instead\n");
    3334      3364069 :           return NULL;
    3335              :         }
    3336              :     }
    3337              : 
    3338      3119795 :   *tree_size += this_tree_size + 1;
    3339              : 
    3340      3119795 :   if (two_operators)
    3341              :     {
    3342              :       /* ???  We'd likely want to either cache in bst_map sth like
    3343              :          { a+b, NULL, a+b, NULL } and { NULL, a-b, NULL, a-b } or
    3344              :          the true { a+b, a+b, a+b, a+b } ... but there we don't have
    3345              :          explicit stmts to put in so the keying on 'stmts' doesn't
    3346              :          work (but we have the same issue with nodes that use 'ops').  */
    3347              : 
    3348         9338 :       if (has_two_operators_perm)
    3349              :         {
    3350           46 :           slp_tree child = children[0];
    3351           46 :           children.truncate (0);
    3352          184 :           for (i = 0; i < 2; i++)
    3353              :             {
    3354           92 :               slp_tree pnode
    3355           92 :                 = vect_create_new_slp_node (two_op_scalar_stmts[i], 2);
    3356           92 :               SLP_TREE_CODE (pnode) = VEC_PERM_EXPR;
    3357           92 :               SLP_TREE_VECTYPE (pnode) = vectype;
    3358           92 :               SLP_TREE_CHILDREN (pnode).quick_push (child);
    3359           92 :               SLP_TREE_CHILDREN (pnode).quick_push (child);
    3360           92 :               SLP_TREE_REPRESENTATIVE (pnode) = NULL;
    3361           92 :               lane_permutation_t& perm = SLP_TREE_LANE_PERMUTATION (pnode);
    3362           92 :               children.safe_push (pnode);
    3363              : 
    3364          808 :               for (unsigned j = 0; j < stmts.length (); j++)
    3365          624 :                 perm.safe_push (std::make_pair (0, two_op_perm_indices[i][j]));
    3366              :             }
    3367              : 
    3368           46 :           SLP_TREE_REF_COUNT (child) += 4;
    3369              :         }
    3370              : 
    3371         9338 :       slp_tree one = new _slp_tree;
    3372         9338 :       slp_tree two = new _slp_tree;
    3373         9338 :       SLP_TREE_DEF_TYPE (one) = vect_internal_def;
    3374         9338 :       SLP_TREE_DEF_TYPE (two) = vect_internal_def;
    3375         9338 :       SLP_TREE_VECTYPE (one) = vectype;
    3376         9338 :       SLP_TREE_VECTYPE (two) = vectype;
    3377         9338 :       SLP_TREE_CHILDREN (one).safe_splice (children);
    3378         9338 :       SLP_TREE_CHILDREN (two).safe_splice (children);
    3379         9338 :       slp_tree child;
    3380        37354 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (two), i, child)
    3381        18678 :         SLP_TREE_REF_COUNT (child)++;
    3382              : 
    3383              :       /* Here we record the original defs since this
    3384              :          node represents the final lane configuration.  */
    3385         9338 :       node = vect_create_new_slp_node (node, stmts, 2);
    3386         9338 :       SLP_TREE_VECTYPE (node) = vectype;
    3387         9338 :       SLP_TREE_CODE (node) = VEC_PERM_EXPR;
    3388         9338 :       SLP_TREE_CHILDREN (node).quick_push (one);
    3389         9338 :       SLP_TREE_CHILDREN (node).quick_push (two);
    3390         9338 :       SLP_TREE_REPRESENTATIVE (node) = NULL;
    3391         9338 :       enum tree_code code0 = ERROR_MARK;
    3392         9338 :       if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt))
    3393         9336 :         code0 = gimple_assign_rhs_code (stmt);
    3394         9338 :       stmt_vec_info ostmt_info;
    3395         9338 :       unsigned j = 0;
    3396        32421 :       FOR_EACH_VEC_ELT (stmts, i, ostmt_info)
    3397              :         {
    3398        23083 :           int op = 0;
    3399        23083 :           gassign *ostmt = dyn_cast<gassign *> (ostmt_info->stmt);
    3400        23079 :           if ((ostmt && gimple_assign_rhs_code (ostmt) != code0)
    3401              :               || (!ostmt
    3402            8 :                   && gimple_call_combined_fn (stmts[0]->stmt)
    3403            4 :                      != gimple_call_combined_fn (ostmt_info->stmt)))
    3404              :             {
    3405              :               op = 1;
    3406              :               j = i;
    3407              :             }
    3408        23083 :           SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i));
    3409              :         }
    3410         9338 :       SLP_TREE_LANES (one) = stmts.length ();
    3411         9338 :       SLP_TREE_LANES (two) = stmts.length ();
    3412         9338 :       SLP_TREE_REPRESENTATIVE (one) = stmts[0];
    3413         9338 :       SLP_TREE_REPRESENTATIVE (two) = stmts[j];
    3414              : 
    3415         9338 :       return node;
    3416              :     }
    3417              : 
    3418      3110457 :   node = vect_create_new_slp_node (node, stmts, nops);
    3419      3110457 :   SLP_TREE_VECTYPE (node) = vectype;
    3420      3110457 :   SLP_TREE_CHILDREN (node).splice (children);
    3421      3110457 :   SLP_TREE_GS_SCALE (node) = gs_scale;
    3422      3110457 :   SLP_TREE_GS_BASE (node) = gs_base;
    3423      3110457 :   if (reduc_idx != -1)
    3424              :     {
    3425       116899 :       gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) != -1
    3426              :                   || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
    3427              :                   || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def);
    3428       116899 :       SLP_TREE_REDUC_IDX (node) = reduc_idx;
    3429       116899 :       node->cycle_info.id = SLP_TREE_CHILDREN (node)[reduc_idx]->cycle_info.id;
    3430              :     }
    3431              :   /* When reaching the reduction PHI, create a vect_reduc_info.  */
    3432      2993558 :   else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
    3433      2993558 :             || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
    3434      2993558 :            && is_a <gphi *> (STMT_VINFO_STMT (stmt_info)))
    3435              :     {
    3436       102095 :       loop_vec_info loop_vinfo = as_a <loop_vec_info> (vinfo);
    3437       102095 :       gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) == -1);
    3438       102095 :       node->cycle_info.id = loop_vinfo->reduc_infos.length ();
    3439       102095 :       vect_reduc_info reduc_info = new vect_reduc_info_s ();
    3440       102095 :       loop_vinfo->reduc_infos.safe_push (reduc_info);
    3441       102095 :       stmt_vec_info reduc_phi = stmt_info;
    3442              :       /* ???  For double reductions vect_is_simple_reduction stores the
    3443              :          reduction type and code on the inner loop header PHI.  */
    3444       102095 :       if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
    3445              :         {
    3446          380 :           use_operand_p use_p;
    3447          380 :           gimple *use_stmt;
    3448          380 :           bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
    3449              :                                      &use_p, &use_stmt);
    3450          380 :           gcc_assert (res);
    3451          380 :           reduc_phi = loop_vinfo->lookup_stmt (use_stmt);
    3452              :         }
    3453       102095 :       VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (stmt_info);
    3454       102095 :       VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (reduc_phi);
    3455       102095 :       VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (reduc_phi);
    3456       102095 :       VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
    3457              :     }
    3458              :   return node;
    3459     10092207 : }
    3460              : 
    3461              : /* Dump a single SLP tree NODE.  */
    3462              : 
    3463              : static void
    3464       492228 : vect_print_slp_tree (dump_flags_t dump_kind, dump_location_t loc,
    3465              :                      slp_tree node)
    3466              : {
    3467       492228 :   unsigned i, j;
    3468       492228 :   slp_tree child;
    3469       492228 :   stmt_vec_info stmt_info;
    3470       492228 :   tree op;
    3471              : 
    3472       492228 :   dump_metadata_t metadata (dump_kind, loc.get_impl_location ());
    3473       492228 :   dump_user_location_t user_loc = loc.get_user_location ();
    3474       492228 :   dump_printf_loc (metadata, user_loc,
    3475              :                    "node%s %p (refcnt=%u)",
    3476       492228 :                    SLP_TREE_DEF_TYPE (node) == vect_external_def
    3477              :                    ? " (external)"
    3478              :                    : (SLP_TREE_DEF_TYPE (node) == vect_constant_def
    3479       475222 :                       ? " (constant)"
    3480              :                       : ""), (void *) node,
    3481              :                    SLP_TREE_REF_COUNT (node));
    3482       492228 :   if (SLP_TREE_VECTYPE (node))
    3483       414875 :     dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
    3484       492228 :   dump_printf (metadata, "%s",
    3485       492228 :                node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
    3486       492228 :   if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
    3487        25083 :     dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
    3488              :                  node->cycle_info.reduc_idx);
    3489       492228 :   if (node->si)
    3490         5460 :     dump_printf (metadata, " @%G", node->si);
    3491              :   else
    3492       486768 :     dump_printf (metadata, "\n");
    3493       492228 :   if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
    3494              :     {
    3495       398631 :       if (SLP_TREE_PERMUTE_P (node))
    3496        17326 :         dump_printf_loc (metadata, user_loc, "op: VEC_PERM_EXPR\n");
    3497              :       else
    3498       381305 :         dump_printf_loc (metadata, user_loc, "op template: %G",
    3499       381305 :                          SLP_TREE_REPRESENTATIVE (node)->stmt);
    3500              :     }
    3501       492228 :   if (SLP_TREE_SCALAR_STMTS (node).exists ())
    3502       951097 :     FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
    3503       561482 :       if (stmt_info)
    3504       555324 :         dump_printf_loc (metadata, user_loc, "\t%sstmt %u %G",
    3505       555324 :                          SLP_TREE_LIVE_LANES (node).contains (i)
    3506       551497 :                          ? "[l*]" : (STMT_VINFO_LIVE_P (stmt_info)
    3507       551497 :                                      ? "[l] " : ""),
    3508              :                          i, stmt_info->stmt);
    3509              :       else
    3510         6158 :         dump_printf_loc (metadata, user_loc, "\tstmt %u ---\n", i);
    3511              :   else
    3512              :     {
    3513       102613 :       dump_printf_loc (metadata, user_loc, "\t{ ");
    3514       226185 :       FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
    3515       123572 :         dump_printf (metadata, "%T%s ", op,
    3516       123572 :                      i < SLP_TREE_SCALAR_OPS (node).length () - 1 ? "," : "");
    3517       102613 :       dump_printf (metadata, "}\n");
    3518              :     }
    3519       492228 :   if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
    3520              :     {
    3521        70067 :       dump_printf_loc (metadata, user_loc, "\tload permutation {");
    3522       159332 :       FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), i, j)
    3523        89265 :         dump_printf (dump_kind, " %u", j);
    3524        70067 :       dump_printf (dump_kind, " }\n");
    3525              :     }
    3526       492228 :   if (SLP_TREE_LANE_PERMUTATION (node).exists ())
    3527              :     {
    3528        17334 :       dump_printf_loc (metadata, user_loc, "\tlane permutation {");
    3529        64890 :       for (i = 0; i < SLP_TREE_LANE_PERMUTATION (node).length (); ++i)
    3530        47556 :         dump_printf (dump_kind, " %u[%u]",
    3531        47556 :                      SLP_TREE_LANE_PERMUTATION (node)[i].first,
    3532        47556 :                      SLP_TREE_LANE_PERMUTATION (node)[i].second);
    3533        17334 :       dump_printf (dump_kind, " }%s\n",
    3534        17334 :                    node->ldst_lanes ? " (load-lanes)" : "");
    3535              :     }
    3536       492228 :   if (SLP_TREE_CHILDREN (node).is_empty ())
    3537       186583 :     return;
    3538       305645 :   dump_printf_loc (metadata, user_loc, "\tchildren");
    3539       806618 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    3540       500973 :     dump_printf (dump_kind, " %p", (void *)child);
    3541       305645 :   dump_printf (dump_kind, "%s\n",
    3542       305645 :                node->ldst_lanes && !SLP_TREE_LANE_PERMUTATION (node).exists ()
    3543              :                ? " (store-lanes)" : "");
    3544              : }
    3545              : 
    3546              : DEBUG_FUNCTION void
    3547            0 : debug (slp_tree node)
    3548              : {
    3549            0 :   debug_dump_context ctx;
    3550            0 :   vect_print_slp_tree (MSG_NOTE,
    3551            0 :                        dump_location_t::from_location_t (UNKNOWN_LOCATION),
    3552              :                        node);
    3553            0 : }
    3554              : 
    3555              : /* Recursive helper for the dot producer below.  */
    3556              : 
    3557              : static void
    3558            0 : dot_slp_tree (FILE *f, slp_tree node, hash_set<slp_tree> &visited)
    3559              : {
    3560            0 :   if (visited.add (node))
    3561              :     return;
    3562              : 
    3563            0 :   fprintf (f, "\"%p\" [label=\"", (void *)node);
    3564            0 :   vect_print_slp_tree (MSG_NOTE,
    3565            0 :                        dump_location_t::from_location_t (UNKNOWN_LOCATION),
    3566              :                        node);
    3567            0 :   fprintf (f, "\"];\n");
    3568              : 
    3569              : 
    3570            0 :   for (slp_tree child : SLP_TREE_CHILDREN (node))
    3571            0 :     fprintf (f, "\"%p\" -> \"%p\";", (void *)node, (void *)child);
    3572              : 
    3573            0 :   for (slp_tree child : SLP_TREE_CHILDREN (node))
    3574            0 :     if (child)
    3575            0 :       dot_slp_tree (f, child, visited);
    3576              : }
    3577              : 
    3578              : DEBUG_FUNCTION void
    3579            0 : dot_slp_tree (const char *fname, slp_tree node)
    3580              : {
    3581            0 :   FILE *f = fopen (fname, "w");
    3582            0 :   fprintf (f, "digraph {\n");
    3583            0 :   fflush (f);
    3584            0 :     {
    3585            0 :       debug_dump_context ctx (f);
    3586            0 :       hash_set<slp_tree> visited;
    3587            0 :       dot_slp_tree (f, node, visited);
    3588            0 :     }
    3589            0 :   fflush (f);
    3590            0 :   fprintf (f, "}\n");
    3591            0 :   fclose (f);
    3592            0 : }
    3593              : 
    3594              : DEBUG_FUNCTION void
    3595            0 : dot_slp_tree (const char *fname, const vec<slp_instance> &slp_instances)
    3596              : {
    3597            0 :   FILE *f = fopen (fname, "w");
    3598            0 :   fprintf (f, "digraph {\n");
    3599            0 :   fflush (f);
    3600            0 :     {
    3601            0 :       debug_dump_context ctx (f);
    3602            0 :       hash_set<slp_tree> visited;
    3603            0 :       for (auto inst : slp_instances)
    3604            0 :         dot_slp_tree (f, SLP_INSTANCE_TREE (inst), visited);
    3605            0 :     }
    3606            0 :   fflush (f);
    3607            0 :   fprintf (f, "}\n");
    3608            0 :   fclose (f);
    3609            0 : }
    3610              : 
    3611              : /* Dump a slp tree NODE using flags specified in DUMP_KIND.  */
    3612              : 
    3613              : static void
    3614       531443 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
    3615              :                       slp_tree node, hash_set<slp_tree> &visited)
    3616              : {
    3617       531443 :   unsigned i;
    3618       531443 :   slp_tree child;
    3619              : 
    3620       531443 :   if (visited.add (node))
    3621       531443 :     return;
    3622              : 
    3623       488397 :   vect_print_slp_tree (dump_kind, loc, node);
    3624              : 
    3625      1472640 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    3626       495846 :     if (child)
    3627       448779 :       vect_print_slp_graph (dump_kind, loc, child, visited);
    3628              : }
    3629              : 
    3630              : static void
    3631        50458 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
    3632              :                       slp_tree entry)
    3633              : {
    3634        50458 :   hash_set<slp_tree> visited;
    3635        50458 :   vect_print_slp_graph (dump_kind, loc, entry, visited);
    3636        50458 : }
    3637              : 
    3638              : DEBUG_FUNCTION void
    3639            0 : debug (slp_instance instance)
    3640              : {
    3641            0 :   debug_dump_context ctx;
    3642            0 :   vect_print_slp_graph (MSG_NOTE,
    3643            0 :                         dump_location_t::from_location_t (UNKNOWN_LOCATION),
    3644              :                         SLP_INSTANCE_TREE (instance));
    3645            0 : }
    3646              : 
    3647              : 
    3648              : /* Compute the set of scalar stmts participating in external nodes.  */
    3649              : 
    3650              : static void
    3651      1717758 : vect_slp_gather_extern_scalar_stmts (vec_info *vinfo, slp_tree node,
    3652              :                                      hash_set<slp_tree> &visited,
    3653              :                                      hash_set<stmt_vec_info> &estmts)
    3654              : {
    3655      1717758 :   if (visited.add (node))
    3656              :     return;
    3657              : 
    3658      1656321 :   if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
    3659              :     {
    3660              :       slp_tree child;
    3661              :       int i;
    3662      1948096 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    3663       995263 :         if (child)
    3664       995263 :           vect_slp_gather_extern_scalar_stmts (vinfo, child, visited, estmts);
    3665              :     }
    3666              :   else
    3667      3936870 :     for (tree def : SLP_TREE_SCALAR_OPS (node))
    3668              :       {
    3669      1828640 :         stmt_vec_info def_stmt = vinfo->lookup_def (def);
    3670      1828640 :         if (def_stmt)
    3671       392060 :           estmts.add (def_stmt);
    3672              :       }
    3673              : }
    3674              : 
    3675              : /* Mark the original scalar stmt coverage of the vector SLP graph of VINFO
    3676              :    with STMT_SLP_TYPE == pure_slp.  */
    3677              : 
    3678              : static void
    3679       248546 : vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
    3680              : {
    3681              :   /* Gather the scalar stmt leafs of the SLP graph to stop the below DFS
    3682              :      walk on.  */
    3683       248546 :   hash_set<stmt_vec_info> scalar_stmts_in_externs;
    3684       248546 :   hash_set<slp_tree> visited;
    3685      1468133 :   for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
    3686       722495 :     vect_slp_gather_extern_scalar_stmts (vinfo, SLP_INSTANCE_TREE (instance),
    3687              :                                          visited, scalar_stmts_in_externs);
    3688              : 
    3689              :   /* DFS walk scalar stmts to compute the vectorized coverage indicated
    3690              :      by STMT_SLP_TYPE (stmt) == pure_slp on the original scalar (non-pattern)
    3691              :      stmts.  */
    3692      1468133 :   for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
    3693              :     {
    3694       928500 :       for (auto stmt : SLP_INSTANCE_ROOT_STMTS (instance))
    3695        95623 :         if (!scalar_stmts_in_externs.contains (stmt))
    3696        94358 :           STMT_SLP_TYPE (stmt) = pure_slp;
    3697       722495 :       auto_vec<stmt_vec_info> worklist;
    3698      4070343 :       for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
    3699              :         {
    3700      1902858 :           stmt = vect_orig_stmt (stmt);
    3701      1902858 :           if (!scalar_stmts_in_externs.contains (stmt)
    3702      1902858 :               && STMT_SLP_TYPE (stmt) != pure_slp)
    3703              :             {
    3704      1885796 :               STMT_SLP_TYPE (stmt) = pure_slp;
    3705      1885796 :               worklist.safe_push (stmt);
    3706              :             }
    3707              :         }
    3708      3811532 :       while (!worklist.is_empty ())
    3709              :         {
    3710      2371462 :           stmt_vec_info stmt = worklist.pop ();
    3711              : 
    3712              :           /* Now walk relevant parts of the SSA use-def graph.  */
    3713      2371462 :           slp_oprnds child_ops (stmt);
    3714      7404880 :           for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
    3715              :             {
    3716      2661956 :               tree op = child_ops.get_op_for_slp_child (stmt, i);
    3717      2661956 :               stmt_vec_info def = vinfo->lookup_def (op);
    3718      2661956 :               if (def
    3719       963592 :                   && !scalar_stmts_in_externs.contains (def)
    3720      3244592 :                   && STMT_SLP_TYPE (def) != pure_slp)
    3721              :                 {
    3722       485666 :                   STMT_SLP_TYPE (def) = pure_slp;
    3723       485666 :                   worklist.safe_push (def);
    3724              :                 }
    3725              :             }
    3726              :         }
    3727       722495 :     }
    3728       248546 : }
    3729              : 
    3730              : /* Mark the statements of the tree rooted at NODE as relevant (vect_used).  */
    3731              : 
    3732              : static void
    3733      2604858 : vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
    3734              : {
    3735      2604858 :   if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
    3736              :     return;
    3737              : 
    3738      1567507 :   if (visited.add (node))
    3739              :     return;
    3740              : 
    3741      7323344 :   for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
    3742      3272436 :     if (stmt_info)
    3743              :       {
    3744      3272436 :         gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
    3745              :                     || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
    3746      3272436 :         STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
    3747              :       }
    3748              : 
    3749      5666978 :   for (auto child: SLP_TREE_CHILDREN (node))
    3750      1779176 :     if (child)
    3751      1779176 :       vect_mark_slp_stmts_relevant (child, visited);
    3752              : }
    3753              : 
    3754              : static void
    3755       825682 : vect_mark_slp_stmts_relevant (slp_tree node)
    3756              : {
    3757       825682 :   hash_set<slp_tree> visited;
    3758       825682 :   vect_mark_slp_stmts_relevant (node, visited);
    3759       825682 : }
    3760              : 
    3761              : 
    3762              : /* Gather loads in the SLP graph NODE and populate the INST loads array.  */
    3763              : 
    3764              : static void
    3765     11514987 : vect_gather_slp_loads (vec<slp_tree> &loads, slp_tree node,
    3766              :                        hash_set<slp_tree> &visited)
    3767              : {
    3768     11514987 :   if (!node || visited.add (node))
    3769              :     return;
    3770              : 
    3771      9617296 :   if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
    3772              :     return;
    3773              : 
    3774      7116632 :   if (!SLP_TREE_PERMUTE_P (node))
    3775              :     {
    3776      6891127 :       stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
    3777      6891127 :       if (STMT_VINFO_DATA_REF (stmt_info)
    3778      2930435 :           && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
    3779      1676655 :         loads.safe_push (node);
    3780              :     }
    3781              : 
    3782      7116632 :   unsigned i;
    3783      7116632 :   slp_tree child;
    3784     16241335 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    3785      9124703 :     vect_gather_slp_loads (loads, child, visited);
    3786              : }
    3787              : 
    3788              : 
    3789              : /* Find the last store in SLP INSTANCE.  */
    3790              : 
    3791              : stmt_vec_info
    3792      2563721 : vect_find_last_scalar_stmt_in_slp (slp_tree node)
    3793              : {
    3794      2563721 :   stmt_vec_info last = NULL;
    3795     14503297 :   for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
    3796      6812134 :     if (stmt_vinfo)
    3797              :       {
    3798      6812134 :         stmt_vinfo = vect_orig_stmt (stmt_vinfo);
    3799      6812134 :         last = last ? get_later_stmt (stmt_vinfo, last) : stmt_vinfo;
    3800              :       }
    3801              : 
    3802      2563721 :   return last;
    3803              : }
    3804              : 
    3805              : /* Find the first stmt in NODE.  */
    3806              : 
    3807              : stmt_vec_info
    3808       664862 : vect_find_first_scalar_stmt_in_slp (slp_tree node)
    3809              : {
    3810       664862 :   stmt_vec_info first = NULL;
    3811              : 
    3812      3595039 :   for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
    3813      1600453 :     if (stmt_vinfo)
    3814              :       {
    3815      1597861 :         stmt_vinfo = vect_orig_stmt (stmt_vinfo);
    3816      1597861 :         if (!first
    3817      1597861 :             || get_later_stmt (stmt_vinfo, first) == first)
    3818              :           first = stmt_vinfo;
    3819              :       }
    3820              : 
    3821       664862 :   return first;
    3822              : }
    3823              : 
    3824              : /* Splits a group of stores, currently beginning at FIRST_VINFO, into
    3825              :    two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
    3826              :    (also containing the first GROUP1_SIZE stmts, since stores are
    3827              :    consecutive), the second containing the remainder.
    3828              :    Return the first stmt in the second group.  */
    3829              : 
    3830              : static stmt_vec_info
    3831       137135 : vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
    3832              : {
    3833       137135 :   gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
    3834       137135 :   gcc_assert (group1_size > 0);
    3835       137135 :   int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
    3836       137135 :   gcc_assert (group2_size > 0);
    3837       137135 :   DR_GROUP_SIZE (first_vinfo) = group1_size;
    3838              : 
    3839       137135 :   stmt_vec_info stmt_info = first_vinfo;
    3840       459875 :   for (unsigned i = group1_size; i > 1; i--)
    3841              :     {
    3842       322740 :       stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
    3843       322740 :       gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
    3844              :     }
    3845              :   /* STMT is now the last element of the first group.  */
    3846       137135 :   stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
    3847       137135 :   DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
    3848              : 
    3849       137135 :   DR_GROUP_SIZE (group2) = group2_size;
    3850       407719 :   for (stmt_info = group2; stmt_info;
    3851       270584 :        stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
    3852              :     {
    3853       270584 :       DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
    3854       270584 :       gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
    3855              :     }
    3856              : 
    3857              :   /* For the second group, the DR_GROUP_GAP is that before the original group,
    3858              :      plus skipping over the first vector.  */
    3859       137135 :   DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
    3860              : 
    3861              :   /* DR_GROUP_GAP of the first group now has to skip over the second group too.  */
    3862       137135 :   DR_GROUP_GAP (first_vinfo) += group2_size;
    3863              : 
    3864       137135 :   if (dump_enabled_p ())
    3865           88 :     dump_printf_loc (MSG_NOTE, vect_location, "Split group into %d and %d\n",
    3866              :                      group1_size, group2_size);
    3867              : 
    3868       137135 :   return group2;
    3869              : }
    3870              : 
    3871              : /* Calculate the unrolling factor for an SLP instance with GROUP_SIZE
    3872              :    statements and a vector of NUNITS elements.  */
    3873              : 
    3874              : static poly_uint64
    3875      2973401 : calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
    3876              : {
    3877      2973401 :   return exact_div (common_multiple (nunits, group_size), group_size);
    3878              : }
    3879              : 
    3880              : /* Helper that checks to see if a node is a load node.  */
    3881              : 
    3882              : static inline bool
    3883          103 : vect_is_slp_load_node  (slp_tree root)
    3884              : {
    3885          103 :   return (!SLP_TREE_PERMUTE_P (root)
    3886          103 :           && SLP_TREE_DEF_TYPE (root) == vect_internal_def
    3887           97 :           && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_REPRESENTATIVE (root))
    3888          167 :           && DR_IS_READ (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (root))));
    3889              : }
    3890              : 
    3891              : 
    3892              : /* Helper function of optimize_load_redistribution that performs the operation
    3893              :    recursively.  */
    3894              : 
    3895              : static slp_tree
    3896        21801 : optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map,
    3897              :                                 vec_info *vinfo, unsigned int group_size,
    3898              :                                 hash_map<slp_tree, slp_tree> *load_map,
    3899              :                                 slp_tree root)
    3900              : {
    3901        21801 :   if (slp_tree *leader = load_map->get (root))
    3902         4549 :     return *leader;
    3903              : 
    3904        17252 :   slp_tree node;
    3905        17252 :   unsigned i;
    3906              : 
    3907              :   /* For now, we don't know anything about externals so do not do anything.  */
    3908        17252 :   if (!root || SLP_TREE_DEF_TYPE (root) != vect_internal_def)
    3909              :     return NULL;
    3910        12116 :   else if (SLP_TREE_PERMUTE_P (root))
    3911              :     {
    3912              :       /* First convert this node into a load node and add it to the leaves
    3913              :          list and flatten the permute from a lane to a load one.  If it's
    3914              :          unneeded it will be elided later.  */
    3915           71 :       vec<stmt_vec_info> stmts;
    3916           71 :       stmts.create (SLP_TREE_LANES (root));
    3917           71 :       lane_permutation_t lane_perm = SLP_TREE_LANE_PERMUTATION (root);
    3918          135 :       for (unsigned j = 0; j < lane_perm.length (); j++)
    3919              :         {
    3920          103 :           std::pair<unsigned, unsigned> perm = lane_perm[j];
    3921          103 :           node = SLP_TREE_CHILDREN (root)[perm.first];
    3922              : 
    3923          103 :           if (!vect_is_slp_load_node (node)
    3924          103 :               || SLP_TREE_CHILDREN (node).exists ())
    3925              :             {
    3926           39 :               stmts.release ();
    3927           39 :               goto next;
    3928              :             }
    3929              : 
    3930           64 :           stmts.quick_push (SLP_TREE_SCALAR_STMTS (node)[perm.second]);
    3931              :         }
    3932              : 
    3933           32 :       if (dump_enabled_p ())
    3934            0 :         dump_printf_loc (MSG_NOTE, vect_location,
    3935              :                          "converting stmts on permute node %p\n",
    3936              :                          (void *) root);
    3937              : 
    3938           32 :       match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    3939           32 :       unsigned tree_size = 0, limit = 1;
    3940           32 :       node = vect_build_slp_tree (vinfo, stmts,
    3941              :                                   matches, &limit, &tree_size, bst_map);
    3942           32 :       if (!node)
    3943            0 :         stmts.release ();
    3944              : 
    3945           32 :       load_map->put (root, node);
    3946           32 :       return node;
    3947              :     }
    3948              : 
    3949        12045 : next:
    3950        12084 :   load_map->put (root, NULL);
    3951              : 
    3952        27523 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
    3953              :     {
    3954        15439 :       slp_tree value
    3955        15439 :         = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
    3956              :                                           node);
    3957        15439 :       if (value)
    3958              :         {
    3959           32 :           SLP_TREE_REF_COUNT (value)++;
    3960           32 :           SLP_TREE_CHILDREN (root)[i] = value;
    3961              :           /* ???  We know the original leafs of the replaced nodes will
    3962              :              be referenced by bst_map, only the permutes created by
    3963              :              pattern matching are not.  */
    3964           32 :           if (SLP_TREE_REF_COUNT (node) == 1)
    3965           32 :             load_map->remove (node);
    3966           32 :           vect_free_slp_tree (node);
    3967              :         }
    3968              :     }
    3969              : 
    3970              :   return NULL;
    3971              : }
    3972              : 
    3973              : /* Temporary workaround for loads not being CSEd during SLP build.  This
    3974              :    function will traverse the SLP tree rooted in ROOT for INSTANCE and find
    3975              :    VEC_PERM nodes that blend vectors from multiple nodes that all read from the
    3976              :    same DR such that the final operation is equal to a permuted load.  Such
    3977              :    NODES are then directly converted into LOADS themselves.  The nodes are
    3978              :    CSEd using BST_MAP.  */
    3979              : 
    3980              : static void
    3981         4180 : optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map,
    3982              :                               vec_info *vinfo, unsigned int group_size,
    3983              :                               hash_map<slp_tree, slp_tree> *load_map,
    3984              :                               slp_tree root)
    3985              : {
    3986         4180 :   slp_tree node;
    3987         4180 :   unsigned i;
    3988              : 
    3989        10542 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
    3990              :     {
    3991         6362 :       slp_tree value
    3992         6362 :         = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
    3993              :                                           node);
    3994         6362 :       if (value)
    3995              :         {
    3996            0 :           SLP_TREE_REF_COUNT (value)++;
    3997            0 :           SLP_TREE_CHILDREN (root)[i] = value;
    3998              :           /* ???  We know the original leafs of the replaced nodes will
    3999              :              be referenced by bst_map, only the permutes created by
    4000              :              pattern matching are not.  */
    4001            0 :           if (SLP_TREE_REF_COUNT (node) == 1)
    4002            0 :             load_map->remove (node);
    4003            0 :           vect_free_slp_tree (node);
    4004              :         }
    4005              :     }
    4006         4180 : }
    4007              : 
    4008              : /* Helper function of vect_match_slp_patterns.
    4009              : 
    4010              :    Attempts to match patterns against the slp tree rooted in REF_NODE using
    4011              :    VINFO.  Patterns are matched in post-order traversal.
    4012              : 
    4013              :    If matching is successful the value in REF_NODE is updated and returned, if
    4014              :    not then it is returned unchanged.  */
    4015              : 
    4016              : static bool
    4017      6656351 : vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo,
    4018              :                            slp_tree_to_load_perm_map_t *perm_cache,
    4019              :                            slp_compat_nodes_map_t *compat_cache,
    4020              :                            hash_set<slp_tree> *visited)
    4021              : {
    4022      6656351 :   unsigned i;
    4023      6656351 :   slp_tree node = *ref_node;
    4024      6656351 :   bool found_p = false;
    4025      6656351 :   if (!node || visited->add (node))
    4026              :     return false;
    4027              : 
    4028              :   slp_tree child;
    4029     10695611 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    4030      4995970 :     found_p |= vect_match_slp_patterns_2 (&SLP_TREE_CHILDREN (node)[i],
    4031              :                                           vinfo, perm_cache, compat_cache,
    4032              :                                           visited);
    4033              : 
    4034     17098923 :   for (unsigned x = 0; x < num__slp_patterns; x++)
    4035              :     {
    4036     11399282 :       vect_pattern *pattern
    4037     11399282 :         = slp_patterns[x] (perm_cache, compat_cache, ref_node);
    4038     11399282 :       if (pattern)
    4039              :         {
    4040         1114 :           pattern->build (vinfo);
    4041         1114 :           delete pattern;
    4042              :           found_p = true;
    4043              :         }
    4044              :     }
    4045              : 
    4046              :   return found_p;
    4047              : }
    4048              : 
    4049              : /* Applies pattern matching to the given SLP tree rooted in REF_NODE using
    4050              :    vec_info VINFO.
    4051              : 
    4052              :    The modified tree is returned.  Patterns are tried in order and multiple
    4053              :    patterns may match.  */
    4054              : 
    4055              : static bool
    4056      1660381 : vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
    4057              :                          hash_set<slp_tree> *visited,
    4058              :                          slp_tree_to_load_perm_map_t *perm_cache,
    4059              :                          slp_compat_nodes_map_t *compat_cache)
    4060              : {
    4061      1660381 :   DUMP_VECT_SCOPE ("vect_match_slp_patterns");
    4062      1660381 :   slp_tree *ref_node = &SLP_INSTANCE_TREE (instance);
    4063              : 
    4064      1660381 :   if (dump_enabled_p ())
    4065        33590 :     dump_printf_loc (MSG_NOTE, vect_location,
    4066              :                      "Analyzing SLP tree %p for patterns\n",
    4067        33590 :                      (void *) SLP_INSTANCE_TREE (instance));
    4068              : 
    4069      1660381 :   return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, compat_cache,
    4070      1660381 :                                     visited);
    4071              : }
    4072              : 
    4073              : /* STMT_INFO is a store group of size GROUP_SIZE that we are considering
    4074              :    vectorizing with VECTYPE that might be NULL.  MASKED_P indicates whether
    4075              :    the stores are masked.
    4076              :    Return true if we could use IFN_STORE_LANES instead and if that appears
    4077              :    to be the better approach.  */
    4078              : 
    4079              : static bool
    4080         6147 : vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
    4081              :                                tree vectype, bool masked_p,
    4082              :                                unsigned int group_size,
    4083              :                                unsigned int new_group_size)
    4084              : {
    4085         6147 :   if (!vectype)
    4086              :     {
    4087         6147 :       tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
    4088         6147 :       vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
    4089              :     }
    4090         6147 :   if (!vectype)
    4091              :     return false;
    4092              :   /* Allow the split if one of the two new groups would operate on full
    4093              :      vectors *within* rather than across one scalar loop iteration.
    4094              :      This is purely a heuristic, but it should work well for group
    4095              :      sizes of 3 and 4, where the possible splits are:
    4096              : 
    4097              :        3->2+1:  OK if the vector has exactly two elements
    4098              :        4->2+2:  Likewise
    4099              :        4->3+1:  Less clear-cut.  */
    4100         6147 :   if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
    4101         6147 :       || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
    4102              :     return false;
    4103         3500 :   return vect_store_lanes_supported (vectype, group_size, masked_p) != IFN_LAST;
    4104              : }
    4105              : 
    4106              : /* Analyze an SLP instance starting from a group of grouped stores.  Call
    4107              :    vect_build_slp_tree to build a tree of packed stmts if possible.
    4108              :    Return FALSE if it's impossible to SLP any stmt in the loop.  */
    4109              : 
    4110              : static bool
    4111              : vect_analyze_slp_instance (vec_info *vinfo,
    4112              :                            scalar_stmts_to_slp_tree_map_t *bst_map,
    4113              :                            stmt_vec_info stmt_info, slp_instance_kind kind,
    4114              :                            unsigned max_tree_size, unsigned *limit,
    4115              :                            bool force_single_lane);
    4116              : 
    4117              : /* Build an interleaving scheme for the store sources RHS_NODES from
    4118              :    SCALAR_STMTS.  */
    4119              : 
    4120              : static slp_tree
    4121        11853 : vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
    4122              :                                    vec<stmt_vec_info> &scalar_stmts)
    4123              : {
    4124        11853 :   unsigned int group_size = scalar_stmts.length ();
    4125        23706 :   slp_tree node = vect_create_new_slp_node (scalar_stmts,
    4126        11853 :                                             SLP_TREE_CHILDREN
    4127              :                                               (rhs_nodes[0]).length ());
    4128        11853 :   SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
    4129        11853 :   for (unsigned l = 0;
    4130        23733 :        l < SLP_TREE_CHILDREN (rhs_nodes[0]).length (); ++l)
    4131              :     {
    4132              :       /* And a permute merging all RHS SLP trees.  */
    4133        11880 :       slp_tree perm = vect_create_new_slp_node (rhs_nodes.length (),
    4134        11880 :                                                 VEC_PERM_EXPR);
    4135        11880 :       SLP_TREE_CHILDREN (node).quick_push (perm);
    4136        11880 :       SLP_TREE_LANE_PERMUTATION (perm).create (group_size);
    4137        11880 :       SLP_TREE_VECTYPE (perm) = SLP_TREE_VECTYPE (node);
    4138        11880 :       SLP_TREE_LANES (perm) = group_size;
    4139        11880 :       SLP_TREE_REPRESENTATIVE (perm) = NULL;
    4140        86356 :       for (unsigned j = 0; j < rhs_nodes.length (); ++j)
    4141              :         {
    4142        31298 :           SLP_TREE_CHILDREN (perm)
    4143        31298 :             .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
    4144        31298 :           SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
    4145        31298 :           for (unsigned k = 0;
    4146        64999 :                k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
    4147              :             {
    4148              :               /* ???  We should populate SLP_TREE_SCALAR_STMTS
    4149              :                  or SLP_TREE_SCALAR_OPS but then we might have
    4150              :                  a mix of both in our children.  */
    4151        33701 :               SLP_TREE_LANE_PERMUTATION (perm)
    4152        33701 :                 .quick_push (std::make_pair (j, k));
    4153              :             }
    4154              :         }
    4155              : 
    4156              :       /* Now we have a single permute node but we cannot code-generate
    4157              :          the case with more than two inputs.
    4158              :          Perform pairwise reduction, reducing the two inputs
    4159              :          with the least number of lanes to one and then repeat until
    4160              :          we end up with two inputs.  That scheme makes sure we end
    4161              :          up with permutes satisfying the restriction of requiring at
    4162              :          most two vector inputs to produce a single vector output
    4163              :          when the number of lanes is even.  */
    4164        19418 :       while (SLP_TREE_CHILDREN (perm).length () > 2)
    4165              :         {
    4166              :           /* When we have three equal sized groups left the pairwise
    4167              :              reduction does not result in a scheme that avoids using
    4168              :              three vectors.  Instead merge the first two groups
    4169              :              to the final size with do-not-care elements (chosen
    4170              :              from the first group) and then merge with the third.
    4171              :                   { A0, B0,  x, A1, B1,  x, ... }
    4172              :                -> { A0, B0, C0, A1, B1, C1, ... }
    4173              :              This handles group size of three (and at least
    4174              :              power-of-two multiples of that).  */
    4175         7538 :           if (SLP_TREE_CHILDREN (perm).length () == 3
    4176         3376 :               && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
    4177         3376 :                   == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[1]))
    4178         7538 :               && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
    4179         2489 :                   == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[2])))
    4180              :             {
    4181         2183 :               int ai = 0;
    4182         2183 :               int bi = 1;
    4183         2183 :               slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
    4184         2183 :               slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
    4185         2183 :               unsigned n = SLP_TREE_LANES (perm);
    4186              : 
    4187         2183 :               slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
    4188         2183 :               SLP_TREE_LANES (permab) = n;
    4189         2183 :               SLP_TREE_LANE_PERMUTATION (permab).create (n);
    4190         2183 :               SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
    4191              :               /* ???  Should be NULL but that's not expected.  */
    4192         2183 :               SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
    4193         2183 :               SLP_TREE_CHILDREN (permab).quick_push (a);
    4194         6563 :               for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
    4195         2197 :                 SLP_TREE_LANE_PERMUTATION (permab)
    4196         2197 :                   .quick_push (std::make_pair (0, k));
    4197         2183 :               SLP_TREE_CHILDREN (permab).quick_push (b);
    4198         6563 :               for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
    4199         2197 :                 SLP_TREE_LANE_PERMUTATION (permab)
    4200         2197 :                   .quick_push (std::make_pair (1, k));
    4201              :               /* Push the do-not-care lanes.  */
    4202         4380 :               for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
    4203         2197 :                 SLP_TREE_LANE_PERMUTATION (permab)
    4204         2197 :                   .quick_push (std::make_pair (0, k));
    4205              : 
    4206              :               /* Put the merged node into 'perm', in place of a.  */
    4207         2183 :               SLP_TREE_CHILDREN (perm)[ai] = permab;
    4208              :               /* Adjust the references to b in the permutation
    4209              :                  of perm and to the later children which we'll
    4210              :                  remove.  */
    4211         8774 :               for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
    4212              :                 {
    4213         6591 :                   std::pair<unsigned, unsigned> &p
    4214         6591 :                     = SLP_TREE_LANE_PERMUTATION (perm)[k];
    4215         6591 :                   if (p.first == (unsigned) bi)
    4216              :                     {
    4217         2197 :                       p.first = ai;
    4218         2197 :                       p.second += SLP_TREE_LANES (a);
    4219              :                     }
    4220         4394 :                   else if (p.first > (unsigned) bi)
    4221         2197 :                     p.first--;
    4222              :                 }
    4223         2183 :               SLP_TREE_CHILDREN (perm).ordered_remove (bi);
    4224         2183 :               break;
    4225              :             }
    4226              : 
    4227              :           /* Pick the two nodes with the least number of lanes,
    4228              :              prefer the earliest candidate and maintain ai < bi.  */
    4229              :           int ai = -1;
    4230              :           int bi = -1;
    4231        50500 :           for (unsigned ci = 0; ci < SLP_TREE_CHILDREN (perm).length (); ++ci)
    4232              :             {
    4233        45145 :               if (ai == -1)
    4234         5355 :                 ai = ci;
    4235        39790 :               else if (bi == -1)
    4236         5355 :                 bi = ci;
    4237        34435 :               else if ((SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
    4238        34435 :                         < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai]))
    4239        34435 :                        || (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
    4240        28605 :                            < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi])))
    4241              :                 {
    4242        12550 :                   if (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai])
    4243         6275 :                       <= SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi]))
    4244         2892 :                     bi = ci;
    4245              :                   else
    4246              :                     {
    4247         3383 :                       ai = bi;
    4248         3383 :                       bi = ci;
    4249              :                     }
    4250              :                 }
    4251              :             }
    4252              : 
    4253              :           /* Produce a merge of nodes ai and bi.  */
    4254         5355 :           slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
    4255         5355 :           slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
    4256         5355 :           unsigned n = SLP_TREE_LANES (a) + SLP_TREE_LANES (b);
    4257         5355 :           slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
    4258         5355 :           SLP_TREE_LANES (permab) = n;
    4259         5355 :           SLP_TREE_LANE_PERMUTATION (permab).create (n);
    4260         5355 :           SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
    4261              :           /* ???  Should be NULL but that's not expected.  */
    4262         5355 :           SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
    4263         5355 :           SLP_TREE_CHILDREN (permab).quick_push (a);
    4264        19589 :           for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
    4265         8879 :             SLP_TREE_LANE_PERMUTATION (permab)
    4266         8879 :               .quick_push (std::make_pair (0, k));
    4267         5355 :           SLP_TREE_CHILDREN (permab).quick_push (b);
    4268        18865 :           for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
    4269         8155 :             SLP_TREE_LANE_PERMUTATION (permab)
    4270         8155 :               .quick_push (std::make_pair (1, k));
    4271              : 
    4272              :           /* Put the merged node into 'perm', in place of a.  */
    4273         5355 :           SLP_TREE_CHILDREN (perm)[ai] = permab;
    4274              :           /* Adjust the references to b in the permutation
    4275              :              of perm and to the later children which we'll
    4276              :              remove.  */
    4277        81325 :           for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
    4278              :             {
    4279        75970 :               std::pair<unsigned, unsigned> &p
    4280        75970 :                 = SLP_TREE_LANE_PERMUTATION (perm)[k];
    4281        75970 :               if (p.first == (unsigned) bi)
    4282              :                 {
    4283         8155 :                   p.first = ai;
    4284         8155 :                   p.second += SLP_TREE_LANES (a);
    4285              :                 }
    4286        67815 :               else if (p.first > (unsigned) bi)
    4287        28698 :                 p.first--;
    4288              :             }
    4289         5355 :           SLP_TREE_CHILDREN (perm).ordered_remove (bi);
    4290              :         }
    4291              :     }
    4292              : 
    4293        11853 :   return node;
    4294              : }
    4295              : 
    4296              : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
    4297              :    of KIND.  Return true if successful.  SCALAR_STMTS is owned by this
    4298              :    function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to
    4299              :    the caller upon failure.  */
    4300              : 
    4301              : static bool
    4302      2140649 : vect_build_slp_instance (vec_info *vinfo,
    4303              :                          slp_instance_kind kind,
    4304              :                          vec<stmt_vec_info> &scalar_stmts,
    4305              :                          vec<stmt_vec_info> &root_stmt_infos,
    4306              :                          vec<tree> &remain,
    4307              :                          unsigned max_tree_size, unsigned *limit,
    4308              :                          scalar_stmts_to_slp_tree_map_t *bst_map,
    4309              :                          bool force_single_lane)
    4310              : {
    4311              :   /* If there's no budget left bail out early.  */
    4312      2140649 :   if (*limit == 0)
    4313              :     {
    4314        16098 :       scalar_stmts.release ();
    4315        16098 :       return false;
    4316              :     }
    4317              : 
    4318      2124551 :   if (kind == slp_inst_kind_ctor)
    4319              :     {
    4320        15570 :       if (dump_enabled_p ())
    4321          102 :         dump_printf_loc (MSG_NOTE, vect_location,
    4322              :                          "Analyzing vectorizable constructor: %G\n",
    4323           51 :                          root_stmt_infos[0]->stmt);
    4324              :     }
    4325      2108981 :   else if (kind == slp_inst_kind_gcond)
    4326              :     {
    4327       305120 :       if (dump_enabled_p ())
    4328         6730 :         dump_printf_loc (MSG_NOTE, vect_location,
    4329              :                          "Analyzing vectorizable control flow: %G",
    4330         3365 :                          root_stmt_infos[0]->stmt);
    4331              :     }
    4332      1803861 :   else if (kind == slp_inst_kind_bb_reduc)
    4333              :     {
    4334      1399712 :       if (dump_enabled_p ())
    4335         7538 :         dump_printf_loc (MSG_NOTE, vect_location,
    4336              :                          "Analyzing vectorizable BB reduction: %G",
    4337         3769 :                          root_stmt_infos[0]->stmt);
    4338              :     }
    4339              : 
    4340      2124551 :   if (dump_enabled_p ())
    4341              :     {
    4342        28546 :       dump_printf_loc (MSG_NOTE, vect_location,
    4343              :                        "Starting SLP discovery for\n");
    4344        61577 :       for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    4345        66062 :         dump_printf_loc (MSG_NOTE, vect_location,
    4346        33031 :                          "  %G", scalar_stmts[i]->stmt);
    4347              :     }
    4348              : 
    4349              :   /* Build the tree for the SLP instance.  */
    4350      2124551 :   unsigned int group_size = scalar_stmts.length ();
    4351      2124551 :   match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    4352      2124551 :   unsigned tree_size = 0;
    4353              : 
    4354      2124551 :   slp_tree node = NULL;
    4355      2124551 :   if (group_size > 1 && force_single_lane)
    4356              :     {
    4357            0 :       matches[0] = 0;
    4358            0 :       matches[1] = -1;
    4359              :     }
    4360              :   else
    4361      2124551 :     node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
    4362              :                                 &tree_size, bst_map);
    4363      2124551 :   if (node != NULL)
    4364              :     {
    4365              :       /* Create a new SLP instance.  */
    4366       851298 :       slp_instance new_instance = XNEW (class _slp_instance);
    4367       851298 :       SLP_INSTANCE_TREE (new_instance) = node;
    4368       851298 :       SLP_INSTANCE_LOADS (new_instance) = vNULL;
    4369       851298 :       SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
    4370       851298 :       SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
    4371       851298 :       SLP_INSTANCE_KIND (new_instance) = kind;
    4372       851298 :       new_instance->reduc_phis = NULL;
    4373       851298 :       new_instance->cost_vec = vNULL;
    4374       851298 :       new_instance->subgraph_entries = vNULL;
    4375              : 
    4376       851298 :       if (dump_enabled_p ())
    4377        25203 :         dump_printf_loc (MSG_NOTE, vect_location,
    4378              :                          "SLP size %u vs. limit %u.\n",
    4379              :                          tree_size, max_tree_size);
    4380              : 
    4381       851298 :       vinfo->slp_instances.safe_push (new_instance);
    4382              : 
    4383              :       /* ???  We've replaced the old SLP_INSTANCE_GROUP_SIZE with
    4384              :          the number of SLP lanes of the root in a few places.
    4385              :          Verify that assumption holds.  */
    4386       851298 :       gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
    4387              :                   == group_size);
    4388              : 
    4389       851298 :       if (dump_enabled_p ())
    4390              :         {
    4391        25203 :           if (kind == slp_inst_kind_reduc_group)
    4392         1779 :             dump_printf_loc (MSG_NOTE, vect_location,
    4393              :                              "SLP discovery of size %d reduction group "
    4394              :                              "succeeded\n", group_size);
    4395        25203 :           dump_printf_loc (MSG_NOTE, vect_location,
    4396              :                            "Final SLP tree for instance %p:\n",
    4397              :                            (void *) new_instance);
    4398        25203 :           vect_print_slp_graph (MSG_NOTE, vect_location,
    4399              :                                 SLP_INSTANCE_TREE (new_instance));
    4400              :         }
    4401              : 
    4402       851298 :       return true;
    4403              :     }
    4404              :   /* Failed to SLP.  */
    4405              : 
    4406              :   /* While we arrive here even with slp_inst_kind_store we should only
    4407              :      for group_size == 1.  The code to split store groups is only in
    4408              :      vect_analyze_slp_instance now.  */
    4409      1273253 :   gcc_assert (kind != slp_inst_kind_store || group_size == 1);
    4410              : 
    4411              :   /* For BB vectorization we get failures only in case of the need of
    4412              :      unrolling, as otherwise we'll simply get operands built from scalars.
    4413              :      Iff there is any mismatches in the toplevel stmts those will prevail,
    4414              :      otherwise we get the non-power-of-two tail of the lanes failed.
    4415              :      For BB reductions we mainly want to catch the first case so we pick
    4416              :      a more useful subset of lanes to reduce.  Pick the largest matching
    4417              :      subset of that covers half of the group or more.  */
    4418      1273253 :   if (kind == slp_inst_kind_bb_reduc && matches[0] != -1)
    4419              :     {
    4420       680321 :       unsigned *n_matching = XALLOCAVEC (unsigned, group_size);
    4421       680321 :       memset (n_matching, 0, sizeof (unsigned) * group_size);
    4422      2201933 :       for (unsigned i = 0; i < group_size; ++i)
    4423      1521612 :         if (matches[i] != -2 && matches[i] != -1)
    4424      1282647 :           n_matching[matches[i].v]++;
    4425              :       unsigned largest_i = 0;
    4426      1521612 :       for (unsigned i = 1; i < group_size; ++i)
    4427       841291 :         if (n_matching[i] > n_matching[largest_i])
    4428        64017 :           largest_i = i;
    4429              :       /* Pick the largest matching part and put the rest to remain.  */
    4430       680321 :       if (n_matching[largest_i] >= 2
    4431        51159 :           && n_matching[largest_i] >= group_size / 2)
    4432              :         {
    4433              :           /* As we know the matches[] stmts match up, recursing for
    4434              :              non-power-of-two sizes will just force-fail the tail
    4435              :              for us at hopefully optimal vector size and succesfully
    4436              :              finish discovery.  */
    4437        49797 :           vec<stmt_vec_info> scalar_stmts2;
    4438        49797 :           scalar_stmts2.create (n_matching[largest_i]);
    4439       294775 :           for (unsigned i = 0; i < group_size; ++i)
    4440       195181 :             if (matches[i] == (int)largest_i)
    4441       133058 :               scalar_stmts2.quick_push (scalar_stmts[i]);
    4442              :             else
    4443        62123 :               remain.safe_push
    4444        66106 :                 (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
    4445        49797 :           if (dump_enabled_p ())
    4446          228 :             dump_printf_loc (MSG_NOTE, vect_location, "Splitting %d "
    4447              :                              "non-matching lanes to scalar remains\n",
    4448          228 :                              scalar_stmts.length () - scalar_stmts2.length ());
    4449        49797 :           scalar_stmts.release ();
    4450        49797 :           return vect_build_slp_instance (vinfo, kind, scalar_stmts2,
    4451              :                                           root_stmt_infos, remain,
    4452              :                                           max_tree_size, limit, bst_map,
    4453              :                                           force_single_lane);
    4454              :         }
    4455              :     }
    4456              : 
    4457              :   /* Free the allocated memory.  */
    4458      1223456 :   scalar_stmts.release ();
    4459              : 
    4460              :   /* Failed to SLP.  */
    4461      1223456 :   if (dump_enabled_p ())
    4462         3229 :     dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
    4463              :   return false;
    4464              : }
    4465              : 
    4466              : /* Analyze an SLP instance starting from a the start of a reduction chain.
    4467              :    Call vect_build_slp_tree to build a tree of packed stmts if possible.
    4468              :    Return FALSE if SLP build fails.  */
    4469              : 
    4470              : static bool
    4471        70486 : vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
    4472              :                               scalar_stmts_to_slp_tree_map_t *bst_map,
    4473              :                               stmt_vec_info scalar_stmt,
    4474              :                               unsigned max_tree_size, unsigned *limit)
    4475              : {
    4476        70486 :   vec<stmt_vec_info> scalar_stmts = vNULL;
    4477              : 
    4478        70486 :   bool fail = false;
    4479              :   /* ???  We could leave operation code checking to SLP discovery.  */
    4480        70486 :   code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF
    4481              :                                               (vect_orig_stmt (scalar_stmt)));
    4482        70486 :   bool first = true;
    4483        70486 :   stmt_vec_info next_stmt = scalar_stmt;
    4484        78937 :   do
    4485              :     {
    4486        78937 :       stmt_vec_info stmt = next_stmt;
    4487        78937 :       gimple_match_op op, orig_op;
    4488        78937 :       if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op))
    4489            0 :         gcc_unreachable ();
    4490       157874 :       tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt),
    4491        78937 :                                    STMT_VINFO_REDUC_IDX (stmt));
    4492        78937 :       next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def));
    4493        78937 :       gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt))
    4494              :                   || STMT_VINFO_REDUC_IDX (next_stmt) != -1);
    4495        84898 :       if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)),
    4496              :                               &orig_op))
    4497            0 :         gcc_unreachable ();
    4498        78937 :       if (CONVERT_EXPR_CODE_P (op.code)
    4499         4999 :           && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0]))
    4500        83924 :           && (first
    4501         2498 :               || is_a <gphi *> (STMT_VINFO_STMT (next_stmt))))
    4502              :         ;
    4503        74010 :       else if (code != orig_op.code)
    4504              :         {
    4505          907 :           fail = true;
    4506          907 :           break;
    4507              :         }
    4508              :       else
    4509        73103 :         scalar_stmts.safe_push (stmt);
    4510        78030 :       first = false;
    4511              :     }
    4512        78030 :   while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt)));
    4513        70486 :   if (fail)
    4514          907 :     return false;
    4515              : 
    4516              :   /* When the SSA def chain through reduc-idx does not form a natural
    4517              :      reduction chain try to linearize an associative operation manually.  */
    4518        69579 :   if (scalar_stmts.length () == 1
    4519        68665 :       && code.is_tree_code ()
    4520        62593 :       && associative_tree_code ((tree_code)code)
    4521              :       /* We may not associate if a fold-left reduction is required.  */
    4522       130751 :       && !needs_fold_left_reduction_p (TREE_TYPE (gimple_get_lhs
    4523              :                                                     (scalar_stmts[0]->stmt)),
    4524              :                                        code))
    4525              :     {
    4526              :       /* Remember a stmt with the actual reduction operation.  */
    4527        57695 :       stmt_vec_info reduc_scalar_stmt = scalar_stmts[0];
    4528        57695 :       auto_vec<chain_op_t> chain;
    4529        57695 :       auto_vec<std::pair<tree_code, gimple *> > worklist;
    4530        57695 :       gimple *op_stmt = NULL, *other_op_stmt = NULL;
    4531        57695 :       if (is_a <gassign *> (scalar_stmts[0]->stmt)
    4532              :           /* We cannot linearize an operation that vect_slp_linearize_chain
    4533              :              would not put on its worklist.  */
    4534        57695 :           && gimple_assign_rhs_code (scalar_stmts[0]->stmt) == (tree_code)code)
    4535              :         {
    4536        57039 :           vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
    4537        57039 :                                     scalar_stmts[0]->stmt, op_stmt,
    4538              :                                     other_op_stmt,
    4539              :                                     NULL);
    4540              : 
    4541        57039 :           scalar_stmts.truncate (0);
    4542        57039 :           stmt_vec_info tail = NULL;
    4543       285493 :           for (auto el : chain)
    4544              :             {
    4545       114675 :               if (el.dt == vect_external_def
    4546       114675 :                   || el.dt == vect_constant_def
    4547       114675 :                   || el.code != (tree_code) code)
    4548              :                 {
    4549          299 :                   scalar_stmts.release ();
    4550          299 :                   return false;
    4551              :                 }
    4552       114376 :               stmt_vec_info stmt = vinfo->lookup_def (el.op);
    4553       114376 :               if (STMT_VINFO_REDUC_IDX (stmt) != -1
    4554       112090 :                   || STMT_VINFO_REDUC_DEF (stmt))
    4555              :                 {
    4556        56943 :                   gcc_assert (tail == NULL);
    4557        56943 :                   tail = stmt;
    4558        56943 :                   continue;
    4559              :                 }
    4560        57433 :               scalar_stmts.safe_push (stmt);
    4561              :             }
    4562        56740 :           gcc_assert (tail);
    4563              :         }
    4564              : 
    4565              :       /* When this linearization didn't produce a chain see if stripping
    4566              :          a wrapping sign conversion produces one.  */
    4567        57396 :       if (scalar_stmts.length () == 1
    4568        57396 :           && (code == PLUS_EXPR || code == MULT_EXPR || code == BIT_IOR_EXPR
    4569              :               || code == BIT_AND_EXPR || code == BIT_XOR_EXPR))
    4570              :         {
    4571        55595 :           gimple *stmt = scalar_stmts[0]->stmt;
    4572        55595 :           if (!is_gimple_assign (stmt)
    4573        54435 :               || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
    4574         4772 :               || TREE_CODE (gimple_assign_rhs1 (stmt)) != SSA_NAME
    4575        60367 :               || !tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
    4576         4772 :                                          TREE_TYPE (gimple_assign_rhs1 (stmt))))
    4577              :             {
    4578        53815 :               scalar_stmts.release ();
    4579        53815 :               return false;
    4580              :             }
    4581         1780 :           stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
    4582         1780 :           if (!is_gimple_assign (stmt)
    4583         1780 :               || gimple_assign_rhs_code (stmt) != (tree_code)code)
    4584              :             {
    4585         1762 :               scalar_stmts.release ();
    4586         1762 :               return false;
    4587              :             }
    4588           18 :           chain.truncate (0);
    4589           18 :           vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
    4590              :                                     stmt, op_stmt, other_op_stmt, NULL);
    4591              : 
    4592           18 :           scalar_stmts.truncate (0);
    4593           18 :           stmt_vec_info tail = NULL;
    4594           88 :           for (auto el : chain)
    4595              :             {
    4596           42 :               if (el.dt == vect_external_def
    4597           42 :                   || el.dt == vect_constant_def
    4598           42 :                   || el.code != (tree_code) code)
    4599              :                 {
    4600            8 :                   scalar_stmts.release ();
    4601            8 :                   return false;
    4602              :                 }
    4603           34 :               stmt_vec_info stmt = vinfo->lookup_def (el.op);
    4604           34 :               if (STMT_VINFO_REDUC_IDX (stmt) != -1
    4605           34 :                   || STMT_VINFO_REDUC_DEF (stmt))
    4606              :                 {
    4607            0 :                   gcc_assert (tail == NULL);
    4608            0 :                   tail = stmt;
    4609            0 :                   continue;
    4610              :                 }
    4611           34 :               scalar_stmts.safe_push (stmt);
    4612              :             }
    4613              :           /* Unlike the above this does not include the reduction SSA
    4614              :              cycle.  */
    4615           10 :           gcc_assert (!tail);
    4616              :         }
    4617              : 
    4618         1811 :       if (scalar_stmts.length () < 2)
    4619              :         {
    4620         1678 :           scalar_stmts.release ();
    4621         1678 :           return false;
    4622              :         }
    4623              : 
    4624          133 :       if (dump_enabled_p ())
    4625              :         {
    4626           43 :           dump_printf_loc (MSG_NOTE, vect_location,
    4627              :                            "Starting SLP discovery of reduction chain for\n");
    4628          173 :           for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    4629          260 :             dump_printf_loc (MSG_NOTE, vect_location,
    4630          130 :                              "  %G", scalar_stmts[i]->stmt);
    4631              :         }
    4632              : 
    4633          133 :       unsigned int group_size = scalar_stmts.length ();
    4634          133 :       match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    4635          133 :       unsigned tree_size = 0;
    4636          133 :       slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
    4637          133 :                                            &tree_size, bst_map);
    4638          133 :       if (!node)
    4639              :         {
    4640           46 :           scalar_stmts.release ();
    4641           46 :           return false;
    4642              :         }
    4643              : 
    4644           87 :       unsigned cycle_id = vinfo->reduc_infos.length ();
    4645           87 :       vect_reduc_info reduc_info = new vect_reduc_info_s ();
    4646           87 :       vinfo->reduc_infos.safe_push (reduc_info);
    4647           87 :       VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (next_stmt);
    4648           87 :       VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (next_stmt);
    4649           87 :       VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (next_stmt);
    4650           87 :       VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
    4651           87 :       reduc_info->is_reduc_chain = true;
    4652              : 
    4653              :       /* Build the node for the PHI and possibly the conversions.  */
    4654           87 :       slp_tree phis = vect_create_new_slp_node (2, ERROR_MARK);
    4655           87 :       SLP_TREE_REPRESENTATIVE (phis) = next_stmt;
    4656           87 :       phis->cycle_info.id = cycle_id;
    4657           87 :       SLP_TREE_LANES (phis) = group_size;
    4658           87 :       if (reduc_scalar_stmt == scalar_stmt)
    4659           83 :         SLP_TREE_VECTYPE (phis) = SLP_TREE_VECTYPE (node);
    4660              :       else
    4661            4 :         SLP_TREE_VECTYPE (phis)
    4662            4 :           = signed_or_unsigned_type_for (TYPE_UNSIGNED
    4663              :                                            (TREE_TYPE (gimple_get_lhs
    4664              :                                                          (scalar_stmt->stmt))),
    4665              :                                          SLP_TREE_VECTYPE (node));
    4666              :       /* ???  vect_cse_slp_nodes cannot cope with cycles without any
    4667              :          SLP_TREE_SCALAR_STMTS.  */
    4668           87 :       SLP_TREE_SCALAR_STMTS (phis).create (group_size);
    4669          513 :       for (unsigned i = 0; i < group_size; ++i)
    4670          339 :         SLP_TREE_SCALAR_STMTS (phis).quick_push (next_stmt);
    4671              : 
    4672           87 :       slp_tree op_input = phis;
    4673           87 :       if (reduc_scalar_stmt != scalar_stmt)
    4674              :         {
    4675            4 :           slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
    4676            4 :           SLP_TREE_REPRESENTATIVE (conv)
    4677            4 :             = vinfo->lookup_def (gimple_arg (reduc_scalar_stmt->stmt,
    4678            4 :                                              STMT_VINFO_REDUC_IDX
    4679              :                                                (reduc_scalar_stmt)));
    4680            4 :           SLP_TREE_CHILDREN (conv).quick_push (phis);
    4681            4 :           conv->cycle_info.id = cycle_id;
    4682            4 :           SLP_TREE_REDUC_IDX (conv) = 0;
    4683            4 :           SLP_TREE_LANES (conv) = group_size;
    4684            4 :           SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (node);
    4685            4 :           SLP_TREE_SCALAR_STMTS (conv) = vNULL;
    4686            4 :           op_input = conv;
    4687              :         }
    4688              : 
    4689           87 :       slp_tree reduc = vect_create_new_slp_node (2, ERROR_MARK);
    4690           87 :       SLP_TREE_REPRESENTATIVE (reduc) = reduc_scalar_stmt;
    4691           87 :       SLP_TREE_CHILDREN (reduc).quick_push (op_input);
    4692           87 :       SLP_TREE_CHILDREN (reduc).quick_push (node);
    4693           87 :       reduc->cycle_info.id = cycle_id;
    4694           87 :       SLP_TREE_REDUC_IDX (reduc) = 0;
    4695           87 :       SLP_TREE_LANES (reduc) = group_size;
    4696           87 :       SLP_TREE_VECTYPE (reduc) = SLP_TREE_VECTYPE (node);
    4697              :       /* ???  For the reduction epilogue we need a live lane.  */
    4698           87 :       SLP_TREE_SCALAR_STMTS (reduc).create (group_size);
    4699           87 :       SLP_TREE_SCALAR_STMTS (reduc).quick_push (reduc_scalar_stmt);
    4700          426 :       for (unsigned i = 1; i < group_size; ++i)
    4701          252 :         SLP_TREE_SCALAR_STMTS (reduc).quick_push (NULL);
    4702              : 
    4703           87 :       if (reduc_scalar_stmt != scalar_stmt)
    4704              :         {
    4705            4 :           slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
    4706            4 :           SLP_TREE_REPRESENTATIVE (conv) = scalar_stmt;
    4707            4 :           SLP_TREE_CHILDREN (conv).quick_push (reduc);
    4708            4 :           conv->cycle_info.id = cycle_id;
    4709            4 :           SLP_TREE_REDUC_IDX (conv) = 0;
    4710            4 :           SLP_TREE_LANES (conv) = group_size;
    4711            4 :           SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (phis);
    4712              :           /* ???  For the reduction epilogue we need a live lane.  */
    4713            4 :           SLP_TREE_SCALAR_STMTS (conv).create (group_size);
    4714            4 :           SLP_TREE_SCALAR_STMTS (conv).quick_push (scalar_stmt);
    4715           12 :           for (unsigned i = 1; i < group_size; ++i)
    4716            4 :             SLP_TREE_SCALAR_STMTS (conv).quick_push (NULL);
    4717            4 :           reduc = conv;
    4718              :         }
    4719              : 
    4720           87 :       edge le = loop_latch_edge (LOOP_VINFO_LOOP (vinfo));
    4721           87 :       SLP_TREE_CHILDREN (phis).quick_push (NULL);
    4722           87 :       SLP_TREE_CHILDREN (phis).quick_push (NULL);
    4723           87 :       SLP_TREE_CHILDREN (phis)[le->dest_idx] = reduc;
    4724           87 :       SLP_TREE_REF_COUNT (reduc)++;
    4725              : 
    4726              :       /* Create a new SLP instance.  */
    4727           87 :       slp_instance new_instance = XNEW (class _slp_instance);
    4728           87 :       SLP_INSTANCE_TREE (new_instance) = reduc;
    4729           87 :       SLP_INSTANCE_LOADS (new_instance) = vNULL;
    4730           87 :       SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
    4731           87 :       SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
    4732           87 :       SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
    4733           87 :       new_instance->reduc_phis = NULL;
    4734           87 :       new_instance->cost_vec = vNULL;
    4735           87 :       new_instance->subgraph_entries = vNULL;
    4736              : 
    4737           87 :       vinfo->slp_instances.safe_push (new_instance);
    4738              : 
    4739           87 :       if (dump_enabled_p ())
    4740              :         {
    4741           33 :           dump_printf_loc (MSG_NOTE, vect_location,
    4742              :                            "Final SLP tree for instance %p:\n",
    4743              :                            (void *) new_instance);
    4744           33 :           vect_print_slp_graph (MSG_NOTE, vect_location,
    4745              :                                 SLP_INSTANCE_TREE (new_instance));
    4746              :         }
    4747              : 
    4748              :       return true;
    4749        57695 :     }
    4750              : 
    4751        11884 :   if (scalar_stmts.length () <= 1)
    4752              :     {
    4753        10976 :       scalar_stmts.release ();
    4754        10976 :       return false;
    4755              :     }
    4756              : 
    4757          908 :   scalar_stmts.reverse ();
    4758          908 :   stmt_vec_info reduc_phi_info = next_stmt;
    4759              : 
    4760              :   /* Build the tree for the SLP instance.  */
    4761          908 :   vec<stmt_vec_info> root_stmt_infos = vNULL;
    4762          908 :   vec<tree> remain = vNULL;
    4763              : 
    4764          908 :   if (dump_enabled_p ())
    4765              :     {
    4766          239 :       dump_printf_loc (MSG_NOTE, vect_location,
    4767              :                        "Starting SLP discovery of reduction chain for\n");
    4768         1207 :       for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    4769         1936 :         dump_printf_loc (MSG_NOTE, vect_location,
    4770          968 :                          "  %G", scalar_stmts[i]->stmt);
    4771              :     }
    4772              : 
    4773              :   /* Build the tree for the SLP instance.  */
    4774          908 :   unsigned int group_size = scalar_stmts.length ();
    4775          908 :   match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    4776          908 :   unsigned tree_size = 0;
    4777              : 
    4778              :   /* ???  We need this only for SLP discovery.  */
    4779         5264 :   for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    4780         4356 :     REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0];
    4781              : 
    4782          908 :   slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
    4783          908 :                                        &tree_size, bst_map);
    4784              : 
    4785         5264 :   for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    4786         4356 :     REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL;
    4787              : 
    4788          908 :   if (node != NULL)
    4789              :     {
    4790              :       /* Create a new SLP instance.  */
    4791          611 :       slp_instance new_instance = XNEW (class _slp_instance);
    4792          611 :       SLP_INSTANCE_TREE (new_instance) = node;
    4793          611 :       SLP_INSTANCE_LOADS (new_instance) = vNULL;
    4794          611 :       SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
    4795          611 :       SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
    4796          611 :       SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
    4797          611 :       new_instance->reduc_phis = NULL;
    4798          611 :       new_instance->cost_vec = vNULL;
    4799          611 :       new_instance->subgraph_entries = vNULL;
    4800              : 
    4801          611 :       vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
    4802          611 :       reduc_info->is_reduc_chain = true;
    4803              : 
    4804          611 :       if (dump_enabled_p ())
    4805          153 :         dump_printf_loc (MSG_NOTE, vect_location,
    4806              :                          "SLP size %u vs. limit %u.\n",
    4807              :                          tree_size, max_tree_size);
    4808              : 
    4809              :       /* Fixup SLP reduction chains.  If this is a reduction chain with
    4810              :          a conversion in front amend the SLP tree with a node for that.  */
    4811          611 :       gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt;
    4812          611 :       if (is_gimple_assign (scalar_def)
    4813          611 :           && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def)))
    4814              :         {
    4815           43 :           stmt_vec_info conv_info = vect_stmt_to_vectorize
    4816           43 :                                         (STMT_VINFO_REDUC_DEF (reduc_phi_info));
    4817           43 :           scalar_stmts = vNULL;
    4818           43 :           scalar_stmts.create (group_size);
    4819          178 :           for (unsigned i = 0; i < group_size; ++i)
    4820           92 :             scalar_stmts.quick_push (conv_info);
    4821           43 :           slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
    4822           43 :           SLP_TREE_VECTYPE (conv)
    4823           43 :             = get_vectype_for_scalar_type (vinfo,
    4824           43 :                                            TREE_TYPE
    4825              :                                              (gimple_assign_lhs (scalar_def)),
    4826              :                                            group_size);
    4827           43 :           SLP_TREE_REDUC_IDX (conv) = 0;
    4828           43 :           conv->cycle_info.id = node->cycle_info.id;
    4829           43 :           SLP_TREE_CHILDREN (conv).quick_push (node);
    4830           43 :           SLP_INSTANCE_TREE (new_instance) = conv;
    4831              :         }
    4832              :       /* Fill the backedge child of the PHI SLP node.  The
    4833              :          general matching code cannot find it because the
    4834              :          scalar code does not reflect how we vectorize the
    4835              :          reduction.  */
    4836          611 :       use_operand_p use_p;
    4837          611 :       imm_use_iterator imm_iter;
    4838          611 :       class loop *loop = LOOP_VINFO_LOOP (vinfo);
    4839         1968 :       FOR_EACH_IMM_USE_FAST (use_p, imm_iter,
    4840              :                              gimple_get_lhs (scalar_def))
    4841              :         /* There are exactly two non-debug uses, the reduction
    4842              :            PHI and the loop-closed PHI node.  */
    4843         1357 :         if (!is_gimple_debug (USE_STMT (use_p))
    4844         1357 :             && gimple_bb (USE_STMT (use_p)) == loop->header)
    4845              :           {
    4846          611 :             auto_vec<stmt_vec_info, 64> phis (group_size);
    4847          611 :             stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p));
    4848         4213 :             for (unsigned i = 0; i < group_size; ++i)
    4849         3602 :               phis.quick_push (phi_info);
    4850          611 :             slp_tree *phi_node = bst_map->get (phis);
    4851          611 :             unsigned dest_idx = loop_latch_edge (loop)->dest_idx;
    4852         1222 :             SLP_TREE_CHILDREN (*phi_node)[dest_idx]
    4853          611 :               = SLP_INSTANCE_TREE (new_instance);
    4854          611 :             SLP_INSTANCE_TREE (new_instance)->refcnt++;
    4855          611 :           }
    4856              : 
    4857          611 :       vinfo->slp_instances.safe_push (new_instance);
    4858              : 
    4859              :       /* ???  We've replaced the old SLP_INSTANCE_GROUP_SIZE with
    4860              :          the number of SLP lanes of the root in a few places.
    4861              :          Verify that assumption holds.  */
    4862          611 :       gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
    4863              :                   == group_size);
    4864              : 
    4865          611 :       if (dump_enabled_p ())
    4866              :         {
    4867          153 :           dump_printf_loc (MSG_NOTE, vect_location,
    4868              :                            "Final SLP tree for instance %p:\n",
    4869              :                            (void *) new_instance);
    4870          153 :           vect_print_slp_graph (MSG_NOTE, vect_location,
    4871              :                                 SLP_INSTANCE_TREE (new_instance));
    4872              :         }
    4873              : 
    4874          611 :       return true;
    4875              :     }
    4876              : 
    4877              :   /* Failed to SLP.  */
    4878          297 :   scalar_stmts.release ();
    4879          297 :   if (dump_enabled_p ())
    4880           86 :     dump_printf_loc (MSG_NOTE, vect_location,
    4881              :                      "SLP discovery of reduction chain failed\n");
    4882              :   return false;
    4883              : }
    4884              : 
    4885              : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
    4886              :    of KIND.  Return true if successful.  */
    4887              : 
    4888              : static bool
    4889        97892 : vect_analyze_slp_reduction (loop_vec_info vinfo,
    4890              :                             stmt_vec_info scalar_stmt,
    4891              :                             unsigned max_tree_size, unsigned *limit,
    4892              :                             scalar_stmts_to_slp_tree_map_t *bst_map,
    4893              :                             bool force_single_lane)
    4894              : {
    4895        97892 :   slp_instance_kind kind = slp_inst_kind_reduc_group;
    4896              : 
    4897              :   /* Try to gather a reduction chain.  Only attempt if there's budget left
    4898              :      since chain analysis may build multi-lane trees that consume limit.  */
    4899        97892 :   if (! force_single_lane
    4900        70765 :       && *limit != 0
    4901        70765 :       && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def
    4902       168378 :       && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt,
    4903              :                                        max_tree_size, limit))
    4904              :     return true;
    4905              : 
    4906        97194 :   vec<stmt_vec_info> scalar_stmts;
    4907        97194 :   scalar_stmts.create (1);
    4908        97194 :   scalar_stmts.quick_push (scalar_stmt);
    4909              : 
    4910        97194 :   if (dump_enabled_p ())
    4911              :     {
    4912         4028 :       dump_printf_loc (MSG_NOTE, vect_location,
    4913              :                        "Starting SLP discovery for\n");
    4914         8056 :       for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    4915         8056 :         dump_printf_loc (MSG_NOTE, vect_location,
    4916         4028 :                          "  %G", scalar_stmts[i]->stmt);
    4917              :     }
    4918              : 
    4919              :   /* Build the tree for the SLP instance.  */
    4920        97194 :   unsigned int group_size = scalar_stmts.length ();
    4921        97194 :   match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    4922        97194 :   unsigned tree_size = 0;
    4923              : 
    4924        97194 :   slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
    4925              :                                        &tree_size, bst_map);
    4926        97194 :   if (node != NULL)
    4927              :     {
    4928              :       /* Create a new SLP instance.  */
    4929        95397 :       slp_instance new_instance = XNEW (class _slp_instance);
    4930        95397 :       SLP_INSTANCE_TREE (new_instance) = node;
    4931        95397 :       SLP_INSTANCE_LOADS (new_instance) = vNULL;
    4932        95397 :       SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
    4933        95397 :       SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
    4934        95397 :       SLP_INSTANCE_KIND (new_instance) = kind;
    4935        95397 :       new_instance->reduc_phis = NULL;
    4936        95397 :       new_instance->cost_vec = vNULL;
    4937        95397 :       new_instance->subgraph_entries = vNULL;
    4938              : 
    4939        95397 :       if (dump_enabled_p ())
    4940         3945 :         dump_printf_loc (MSG_NOTE, vect_location,
    4941              :                          "SLP size %u vs. limit %u.\n",
    4942              :                          tree_size, max_tree_size);
    4943              : 
    4944        95397 :       vinfo->slp_instances.safe_push (new_instance);
    4945              : 
    4946              :       /* ???  We've replaced the old SLP_INSTANCE_GROUP_SIZE with
    4947              :          the number of SLP lanes of the root in a few places.
    4948              :          Verify that assumption holds.  */
    4949        95397 :       gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
    4950              :                   == group_size);
    4951              : 
    4952        95397 :       if (dump_enabled_p ())
    4953              :         {
    4954         3945 :           dump_printf_loc (MSG_NOTE, vect_location,
    4955              :                            "Final SLP tree for instance %p:\n",
    4956              :                            (void *) new_instance);
    4957         3945 :           vect_print_slp_graph (MSG_NOTE, vect_location,
    4958              :                                 SLP_INSTANCE_TREE (new_instance));
    4959              :         }
    4960              : 
    4961        95397 :       return true;
    4962              :     }
    4963              :   /* Failed to SLP.  */
    4964              : 
    4965              :   /* Free the allocated memory.  */
    4966         1797 :   scalar_stmts.release ();
    4967              : 
    4968              :   /* Failed to SLP.  */
    4969         1797 :   if (dump_enabled_p ())
    4970           83 :     dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
    4971              :   return false;
    4972              : }
    4973              : 
    4974              : /* Analyze SCALAR_STMTS for SLP reduction groups.  If successful add
    4975              :    SLP instances for them and return true.  Any groups of size one
    4976              :    are handled appropriately.  Otherwise return false.  */
    4977              : 
    4978              : static bool
    4979        12140 : vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
    4980              :                                   vec<stmt_vec_info> scalar_stmts,
    4981              :                                   scalar_stmts_to_slp_tree_map_t *bst_map,
    4982              :                                   unsigned max_tree_size, unsigned *limit)
    4983              : {
    4984              :   /* Try to form a reduction group.  Size-1 groups are not suitable
    4985              :      for SLP reduction and should fall back to single-lane reduction.  */
    4986        12140 :   unsigned int group_size = scalar_stmts.length ();
    4987        12140 :   gcc_assert (group_size > 1);
    4988        12140 :   match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    4989        12140 :   unsigned tree_size = 0;
    4990        12140 :   slp_tree node = vect_build_slp_tree (loop_vinfo, scalar_stmts, matches, limit,
    4991              :                                        &tree_size, bst_map);
    4992        12140 :   if (node)
    4993              :     {
    4994              :       /* Create a new SLP instance.  */
    4995         4561 :       slp_instance new_instance = XNEW (class _slp_instance);
    4996         4561 :       SLP_INSTANCE_TREE (new_instance) = node;
    4997         4561 :       SLP_INSTANCE_LOADS (new_instance) = vNULL;
    4998         4561 :       SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
    4999         4561 :       SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
    5000         4561 :       SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_group;
    5001         4561 :       new_instance->reduc_phis = NULL;
    5002         4561 :       new_instance->cost_vec = vNULL;
    5003         4561 :       new_instance->subgraph_entries = vNULL;
    5004              : 
    5005         4561 :       if (dump_enabled_p ())
    5006          203 :         dump_printf_loc (MSG_NOTE, vect_location,
    5007              :                          "SLP size %u vs. limit %u.\n",
    5008              :                          tree_size, max_tree_size);
    5009              : 
    5010         4561 :       loop_vinfo->slp_instances.safe_push (new_instance);
    5011              : 
    5012              :       /* ???  We've replaced the old SLP_INSTANCE_GROUP_SIZE with
    5013              :          the number of SLP lanes of the root in a few places.
    5014              :          Verify that assumption holds.  */
    5015         4561 :       gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
    5016              :                   == group_size);
    5017              : 
    5018         4561 :       if (dump_enabled_p ())
    5019              :         {
    5020          203 :           dump_printf_loc (MSG_NOTE, vect_location,
    5021              :                            "SLP discovery of size %d reduction group "
    5022              :                            "succeeded\n", group_size);
    5023          203 :           dump_printf_loc (MSG_NOTE, vect_location,
    5024              :                            "Final SLP tree for instance %p:\n",
    5025              :                            (void *) new_instance);
    5026          203 :           vect_print_slp_graph (MSG_NOTE, vect_location,
    5027              :                                 SLP_INSTANCE_TREE (new_instance));
    5028              :         }
    5029              : 
    5030         4561 :       return true;
    5031              :     }
    5032              : 
    5033              :   /* Bail on fatal error.  */
    5034         7579 :   if (matches[0] == -1)
    5035              :     return false;
    5036              : 
    5037              :   /* When analysis as a single SLP reduction group failed try to form
    5038              :      sub-groups by collecting matching lanes.  Everything not
    5039              :      covered by a sub-group gets single-reduction treatment.  */
    5040         7455 :   vec<stmt_vec_info> cands = vNULL;
    5041         7455 :   int first_cand = -1;
    5042        31332 :   do
    5043              :     {
    5044        31332 :       cands.truncate (0);
    5045        31332 :       cands.reserve (group_size, true);
    5046        31332 :       int new_first_cand = -1;
    5047       264782 :       for (unsigned i = 0; i < group_size; ++i)
    5048       202118 :         if (matches[i] == first_cand)
    5049              :           {
    5050        36347 :             cands.quick_push (scalar_stmts[i]);
    5051        36347 :             scalar_stmts[i] = NULL;
    5052              :           }
    5053       165771 :         else if (new_first_cand == -1
    5054       165771 :                  && matches[i] > first_cand)
    5055              :           {
    5056        23910 :             gcc_assert (matches[i] == (int)i);
    5057              :             new_first_cand = i;
    5058              :           }
    5059              : 
    5060        31332 :       if (cands.is_empty ())
    5061              :         ;
    5062        24016 :       else if (cands.length () == 1)
    5063              :         {
    5064        33092 :           if (! vect_analyze_slp_reduction (loop_vinfo,
    5065        16546 :                                             vect_stmt_to_vectorize (cands[0]),
    5066              :                                             max_tree_size, limit,
    5067              :                                             bst_map, false))
    5068              :             {
    5069           36 :               cands.release ();
    5070           36 :               return false;
    5071              :             }
    5072        16510 :           cands.release ();
    5073              :         }
    5074              :       /* Try to form a reduction group.  */
    5075         7470 :       else if (!vect_analyze_slp_reduction_group (loop_vinfo, cands, bst_map,
    5076              :                                                   max_tree_size, limit))
    5077              :         {
    5078           14 :           cands.release ();
    5079           14 :           return false;
    5080              :         }
    5081              :       else
    5082         7456 :         cands = vNULL;
    5083        31282 :       first_cand = new_first_cand;
    5084              :     }
    5085        31282 :   while (first_cand >= 0);
    5086         7405 :   cands.release ();
    5087              : 
    5088         7405 :   scalar_stmts.release ();
    5089         7405 :   return true;
    5090              : }
    5091              : 
    5092              : /* Analyze reductions in LOOP_VINFO and populate SLP instances
    5093              :    accordingly.  Returns false if something fails.  */
    5094              : 
    5095              : static bool
    5096       533325 : vect_analyze_slp_reductions (loop_vec_info loop_vinfo,
    5097              :                              unsigned max_tree_size, unsigned *limit,
    5098              :                              scalar_stmts_to_slp_tree_map_t *bst_map,
    5099              :                              bool force_single_lane)
    5100              : {
    5101       533325 :   if (loop_vinfo->reductions.is_empty ())
    5102              :     return true;
    5103              : 
    5104              :   /* Collect reduction statements we can combine into
    5105              :      a SLP reduction.  */
    5106        75687 :   vec<stmt_vec_info> scalar_stmts;
    5107        75687 :   scalar_stmts.create (loop_vinfo->reductions.length ());
    5108       335067 :   for (auto next_info : loop_vinfo->reductions)
    5109              :     {
    5110       108006 :       next_info = vect_stmt_to_vectorize (next_info);
    5111       108006 :       if ((STMT_VINFO_RELEVANT_P (next_info)
    5112           14 :            || STMT_VINFO_LIVE_P (next_info))
    5113              :           /* ???  Make sure we didn't skip a conversion around a
    5114              :              reduction path.  In that case we'd have to reverse
    5115              :              engineer that conversion stmt following the chain using
    5116              :              reduc_idx and from the PHI using reduc_def.  */
    5117       107992 :           && (STMT_VINFO_DEF_TYPE (next_info) == vect_reduction_def
    5118       107992 :               || (STMT_VINFO_DEF_TYPE (next_info)
    5119              :                   == vect_double_reduction_def)))
    5120              :         {
    5121              :           /* Do not discover SLP reductions combining lane-reducing
    5122              :              ops, that will fail later.  */
    5123       107992 :           if (!force_single_lane
    5124       107992 :               && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info)))
    5125        80150 :             scalar_stmts.quick_push (next_info);
    5126              :           /* Do SLP discovery for single-lane reductions.  */
    5127        27842 :           else if (! vect_analyze_slp_reduction (loop_vinfo, next_info,
    5128              :                                                  max_tree_size, limit,
    5129              :                                                  bst_map,
    5130              :                                                  force_single_lane))
    5131              :             {
    5132            0 :               scalar_stmts.release ();
    5133            0 :               return false;
    5134              :             }
    5135              :         }
    5136              :     }
    5137              : 
    5138              :   /* Try to form reduction groups.  */
    5139        75687 :   if (scalar_stmts.length () > 1)
    5140              :     {
    5141         4670 :       if (!vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts, bst_map,
    5142              :                                              max_tree_size, limit))
    5143              :         {
    5144          160 :           scalar_stmts.release ();
    5145          160 :           return false;
    5146              :         }
    5147              :       return true;
    5148              :     }
    5149              : 
    5150              :   /* Do SLP discovery for a single reduction.  */
    5151        71017 :   if (scalar_stmts.length () == 1
    5152       124521 :       && !vect_analyze_slp_reduction (loop_vinfo,
    5153        53504 :                                       vect_stmt_to_vectorize (scalar_stmts[0]),
    5154              :                                       max_tree_size, limit,
    5155              :                                       bst_map, force_single_lane))
    5156              :     {
    5157         1761 :       scalar_stmts.release ();
    5158         1761 :       return false;
    5159              :     }
    5160              : 
    5161        69256 :   scalar_stmts.release ();
    5162        69256 :   return true;
    5163              : }
    5164              : 
    5165              : /* Analyze an SLP instance starting from a group of grouped stores.  Call
    5166              :    vect_build_slp_tree to build a tree of packed stmts if possible.
    5167              :    Return FALSE if it's impossible to SLP any stmt in the group.  */
    5168              : 
    5169              : static bool
    5170      1106300 : vect_analyze_slp_instance (vec_info *vinfo,
    5171              :                            scalar_stmts_to_slp_tree_map_t *bst_map,
    5172              :                            stmt_vec_info stmt_info,
    5173              :                            slp_instance_kind kind,
    5174              :                            unsigned max_tree_size, unsigned *limit,
    5175              :                            bool force_single_lane)
    5176              : {
    5177      1106300 :   vec<stmt_vec_info> scalar_stmts;
    5178              : 
    5179      1106300 :   if (is_a <bb_vec_info> (vinfo))
    5180      1072046 :     vect_location = stmt_info->stmt;
    5181              : 
    5182      1106300 :   gcc_assert (kind == slp_inst_kind_store);
    5183              : 
    5184              :   /* Collect the stores and store them in scalar_stmts.  */
    5185      1106300 :   scalar_stmts.create (DR_GROUP_SIZE (stmt_info));
    5186      1106300 :   stmt_vec_info next_info = stmt_info;
    5187      5491928 :   while (next_info)
    5188              :     {
    5189      3279328 :       scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info));
    5190      3279328 :       next_info = DR_GROUP_NEXT_ELEMENT (next_info);
    5191              :     }
    5192              : 
    5193      1106300 :   vec<stmt_vec_info> root_stmt_infos = vNULL;
    5194      1106300 :   vec<tree> remain = vNULL;
    5195              : 
    5196              :   /* Build the tree for the SLP instance.  */
    5197              : 
    5198              :   /* If there's no budget left bail out early.  */
    5199      1106300 :   if (*limit == 0)
    5200              :     return false;
    5201              : 
    5202      1106280 :   if (dump_enabled_p ())
    5203              :     {
    5204         4252 :       dump_printf_loc (MSG_NOTE, vect_location,
    5205              :                        "Starting SLP discovery for\n");
    5206        24675 :       for (unsigned i = 0; i < scalar_stmts.length (); ++i)
    5207        40846 :         dump_printf_loc (MSG_NOTE, vect_location,
    5208        20423 :                          "  %G", scalar_stmts[i]->stmt);
    5209              :     }
    5210              : 
    5211              :   /* Build the tree for the SLP instance.  */
    5212      1106280 :   unsigned int group_size = scalar_stmts.length ();
    5213      1106280 :   match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
    5214      1106280 :   unsigned tree_size = 0;
    5215      1106280 :   unsigned i;
    5216              : 
    5217      1106280 :   slp_tree node = NULL;
    5218      1106280 :   if (group_size > 1 && force_single_lane)
    5219              :     {
    5220         3575 :       matches[0] = 0;
    5221         3575 :       matches[1] = -1;
    5222              :     }
    5223              :   else
    5224      1102705 :     node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
    5225              :                                 &tree_size, bst_map);
    5226      1106280 :   if (node != NULL)
    5227              :     {
    5228              :       /* Create a new SLP instance.  */
    5229       698753 :       slp_instance new_instance = XNEW (class _slp_instance);
    5230       698753 :       SLP_INSTANCE_TREE (new_instance) = node;
    5231       698753 :       SLP_INSTANCE_LOADS (new_instance) = vNULL;
    5232       698753 :       SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
    5233       698753 :       SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
    5234       698753 :       SLP_INSTANCE_KIND (new_instance) = kind;
    5235       698753 :       new_instance->reduc_phis = NULL;
    5236       698753 :       new_instance->cost_vec = vNULL;
    5237       698753 :       new_instance->subgraph_entries = vNULL;
    5238              : 
    5239       698753 :       if (dump_enabled_p ())
    5240         3244 :         dump_printf_loc (MSG_NOTE, vect_location,
    5241              :                          "SLP size %u vs. limit %u.\n",
    5242              :                          tree_size, max_tree_size);
    5243              : 
    5244       698753 :       vinfo->slp_instances.safe_push (new_instance);
    5245              : 
    5246              :       /* ???  We've replaced the old SLP_INSTANCE_GROUP_SIZE with
    5247              :          the number of SLP lanes of the root in a few places.
    5248              :          Verify that assumption holds.  */
    5249       698753 :       gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
    5250              :                   == group_size);
    5251              : 
    5252       698753 :       if (dump_enabled_p ())
    5253              :         {
    5254         3244 :           dump_printf_loc (MSG_NOTE, vect_location,
    5255              :                            "Final SLP tree for instance %p:\n",
    5256              :                            (void *) new_instance);
    5257         3244 :           vect_print_slp_graph (MSG_NOTE, vect_location,
    5258              :                                 SLP_INSTANCE_TREE (new_instance));
    5259              :         }
    5260       698753 :       return true;
    5261              :     }
    5262              :   /* Failed to SLP.  */
    5263              : 
    5264              :   /* Try to break the group up into pieces.  */
    5265              :   /* ???  Improve this with the matches[] improvements.  */
    5266       407527 :   if (*limit > 0 && kind == slp_inst_kind_store)
    5267              :     {
    5268              :       /* ???  We could delay all the actual splitting of store-groups
    5269              :          until after SLP discovery of the original group completed.
    5270              :          Then we can recurse to vect_build_slp_instance directly.  */
    5271      1015204 :       for (i = 0; i < group_size; i++)
    5272      1015204 :         if (matches[i] != 0)
    5273              :           break;
    5274              : 
    5275              :       /* For basic block SLP, try to break the group up into multiples of
    5276              :          a vector size.  */
    5277       407526 :       if (is_a <bb_vec_info> (vinfo)
    5278       407526 :           && (i > 1 && i < group_size))
    5279              :         {
    5280              :           /* Free the allocated memory.  */
    5281       133883 :           scalar_stmts.release ();
    5282              : 
    5283       133883 :           tree scalar_type
    5284       133883 :             = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
    5285       267766 :           tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
    5286       133883 :                                                       1 << floor_log2 (i));
    5287       133883 :           unsigned HOST_WIDE_INT const_nunits;
    5288       133883 :           if (vectype
    5289       133883 :               && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits))
    5290              :             {
    5291              :               /* Split into two groups at the first vector boundary.  */
    5292       133883 :               gcc_assert ((const_nunits & (const_nunits - 1)) == 0);
    5293       133883 :               unsigned group1_size = i & ~(const_nunits - 1);
    5294              : 
    5295       133883 :               if (dump_enabled_p ())
    5296           83 :                 dump_printf_loc (MSG_NOTE, vect_location,
    5297              :                                  "Splitting SLP group at stmt %u\n", i);
    5298       133883 :               stmt_vec_info rest = vect_split_slp_store_group (stmt_info,
    5299              :                                                                group1_size);
    5300       133883 :               bool res = vect_analyze_slp_instance (vinfo, bst_map, stmt_info,
    5301              :                                                     kind, max_tree_size,
    5302              :                                                     limit, false);
    5303              :               /* Split the rest at the failure point and possibly
    5304              :                  re-analyze the remaining matching part if it has
    5305              :                  at least two lanes.  */
    5306       133883 :               if (group1_size < i
    5307         5797 :                   && (i + 1 < group_size
    5308         2579 :                       || i - group1_size > 1))
    5309              :                 {
    5310         3252 :                   stmt_vec_info rest2 = rest;
    5311         3252 :                   rest = vect_split_slp_store_group (rest, i - group1_size);
    5312         3252 :                   if (i - group1_size > 1)
    5313           71 :                     res |= vect_analyze_slp_instance (vinfo, bst_map, rest2,
    5314              :                                                       kind, max_tree_size,
    5315              :                                                       limit, false);
    5316              :                 }
    5317              :               /* Re-analyze the non-matching tail if it has at least
    5318              :                  two lanes.  */
    5319       133883 :               if (i + 1 < group_size)
    5320        22440 :                 res |= vect_analyze_slp_instance (vinfo, bst_map,
    5321              :                                                   rest, kind, max_tree_size,
    5322              :                                                   limit, false);
    5323      1106300 :               return res;
    5324              :             }
    5325              :         }
    5326              : 
    5327              :       /* For loop vectorization split the RHS into arbitrary pieces of
    5328              :          size >= 1.  */
    5329       273643 :       else if (is_a <loop_vec_info> (vinfo)
    5330       273643 :                && (group_size != 1 && i < group_size))
    5331              :         {
    5332        12187 :           gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
    5333           28 :           bool masked_p = call
    5334           28 :               && gimple_call_internal_p (call)
    5335           28 :               && internal_fn_mask_index (gimple_call_internal_fn (call)) != -1;
    5336              :           /* There are targets that cannot do even/odd interleaving schemes
    5337              :              so they absolutely need to use load/store-lanes.  For now
    5338              :              force single-lane SLP for them - they would be happy with
    5339              :              uniform power-of-two lanes (but depending on element size),
    5340              :              but even if we can use 'i' as indicator we would need to
    5341              :              backtrack when later lanes fail to discover with the same
    5342              :              granularity.  We cannot turn any of strided or scatter store
    5343              :              into store-lanes.  */
    5344              :           /* ???  If this is not in sync with what get_load_store_type
    5345              :              later decides the SLP representation is not good for other
    5346              :              store vectorization methods.  */
    5347        12187 :           bool want_store_lanes
    5348        12187 :             = (! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
    5349        12187 :                && ! STMT_VINFO_STRIDED_P (stmt_info)
    5350        10064 :                && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
    5351        10060 :                && compare_step_with_zero (vinfo, stmt_info) > 0
    5352        18334 :                && vect_slp_prefer_store_lanes_p (vinfo, stmt_info, NULL_TREE,
    5353        24374 :                                                  masked_p, group_size, i));
    5354        12187 :           if (want_store_lanes || force_single_lane)
    5355              :             i = 1;
    5356              : 
    5357              :           /* A fatal discovery fail doesn't always mean single-lane SLP
    5358              :              isn't a possibility, so try.  */
    5359         8612 :           if (i == 0)
    5360              :             i = 1;
    5361              : 
    5362        12187 :           if (dump_enabled_p ())
    5363          875 :             dump_printf_loc (MSG_NOTE, vect_location,
    5364              :                              "Splitting SLP group at stmt %u\n", i);
    5365              : 
    5366              :           /* Analyze the stored values and pinch them together with
    5367              :              a permute node so we can preserve the whole store group.  */
    5368        12187 :           auto_vec<slp_tree> rhs_nodes;
    5369              : 
    5370        12187 :           unsigned int rhs_common_nlanes = 0;
    5371        12187 :           unsigned int start = 0, end = i;
    5372        49367 :           while (start < group_size)
    5373              :             {
    5374        37514 :               gcc_assert (end - start >= 1);
    5375        37514 :               vec<stmt_vec_info> substmts;
    5376        37514 :               substmts.create (end - start);
    5377       148731 :               for (unsigned j = start; j < end; ++j)
    5378        73703 :                 substmts.quick_push (scalar_stmts[j]);
    5379        37514 :               node = vect_build_slp_tree (vinfo, substmts,
    5380              :                                           matches, limit, &tree_size, bst_map);
    5381        37514 :               if (node)
    5382              :                 {
    5383        31235 :                   rhs_nodes.safe_push (node);
    5384        31235 :                   if (start == 0)
    5385        11853 :                     rhs_common_nlanes = SLP_TREE_LANES (node);
    5386        19382 :                   else if (rhs_common_nlanes != SLP_TREE_LANES (node))
    5387         1425 :                     rhs_common_nlanes = 0;
    5388        31235 :                   start = end;
    5389        31235 :                   if (want_store_lanes || force_single_lane)
    5390         8937 :                     end = start + 1;
    5391              :                   else
    5392              :                     end = group_size;
    5393              :                 }
    5394              :               else
    5395              :                 {
    5396         6279 :                   substmts.release ();
    5397         6279 :                   if (end - start == 1)
    5398              :                     {
    5399              :                       /* Single-lane discovery failed.  Free resources.  */
    5400          334 :                       for (auto node : rhs_nodes)
    5401            0 :                         vect_free_slp_tree (node);
    5402          334 :                       scalar_stmts.release ();
    5403          334 :                       if (dump_enabled_p ())
    5404           14 :                         dump_printf_loc (MSG_NOTE, vect_location,
    5405              :                                          "SLP discovery failed\n");
    5406          334 :                       return false;
    5407              :                     }
    5408              : 
    5409              :                   /* ???  It really happens that we soft-fail SLP
    5410              :                      build at a mismatch but the matching part hard-fails
    5411              :                      later.  As we know we arrived here with a group
    5412              :                      larger than one try a group of size one!  */
    5413         5945 :                   if (matches[0] != 0)
    5414           12 :                     end = start + 1;
    5415              :                   else
    5416        13037 :                     for (unsigned j = start; j < end; j++)
    5417        13037 :                       if (matches[j - start] != 0)
    5418              :                         {
    5419              :                           end = j;
    5420              :                           break;
    5421              :                         }
    5422              :                 }
    5423              :             }
    5424              : 
    5425              :           /* Now re-assess whether we want store lanes in case the
    5426              :              discovery ended up producing all single-lane RHSs.  */
    5427        11853 :           if (! want_store_lanes
    5428        11853 :               && rhs_common_nlanes == 1
    5429        10761 :               && ! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
    5430        10761 :               && ! STMT_VINFO_STRIDED_P (stmt_info)
    5431         9009 :               && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
    5432         9006 :               && compare_step_with_zero (vinfo, stmt_info) > 0
    5433        17182 :               && (vect_store_lanes_supported (SLP_TREE_VECTYPE (rhs_nodes[0]),
    5434              :                                               group_size, masked_p)
    5435              :                   != IFN_LAST))
    5436              :             want_store_lanes = true;
    5437              : 
    5438              :           /* Now we assume we can build the root SLP node from all stores.  */
    5439        11853 :           if (want_store_lanes)
    5440              :             {
    5441              :               /* For store-lanes feed the store node with all RHS nodes
    5442              :                  in order.  */
    5443            0 :               node = vect_create_new_slp_node (scalar_stmts,
    5444            0 :                                                SLP_TREE_CHILDREN
    5445              :                                                  (rhs_nodes[0]).length ());
    5446            0 :               SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
    5447            0 :               node->ldst_lanes = true;
    5448            0 :               SLP_TREE_CHILDREN (node)
    5449            0 :                 .reserve_exact (SLP_TREE_CHILDREN (rhs_nodes[0]).length ()
    5450            0 :                                 + rhs_nodes.length () - 1);
    5451              :               /* First store value and possibly mask.  */
    5452            0 :               SLP_TREE_CHILDREN (node)
    5453            0 :                 .splice (SLP_TREE_CHILDREN (rhs_nodes[0]));
    5454              :               /* Rest of the store values.  All mask nodes are the same,
    5455              :                  this should be guaranteed by dataref group discovery.  */
    5456            0 :               for (unsigned j = 1; j < rhs_nodes.length (); ++j)
    5457            0 :                 SLP_TREE_CHILDREN (node)
    5458            0 :                   .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[0]);
    5459            0 :               for (slp_tree child : SLP_TREE_CHILDREN (node))
    5460            0 :                 child->refcnt++;
    5461              :             }
    5462              :           else
    5463        11853 :             node = vect_build_slp_store_interleaving (rhs_nodes, scalar_stmts);
    5464              : 
    5465        43088 :           while (!rhs_nodes.is_empty ())
    5466        31235 :             vect_free_slp_tree (rhs_nodes.pop ());
    5467              : 
    5468              :           /* Create a new SLP instance.  */
    5469        11853 :           slp_instance new_instance = XNEW (class _slp_instance);
    5470        11853 :           SLP_INSTANCE_TREE (new_instance) = node;
    5471        11853 :           SLP_INSTANCE_LOADS (new_instance) = vNULL;
    5472        11853 :           SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
    5473        11853 :           SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
    5474        11853 :           SLP_INSTANCE_KIND (new_instance) = kind;
    5475        11853 :           new_instance->reduc_phis = NULL;
    5476        11853 :           new_instance->cost_vec = vNULL;
    5477        11853 :           new_instance->subgraph_entries = vNULL;
    5478              : 
    5479        11853 :           if (dump_enabled_p ())
    5480          861 :             dump_printf_loc (MSG_NOTE, vect_location,
    5481              :                              "SLP size %u vs. limit %u.\n",
    5482              :                              tree_size, max_tree_size);
    5483              : 
    5484        11853 :           vinfo->slp_instances.safe_push (new_instance);
    5485              : 
    5486              :           /* ???  We've replaced the old SLP_INSTANCE_GROUP_SIZE with
    5487              :              the number of SLP lanes of the root in a few places.
    5488              :              Verify that assumption holds.  */
    5489        11853 :           gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
    5490              :                       == group_size);
    5491              : 
    5492        11853 :           if (dump_enabled_p ())
    5493              :             {
    5494          861 :               dump_printf_loc (MSG_NOTE, vect_location,
    5495              :                                "Final SLP tree for instance %p:\n",
    5496              :                                (void *) new_instance);
    5497          861 :               vect_print_slp_graph (MSG_NOTE, vect_location,
    5498              :                                     SLP_INSTANCE_TREE (new_instance));
    5499              :             }
    5500              :           return true;
    5501        12187 :         }
    5502              :       else
    5503              :         /* Free the allocated memory.  */
    5504       261456 :         scalar_stmts.release ();
    5505              : 
    5506              :       /* Even though the first vector did not all match, we might be able to SLP
    5507              :          (some) of the remainder.  FORNOW ignore this possibility.  */
    5508              :     }
    5509              :   else
    5510              :     /* Free the allocated memory.  */
    5511            1 :     scalar_stmts.release ();
    5512              : 
    5513              :   /* Failed to SLP.  */
    5514       261457 :   if (dump_enabled_p ())
    5515           50 :     dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
    5516              :   return false;
    5517              : }
    5518              : 
    5519              : /* qsort comparator ordering SLP load nodes.  */
    5520              : 
    5521              : static int
    5522      2696294 : vllp_cmp (const void *a_, const void *b_)
    5523              : {
    5524      2696294 :   const slp_tree a = *(const slp_tree *)a_;
    5525      2696294 :   const slp_tree b = *(const slp_tree *)b_;
    5526      2696294 :   stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (a)[0];
    5527      2696294 :   stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (b)[0];
    5528      2696294 :   if (STMT_VINFO_GROUPED_ACCESS (a0)
    5529      1518131 :       && STMT_VINFO_GROUPED_ACCESS (b0)
    5530      4151961 :       && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
    5531              :     {
    5532              :       /* Same group, order after lanes used.  */
    5533       343508 :       if (SLP_TREE_LANES (a) < SLP_TREE_LANES (b))
    5534              :         return 1;
    5535       340519 :       else if (SLP_TREE_LANES (a) > SLP_TREE_LANES (b))
    5536              :         return -1;
    5537              :       else
    5538              :         {
    5539              :           /* Try to order loads using the same lanes together, breaking
    5540              :              the tie with the lane number that first differs.  */
    5541       337038 :           if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
    5542       337038 :               && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
    5543              :             return 0;
    5544       337038 :           else if (SLP_TREE_LOAD_PERMUTATION (a).exists ()
    5545       337038 :                    && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
    5546              :             return 1;
    5547       331611 :           else if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
    5548       331611 :                    && SLP_TREE_LOAD_PERMUTATION (b).exists ())
    5549              :             return -1;
    5550              :           else
    5551              :             {
    5552       323849 :               for (unsigned i = 0; i < SLP_TREE_LANES (a); ++i)
    5553       323849 :                 if (SLP_TREE_LOAD_PERMUTATION (a)[i]
    5554       323849 :                     != SLP_TREE_LOAD_PERMUTATION (b)[i])
    5555              :                   {
    5556              :                     /* In-order lane first, that's what the above case for
    5557              :                        no permutation does.  */
    5558       320361 :                     if (SLP_TREE_LOAD_PERMUTATION (a)[i] == i)
    5559              :                       return -1;
    5560       197085 :                     else if (SLP_TREE_LOAD_PERMUTATION (b)[i] == i)
    5561              :                       return 1;
    5562       103456 :                     else if (SLP_TREE_LOAD_PERMUTATION (a)[i]
    5563       103456 :                              < SLP_TREE_LOAD_PERMUTATION (b)[i])
    5564              :                       return -1;
    5565              :                     else
    5566        38798 :                       return 1;
    5567              :                   }
    5568              :               return 0;
    5569              :             }
    5570              :         }
    5571              :     }
    5572              :   else /* Different groups or non-groups.  */
    5573              :     {
    5574              :       /* Order groups as their first element to keep them together.  */
    5575      2352786 :       if (STMT_VINFO_GROUPED_ACCESS (a0))
    5576      2352786 :         a0 = DR_GROUP_FIRST_ELEMENT (a0);
    5577      2352786 :       if (STMT_VINFO_GROUPED_ACCESS (b0))
    5578      2352786 :         b0 = DR_GROUP_FIRST_ELEMENT (b0);
    5579      2352786 :       if (a0 == b0)
    5580              :         return 0;
    5581              :       /* Tie using UID.  */
    5582      2352666 :       else if (gimple_uid (STMT_VINFO_STMT (a0))
    5583      2352666 :                < gimple_uid (STMT_VINFO_STMT (b0)))
    5584              :         return -1;
    5585              :       else
    5586              :         {
    5587      1042915 :           gcc_assert (gimple_uid (STMT_VINFO_STMT (a0))
    5588              :                       != gimple_uid (STMT_VINFO_STMT (b0)));
    5589              :           return 1;
    5590              :         }
    5591              :     }
    5592              : }
    5593              : 
    5594              : /* Return whether if the load permutation of NODE is consecutive starting
    5595              :    with value START_VAL in the first element.  If START_VAL is not given
    5596              :    the first element's value is used.  */
    5597              : 
    5598              : bool
    5599       664804 : vect_load_perm_consecutive_p (slp_tree node, unsigned start_val)
    5600              : {
    5601       664804 :   load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
    5602              : 
    5603       664804 :   if (!perm.exists () || !perm.length ())
    5604              :     return false;
    5605              : 
    5606       664804 :   if (start_val == UINT_MAX)
    5607        80019 :     start_val = perm[0];
    5608              : 
    5609      1313944 :   for (unsigned int i = 0; i < perm.length (); i++)
    5610       673131 :     if (perm[i] != start_val + (unsigned int) i)
    5611              :       return false;
    5612              : 
    5613              :   return true;
    5614              : }
    5615              : 
    5616              : /* Process the set of LOADS that are all from the same dataref group.  */
    5617              : 
    5618              : static void
    5619       163371 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
    5620              :                               scalar_stmts_to_slp_tree_map_t *bst_map,
    5621              :                               const array_slice<slp_tree> &loads,
    5622              :                               bool force_single_lane)
    5623              : {
    5624              :   /* We at this point want to lower without a fixed VF or vector
    5625              :      size in mind which means we cannot actually compute whether we
    5626              :      need three or more vectors for a load permutation yet.  So always
    5627              :      lower.  */
    5628       163371 :   stmt_vec_info first
    5629       163371 :     = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
    5630       163371 :   unsigned group_lanes = DR_GROUP_SIZE (first);
    5631              : 
    5632              :   /* Verify if all load permutations can be implemented with a suitably
    5633              :      large element load-lanes operation.  */
    5634       163371 :   unsigned ld_lanes_lanes = SLP_TREE_LANES (loads[0]);
    5635       163371 :   if (STMT_VINFO_STRIDED_P (first)
    5636       160893 :       || compare_step_with_zero (loop_vinfo, first) <= 0
    5637       158236 :       || exact_log2 (ld_lanes_lanes) == -1
    5638              :       /* ???  For now only support the single-lane case as there is
    5639              :          missing support on the store-lane side and code generation
    5640              :          isn't up to the task yet.  */
    5641       155417 :       || ld_lanes_lanes != 1
    5642       307710 :       || vect_load_lanes_supported (SLP_TREE_VECTYPE (loads[0]),
    5643              :                                     group_lanes / ld_lanes_lanes,
    5644              :                                     false) == IFN_LAST)
    5645              :     ld_lanes_lanes = 0;
    5646              :   else
    5647              :     /* Verify the loads access the same number of lanes aligned to
    5648              :        ld_lanes_lanes.  */
    5649            0 :     for (slp_tree load : loads)
    5650              :       {
    5651            0 :         if (SLP_TREE_LANES (load) != ld_lanes_lanes)
    5652              :           {
    5653              :             ld_lanes_lanes = 0;
    5654              :             break;
    5655              :           }
    5656            0 :         unsigned first = SLP_TREE_LOAD_PERMUTATION (load)[0];
    5657            0 :         if (first % ld_lanes_lanes != 0)
    5658              :           {
    5659              :             ld_lanes_lanes = 0;
    5660              :             break;
    5661              :           }
    5662            0 :         if (!vect_load_perm_consecutive_p (load))
    5663              :           {
    5664              :             ld_lanes_lanes = 0;
    5665              :             break;
    5666              :           }
    5667              :       }
    5668              : 
    5669              :   /* Only a power-of-two number of lanes matches interleaving with N levels.
    5670              :      ???  An even number of lanes could be reduced to 1<<ceil_log2(N)-1 lanes
    5671              :      at each step.  */
    5672       264739 :   if (ld_lanes_lanes == 0 && exact_log2 (group_lanes) == -1 && group_lanes != 3)
    5673              :     return;
    5674              : 
    5675       267379 :   for (slp_tree load : loads)
    5676              :     {
    5677              :       /* Leave masked or gather loads alone for now.  */
    5678       188420 :       if (!SLP_TREE_CHILDREN (load).is_empty ())
    5679        60667 :         continue;
    5680              : 
    5681              :       /* For single-element interleaving spanning multiple vectors avoid
    5682              :          lowering, we want to use VMAT_ELEMENTWISE later.  */
    5683       188414 :       if (ld_lanes_lanes == 0
    5684       188414 :           && SLP_TREE_LANES (load) == 1
    5685       167526 :           && !DR_GROUP_NEXT_ELEMENT (first)
    5686       269099 :           && maybe_gt (group_lanes,
    5687              :                        TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load))))
    5688        51824 :         return;
    5689              : 
    5690              :       /* We want to pattern-match special cases here and keep those
    5691              :          alone.  Candidates are splats and load-lane.  */
    5692              : 
    5693              :       /* We need to lower only loads of less than half of the groups
    5694              :          lanes, including duplicate lanes.  Note this leaves nodes
    5695              :          with a non-1:1 load permutation around instead of canonicalizing
    5696              :          those into a load and a permute node.  Removing this early
    5697              :          check would do such canonicalization.  */
    5698       136590 :       if (SLP_TREE_LANES (load) >= (group_lanes + 1) / 2
    5699        57060 :           && ld_lanes_lanes == 0)
    5700        57060 :         continue;
    5701              : 
    5702              :       /* Build the permute to get the original load permutation order.  */
    5703        79530 :       bool contiguous = vect_load_perm_consecutive_p (load);
    5704        79530 :       lane_permutation_t final_perm;
    5705        79530 :       final_perm.create (SLP_TREE_LANES (load));
    5706       239534 :       for (unsigned i = 0; i < SLP_TREE_LANES (load); ++i)
    5707       160948 :         final_perm.quick_push (
    5708        80474 :           std::make_pair (0, SLP_TREE_LOAD_PERMUTATION (load)[i]));
    5709              : 
    5710              :       /* When the load permutation accesses a contiguous unpermuted,
    5711              :          power-of-two aligned and sized chunk leave the load alone.
    5712              :          We can likely (re-)load it more efficiently rather than
    5713              :          extracting it from the larger load.
    5714              :          ???  Long-term some of the lowering should move to where
    5715              :          the vector types involved are fixed.  */
    5716        83131 :       if (!force_single_lane
    5717        79530 :           && ld_lanes_lanes == 0
    5718        53617 :           && contiguous
    5719        53357 :           && (SLP_TREE_LANES (load) > 1 || loads.size () == 1)
    5720         6653 :           && pow2p_hwi (SLP_TREE_LANES (load))
    5721         6617 :           && pow2p_hwi (group_lanes)
    5722         3601 :           && SLP_TREE_LOAD_PERMUTATION (load)[0] % SLP_TREE_LANES (load) == 0
    5723        83131 :           && group_lanes % SLP_TREE_LANES (load) == 0)
    5724              :         {
    5725         3601 :           final_perm.release ();
    5726         3601 :           continue;
    5727              :         }
    5728              : 
    5729              :       /* First build (and possibly re-use) a load node for the
    5730              :          unpermuted group.  Gaps in the middle and on the end are
    5731              :          represented with NULL stmts.  */
    5732        75929 :       vec<stmt_vec_info> stmts;
    5733        75929 :       stmts.create (group_lanes);
    5734       271167 :       for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
    5735              :         {
    5736       195238 :           if (s != first)
    5737       124994 :             for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
    5738         5685 :               stmts.quick_push (NULL);
    5739       195238 :           stmts.quick_push (s);
    5740              :         }
    5741       142961 :       for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
    5742        67032 :         stmts.quick_push (NULL);
    5743        75929 :       match_elt_t *matches = XALLOCAVEC (match_elt_t, group_lanes);
    5744        75929 :       unsigned limit = 1;
    5745        75929 :       unsigned tree_size = 0;
    5746        75929 :       slp_tree l0 = vect_build_slp_tree (loop_vinfo, stmts, matches, &limit,
    5747        75929 :                                          &tree_size, bst_map);
    5748        75929 :       gcc_assert (!SLP_TREE_LOAD_PERMUTATION (l0).exists ());
    5749              : 
    5750        75929 :       if (ld_lanes_lanes != 0)
    5751              :         {
    5752              :           /* ???  If this is not in sync with what get_load_store_type
    5753              :              later decides the SLP representation is not good for other
    5754              :              store vectorization methods.  */
    5755            0 :           l0->ldst_lanes = true;
    5756            0 :           load->ldst_lanes = true;
    5757              :         }
    5758              : 
    5759       236629 :       while (1)
    5760              :         {
    5761       156279 :           unsigned group_lanes = SLP_TREE_LANES (l0);
    5762       156279 :           if (ld_lanes_lanes != 0
    5763       156279 :               || SLP_TREE_LANES (load) >= (group_lanes + 1) / 2)
    5764              :             break;
    5765              : 
    5766              :           /* Try to lower by reducing the group to half its size using an
    5767              :              interleaving scheme.  For this try to compute whether all
    5768              :              elements needed for this load are in even or odd elements of
    5769              :              an even/odd decomposition with N consecutive elements.
    5770              :              Thus { e, e, o, o, e, e, o, o } would be an even/odd decomposition
    5771              :              with N == 2.  */
    5772              :           /* ???  Only an even number of lanes can be handed this way, but the
    5773              :              fallback below could work for any number.  We have to make sure
    5774              :              to round up in that case.  */
    5775        80350 :           gcc_assert ((group_lanes & 1) == 0 || group_lanes == 3);
    5776        12289 :           unsigned even = 0, odd = 0;
    5777        12289 :           if ((group_lanes & 1) == 0)
    5778              :             {
    5779        12289 :               even = (1 << ceil_log2 (group_lanes)) - 1;
    5780        12289 :               odd = even;
    5781        49905 :               for (auto l : final_perm)
    5782              :                 {
    5783        13038 :                   even &= ~l.second;
    5784        13038 :                   odd &= l.second;
    5785              :                 }
    5786              :             }
    5787              : 
    5788              :           /* Now build an even or odd extraction from the unpermuted load.  */
    5789        80350 :           lane_permutation_t perm;
    5790        80350 :           perm.create ((group_lanes + 1) / 2);
    5791        80350 :           unsigned even_level = even ? 1 << ctz_hwi (even) : 0;
    5792        80350 :           unsigned odd_level = odd ? 1 << ctz_hwi (odd) : 0;
    5793        80350 :           if (even_level
    5794        11307 :               && group_lanes % (2 * even_level) == 0
    5795              :               /* ???  When code generating permutes we do not try to pun
    5796              :                  to larger component modes so level != 1 isn't a natural
    5797              :                  even/odd extract.  Prefer one if possible.  */
    5798        11307 :               && (even_level == 1 || !odd_level || odd_level != 1))
    5799              :             {
    5800              :               /* { 0, 1, ... 4, 5 ..., } */
    5801        43060 :               for (unsigned i = 0; i < group_lanes / 2 / even_level; ++i)
    5802        69241 :                 for (unsigned j = 0; j < even_level; ++j)
    5803        34816 :                   perm.quick_push (std::make_pair (0, 2 * i * even_level + j));
    5804              :             }
    5805        69043 :           else if (odd_level)
    5806              :             {
    5807              :               /* { ..., 2, 3, ... 6, 7 } */
    5808         3624 :               gcc_assert (group_lanes % (2 * odd_level) == 0);
    5809        15946 :               for (unsigned i = 0; i < group_lanes / 2 / odd_level; ++i)
    5810        24698 :                 for (unsigned j = 0; j < odd_level; ++j)
    5811        12376 :                   perm.quick_push
    5812        12376 :                     (std::make_pair (0, (2 * i + 1) * odd_level + j));
    5813              :             }
    5814              :           else
    5815              :             {
    5816              :               /* As fallback extract all used lanes and fill to half the
    5817              :                  group size by repeating the last element.
    5818              :                  ???  This is quite a bad strathegy for re-use - we could
    5819              :                  brute force our way to find more optimal filling lanes to
    5820              :                  maximize re-use when looking at all loads from the group.  */
    5821        68091 :               auto_bitmap l;
    5822       272420 :               for (auto p : final_perm)
    5823        68147 :                 bitmap_set_bit (l, p.second);
    5824        68091 :               unsigned i = 0;
    5825        68091 :               bitmap_iterator bi;
    5826       136238 :               EXECUTE_IF_SET_IN_BITMAP (l, 0, i, bi)
    5827        68147 :                   perm.quick_push (std::make_pair (0, i));
    5828       272516 :               while (perm.length () < (group_lanes + 1) / 2)
    5829        68167 :                 perm.quick_push (perm.last ());
    5830        68091 :             }
    5831              : 
    5832              :           /* Update final_perm with the intermediate permute.  */
    5833       161449 :           for (unsigned i = 0; i < final_perm.length (); ++i)
    5834              :             {
    5835        81099 :               unsigned l = final_perm[i].second;
    5836        81099 :               unsigned j;
    5837        91073 :               for (j = 0; j < perm.length (); ++j)
    5838        91073 :                 if (perm[j].second == l)
    5839              :                   {
    5840        81099 :                     final_perm[i].second = j;
    5841        81099 :                     break;
    5842              :                   }
    5843        81099 :               gcc_assert (j < perm.length ());
    5844              :             }
    5845              : 
    5846              :           /* And create scalar stmts.  */
    5847        80350 :           vec<stmt_vec_info> perm_stmts;
    5848        80350 :           perm_stmts.create (perm.length ());
    5849       344206 :           for (unsigned i = 0; i < perm.length (); ++i)
    5850       183506 :             perm_stmts.quick_push (SLP_TREE_SCALAR_STMTS (l0)[perm[i].second]);
    5851              : 
    5852        80350 :           slp_tree p = vect_create_new_slp_node (1, VEC_PERM_EXPR);
    5853        80350 :           SLP_TREE_CHILDREN (p).quick_push (l0);
    5854        80350 :           SLP_TREE_LANE_PERMUTATION (p) = perm;
    5855        80350 :           SLP_TREE_VECTYPE (p) = SLP_TREE_VECTYPE (load);
    5856        80350 :           SLP_TREE_LANES (p) = perm.length ();
    5857              :           /* ???  As we have scalar stmts for this intermediate permute we
    5858              :              could CSE it via bst_map but we do not want to pick up
    5859              :              another SLP node with a load permutation.  We instead should
    5860              :              have a "local" CSE map here.  */
    5861        80350 :           SLP_TREE_SCALAR_STMTS (p) = perm_stmts;
    5862              : 
    5863              :           /* We now have a node for (group_lanes + 1) / 2 lanes.  */
    5864        80350 :           l0 = p;
    5865        80350 :         }
    5866              : 
    5867              :       /* And finally from the ordered reduction node create the
    5868              :          permute to shuffle the lanes into the original load-permutation
    5869              :          order.  We replace the original load node with this.  */
    5870        75929 :       SLP_TREE_CODE (load) = VEC_PERM_EXPR;
    5871        75929 :       SLP_TREE_LOAD_PERMUTATION (load).release ();
    5872        75929 :       SLP_TREE_LANE_PERMUTATION (load) = final_perm;
    5873        75929 :       SLP_TREE_CHILDREN (load).create (1);
    5874        75929 :       SLP_TREE_CHILDREN (load).quick_push (l0);
    5875        75929 :       SLP_TREE_REPRESENTATIVE (load) = NULL;
    5876              :     }
    5877              : }
    5878              : 
    5879              : /* Transform SLP loads in the SLP graph created by SLP discovery to
    5880              :    group loads from the same group and lower load permutations that
    5881              :    are unlikely to be supported into a series of permutes.
    5882              :    In the degenerate case of having only single-lane SLP instances
    5883              :    this should result in a series of permute nodes emulating an
    5884              :    interleaving scheme.  */
    5885              : 
    5886              : static void
    5887       517993 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
    5888              :                               scalar_stmts_to_slp_tree_map_t *bst_map,
    5889              :                               bool force_single_lane)
    5890              : {
    5891              :   /* Gather and sort loads across all instances.  */
    5892       517993 :   hash_set<slp_tree> visited;
    5893       517993 :   auto_vec<slp_tree> loads;
    5894      2377811 :   for (auto inst : loop_vinfo->slp_instances)
    5895       825900 :     vect_gather_slp_loads (loads, SLP_INSTANCE_TREE (inst), visited);
    5896       517993 :   if (loads.is_empty ())
    5897        97472 :     return;
    5898       420521 :   loads.qsort (vllp_cmp);
    5899              : 
    5900              :   /* Now process each dataref group separately.  */
    5901       420521 :   unsigned firsti = 0;
    5902      1193552 :   for (unsigned i = 1; i < loads.length (); ++i)
    5903              :     {
    5904       352510 :       slp_tree first = loads[firsti];
    5905       352510 :       slp_tree next = loads[i];
    5906       352510 :       stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (first)[0];
    5907       352510 :       stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (next)[0];
    5908       352510 :       if (STMT_VINFO_GROUPED_ACCESS (a0)
    5909       158558 :           && STMT_VINFO_GROUPED_ACCESS (b0)
    5910       497763 :           && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
    5911        62828 :         continue;
    5912              :       /* Now we have one or multiple SLP loads of the same group from
    5913              :          firsti to i - 1.  */
    5914       289682 :       if (STMT_VINFO_GROUPED_ACCESS (a0))
    5915        95730 :         vect_lower_load_permutations (loop_vinfo, bst_map,
    5916        95730 :                                       make_array_slice (&loads[firsti],
    5917              :                                                         i - firsti),
    5918              :                                       force_single_lane);
    5919              :       firsti = i;
    5920              :     }
    5921       841042 :   if (firsti < loads.length ()
    5922       841042 :       && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (loads[firsti])[0]))
    5923        67641 :     vect_lower_load_permutations (loop_vinfo, bst_map,
    5924        67641 :                                   make_array_slice (&loads[firsti],
    5925        67641 :                                                     loads.length () - firsti),
    5926              :                                   force_single_lane);
    5927       517993 : }
    5928              : 
    5929              : /* Check if there are stmts in the loop can be vectorized using SLP.  Build SLP
    5930              :    trees of packed scalar stmts if SLP is possible.  */
    5931              : 
    5932              : opt_result
    5933      1215145 : vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
    5934              :                   bool force_single_lane)
    5935              : {
    5936      1215145 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    5937      1215145 :   unsigned int i;
    5938      1215145 :   stmt_vec_info first_element;
    5939      1215145 :   slp_instance instance;
    5940              : 
    5941      1215145 :   DUMP_VECT_SCOPE ("vect_analyze_slp");
    5942              : 
    5943      1215145 :   unsigned limit = max_tree_size;
    5944              : 
    5945      1215145 :   scalar_stmts_to_slp_tree_map_t *bst_map
    5946      1215145 :     = new scalar_stmts_to_slp_tree_map_t ();
    5947              : 
    5948              :   /* Find SLP sequences starting from groups of grouped stores.  */
    5949      3379854 :   FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
    5950       949906 :     if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
    5951              :                                      slp_inst_kind_store, max_tree_size, &limit,
    5952              :                                      force_single_lane)
    5953       949906 :         && loop_vinfo)
    5954              :       {
    5955          342 :         release_scalar_stmts_to_slp_tree_map (bst_map);
    5956          342 :         return opt_result::failure_at (vect_location, "SLP build failed.\n");
    5957              :       }
    5958              : 
    5959              :   /* For loops also start SLP discovery from non-grouped stores.  */
    5960      1214803 :   if (loop_vinfo)
    5961              :     {
    5962              :       data_reference_p dr;
    5963      1745448 :       FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
    5964      1212123 :         if (DR_IS_WRITE (dr))
    5965              :           {
    5966       394362 :             stmt_vec_info stmt_info = vinfo->lookup_dr (dr)->stmt;
    5967              :             /* Grouped stores are already handled above.  */
    5968       394362 :             if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    5969       111069 :               continue;
    5970       283293 :             vec<stmt_vec_info> stmts;
    5971       283293 :             vec<stmt_vec_info> roots = vNULL;
    5972       283293 :             vec<tree> remain = vNULL;
    5973       283293 :             stmts.create (1);
    5974       283293 :             stmts.quick_push (stmt_info);
    5975       283293 :             if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
    5976              :                                            stmts, roots, remain, max_tree_size,
    5977              :                                            &limit, bst_map, force_single_lane))
    5978              :               {
    5979         4111 :                 release_scalar_stmts_to_slp_tree_map (bst_map);
    5980         4111 :                 return opt_result::failure_at (vect_location,
    5981              :                                                "SLP build failed.\n");
    5982              :               }
    5983              :           }
    5984              : 
    5985              :       stmt_vec_info stmt_info;
    5986       533385 :       FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
    5987              :         {
    5988           30 :           vec<stmt_vec_info> stmts;
    5989           30 :           vec<stmt_vec_info> roots = vNULL;
    5990           30 :           vec<tree> remain = vNULL;
    5991           30 :           stmts.create (1);
    5992           30 :           stmts.quick_push (stmt_info);
    5993           30 :           if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
    5994              :                                          stmts, roots, remain, max_tree_size,
    5995              :                                          &limit, bst_map, force_single_lane))
    5996              :             {
    5997            0 :               release_scalar_stmts_to_slp_tree_map (bst_map);
    5998            0 :               return opt_result::failure_at (vect_location,
    5999              :                                              "SLP build failed.\n");
    6000              :             }
    6001              :         }
    6002              :     }
    6003              : 
    6004      1210692 :   if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
    6005              :     {
    6006      2058950 :       for (unsigned i = 0; i < bb_vinfo->roots.length (); ++i)
    6007              :         {
    6008      1381583 :           vect_location = bb_vinfo->roots[i].roots[0]->stmt;
    6009              :           /* Apply patterns.  */
    6010      4517605 :           for (unsigned j = 0; j < bb_vinfo->roots[i].stmts.length (); ++j)
    6011      6272044 :             bb_vinfo->roots[i].stmts[j]
    6012      3244637 :               = vect_stmt_to_vectorize (bb_vinfo->roots[i].stmts[j]);
    6013      1381583 :           if (vect_build_slp_instance (bb_vinfo, bb_vinfo->roots[i].kind,
    6014      1381583 :                                        bb_vinfo->roots[i].stmts,
    6015      1381583 :                                        bb_vinfo->roots[i].roots,
    6016      1381583 :                                        bb_vinfo->roots[i].remain,
    6017              :                                        max_tree_size, &limit, bst_map, false))
    6018              :             {
    6019       157787 :               bb_vinfo->roots[i].roots = vNULL;
    6020       157787 :               bb_vinfo->roots[i].remain = vNULL;
    6021              :             }
    6022      1381583 :           bb_vinfo->roots[i].stmts = vNULL;
    6023              :         }
    6024              :     }
    6025              : 
    6026      1210692 :   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
    6027              :     {
    6028              :       /* Find SLP sequences starting from groups of reductions.  */
    6029       533325 :       if (!vect_analyze_slp_reductions (loop_vinfo, max_tree_size, &limit,
    6030              :                                         bst_map, force_single_lane))
    6031              :         {
    6032         1921 :           release_scalar_stmts_to_slp_tree_map (bst_map);
    6033         1921 :           return opt_result::failure_at (vect_location, "SLP build failed.\n");
    6034              :         }
    6035              : 
    6036              :       /* Make sure to vectorize only-live stmts, usually inductions.  */
    6037      2394018 :       for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
    6038      1556592 :         for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
    6039       746064 :              gsi_next (&gsi))
    6040              :           {
    6041       756786 :             gphi *lc_phi = *gsi;
    6042       756786 :             tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
    6043       756786 :             stmt_vec_info stmt_info;
    6044       756786 :             if (TREE_CODE (def) == SSA_NAME
    6045       639361 :                 && !virtual_operand_p (def)
    6046       331262 :                 && (stmt_info = loop_vinfo->lookup_def (def))
    6047       294180 :                 && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
    6048       294180 :                 && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
    6049       226323 :                 && STMT_VINFO_LIVE_P (stmt_info)
    6050       226323 :                 && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
    6051       877700 :                 && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
    6052              :               {
    6053       120826 :                 vec<stmt_vec_info> stmts;
    6054       120826 :                 vec<stmt_vec_info> roots = vNULL;
    6055       120826 :                 vec<tree> remain = vNULL;
    6056       120826 :                 stmts.create (1);
    6057       120826 :                 stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
    6058       120826 :                 if (! vect_build_slp_instance (vinfo,
    6059              :                                                slp_inst_kind_reduc_group,
    6060              :                                                stmts, roots, remain,
    6061              :                                                max_tree_size, &limit,
    6062              :                                                bst_map, force_single_lane))
    6063              :                   {
    6064        10722 :                     release_scalar_stmts_to_slp_tree_map (bst_map);
    6065        10722 :                     return opt_result::failure_at (vect_location,
    6066              :                                                    "SLP build failed.\n");
    6067              :                   }
    6068              :               }
    6069        10722 :           }
    6070              : 
    6071              :       /* Find SLP sequences starting from gconds.  */
    6072      1313719 :       for (auto cond : LOOP_VINFO_LOOP_CONDS (loop_vinfo))
    6073              :         {
    6074       306884 :           auto cond_info = loop_vinfo->lookup_stmt (cond);
    6075              : 
    6076       306884 :           cond_info = vect_stmt_to_vectorize (cond_info);
    6077       306884 :           vec<stmt_vec_info> roots = vNULL;
    6078       306884 :           roots.safe_push (cond_info);
    6079       306884 :           gimple *stmt = STMT_VINFO_STMT (cond_info);
    6080       306884 :           tree args0 = gimple_cond_lhs (stmt);
    6081       306884 :           tree args1 = gimple_cond_rhs (stmt);
    6082              : 
    6083              :           /* These should be enforced by cond lowering, but if it failed
    6084              :              bail.  */
    6085       306884 :           if (gimple_cond_code (stmt) != NE_EXPR
    6086       305725 :               || TREE_TYPE (args0) != boolean_type_node
    6087       612004 :               || !integer_zerop (args1))
    6088              :             {
    6089         1764 :               roots.release ();
    6090         1764 :               release_scalar_stmts_to_slp_tree_map (bst_map);
    6091         1764 :               return opt_result::failure_at (vect_location,
    6092              :                                              "SLP build failed.\n");
    6093              :             }
    6094              : 
    6095              :           /* An argument without a loop def will be codegened from vectorizing the
    6096              :              root gcond itself.  As such we don't need to try to build an SLP tree
    6097              :              from them.  It's highly likely that the resulting SLP tree here if both
    6098              :              arguments have a def will be incompatible, but we rely on it being split
    6099              :              later on.  */
    6100       305120 :           auto varg = loop_vinfo->lookup_def (args0);
    6101       305120 :           vec<stmt_vec_info> stmts;
    6102       305120 :           vec<tree> remain = vNULL;
    6103       305120 :           stmts.create (1);
    6104       305120 :           stmts.quick_push (vect_stmt_to_vectorize (varg));
    6105              : 
    6106       305120 :           if (! vect_build_slp_instance (vinfo, slp_inst_kind_gcond,
    6107              :                                          stmts, roots, remain,
    6108              :                                          max_tree_size, &limit,
    6109              :                                          bst_map, force_single_lane))
    6110              :             {
    6111          925 :               roots.release ();
    6112          925 :               release_scalar_stmts_to_slp_tree_map (bst_map);
    6113          925 :               return opt_result::failure_at (vect_location,
    6114              :                                              "SLP build failed.\n");
    6115              :             }
    6116              :         }
    6117              :     }
    6118              : 
    6119      1195360 :   hash_set<slp_tree> visited_patterns;
    6120      1195360 :   slp_tree_to_load_perm_map_t perm_cache;
    6121      1195360 :   slp_compat_nodes_map_t compat_cache;
    6122              : 
    6123              :   /* See if any patterns can be found in the SLP tree.  */
    6124      1195360 :   bool pattern_found = false;
    6125      4051101 :   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
    6126      1660381 :     pattern_found |= vect_match_slp_patterns (instance, vinfo,
    6127              :                                               &visited_patterns, &perm_cache,
    6128              :                                               &compat_cache);
    6129              : 
    6130              :   /* If any were found optimize permutations of loads.  */
    6131      1195360 :   if (pattern_found)
    6132              :     {
    6133          267 :       hash_map<slp_tree, slp_tree> load_map;
    6134         4981 :       FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
    6135              :         {
    6136         4180 :           slp_tree root = SLP_INSTANCE_TREE (instance);
    6137         4180 :           optimize_load_redistribution (bst_map, vinfo, SLP_TREE_LANES (root),
    6138              :                                         &load_map, root);
    6139              :         }
    6140          267 :     }
    6141              : 
    6142              :   /* Check whether we should force some SLP instances to use load/store-lanes
    6143              :      and do so by forcing SLP re-discovery with single lanes.  We used
    6144              :      to cancel SLP when this applied to all instances in a loop but now
    6145              :      we decide this per SLP instance.  It's important to do this only
    6146              :      after SLP pattern recognition.  */
    6147      1195360 :   if (is_a <loop_vec_info> (vinfo))
    6148      1343893 :     FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
    6149       825900 :       if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
    6150       311628 :           && !SLP_INSTANCE_TREE (instance)->ldst_lanes)
    6151              :         {
    6152       311628 :           slp_tree slp_root = SLP_INSTANCE_TREE (instance);
    6153       311628 :           unsigned int group_size = SLP_TREE_LANES (slp_root);
    6154       311628 :           tree vectype = SLP_TREE_VECTYPE (slp_root);
    6155              : 
    6156       311628 :           stmt_vec_info rep_info = SLP_TREE_REPRESENTATIVE (slp_root);
    6157       311628 :           gimple *rep = STMT_VINFO_STMT (rep_info);
    6158       311628 :           bool masked = (is_gimple_call (rep)
    6159         2493 :                          && gimple_call_internal_p (rep)
    6160       314091 :                          && internal_fn_mask_index
    6161         2463 :                               (gimple_call_internal_fn (rep)) != -1);
    6162       311598 :           if (!STMT_VINFO_GROUPED_ACCESS (rep_info)
    6163        33877 :               || slp_root->ldst_lanes
    6164       345505 :               || (vect_store_lanes_supported (vectype, group_size, masked)
    6165              :                   == IFN_LAST))
    6166       311628 :             continue;
    6167              : 
    6168            0 :           auto_vec<slp_tree> loads;
    6169            0 :           hash_set<slp_tree> visited;
    6170            0 :           vect_gather_slp_loads (loads, slp_root, visited);
    6171              : 
    6172              :           /* Check whether any load in the SLP instance is possibly
    6173              :              permuted.  */
    6174            0 :           bool loads_permuted = false;
    6175            0 :           slp_tree load_node;
    6176            0 :           unsigned j;
    6177            0 :           FOR_EACH_VEC_ELT (loads, j, load_node)
    6178              :             {
    6179            0 :               if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
    6180            0 :                 continue;
    6181            0 :               for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
    6182            0 :                 if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
    6183              :                   {
    6184              :                     loads_permuted = true;
    6185              :                     break;
    6186              :                   }
    6187              :             }
    6188              : 
    6189              :           /* If the loads and stores can use load/store-lanes force re-discovery
    6190              :              with single lanes.  */
    6191            0 :           if (loads_permuted)
    6192              :             {
    6193            0 :               bool can_use_lanes = true;
    6194              :               bool prefer_load_lanes = false;
    6195            0 :               FOR_EACH_VEC_ELT (loads, j, load_node)
    6196            0 :                 if (STMT_VINFO_GROUPED_ACCESS
    6197              :                       (SLP_TREE_REPRESENTATIVE (load_node)))
    6198              :                   {
    6199            0 :                     stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
    6200              :                         (SLP_TREE_REPRESENTATIVE (load_node));
    6201            0 :                     rep = STMT_VINFO_STMT (stmt_vinfo);
    6202            0 :                     masked = (is_gimple_call (rep)
    6203            0 :                               && gimple_call_internal_p (rep)
    6204            0 :                               && internal_fn_mask_index
    6205            0 :                                    (gimple_call_internal_fn (rep)));
    6206              :                     /* Use SLP for strided accesses (or if we can't
    6207              :                        load-lanes).  */
    6208            0 :                     if (STMT_VINFO_STRIDED_P (stmt_vinfo)
    6209            0 :                         || compare_step_with_zero (vinfo, stmt_vinfo) <= 0
    6210            0 :                         || vect_load_lanes_supported
    6211            0 :                              (SLP_TREE_VECTYPE (load_node),
    6212            0 :                               DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
    6213              :                         /* ???  During SLP re-discovery with a single lane
    6214              :                            a masked grouped load will appear permuted and
    6215              :                            discovery will fail.  We have to rework this
    6216              :                            on the discovery side - for now avoid ICEing.  */
    6217            0 :                         || masked)
    6218              :                       {
    6219              :                         can_use_lanes = false;
    6220              :                         break;
    6221              :                       }
    6222              :                     /* Make sure that the target would prefer store-lanes
    6223              :                        for at least one of the loads.
    6224              : 
    6225              :                        ??? Perhaps we should instead require this for
    6226              :                        all loads?  */
    6227            0 :                     prefer_load_lanes
    6228              :                       = (prefer_load_lanes
    6229            0 :                          || SLP_TREE_LANES (load_node) == group_size
    6230            0 :                          || (vect_slp_prefer_store_lanes_p
    6231            0 :                              (vinfo, stmt_vinfo,
    6232              :                               SLP_TREE_VECTYPE (load_node), masked,
    6233              :                               group_size, SLP_TREE_LANES (load_node))));
    6234              :                   }
    6235              : 
    6236            0 :               if (can_use_lanes && prefer_load_lanes)
    6237              :                 {
    6238            0 :                   if (dump_enabled_p ())
    6239            0 :                     dump_printf_loc (MSG_NOTE, vect_location,
    6240              :                                      "SLP instance %p can use load/store-lanes,"
    6241              :                                      " re-discovering with single-lanes\n",
    6242              :                                      (void *) instance);
    6243              : 
    6244            0 :                   stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_root);
    6245              : 
    6246            0 :                   vect_free_slp_instance (instance);
    6247            0 :                   limit = max_tree_size;
    6248            0 :                   bool res = vect_analyze_slp_instance (vinfo, bst_map,
    6249              :                                                         stmt_info,
    6250              :                                                         slp_inst_kind_store,
    6251              :                                                         max_tree_size, &limit,
    6252              :                                                         true);
    6253            0 :                   gcc_assert (res);
    6254            0 :                   auto new_inst = LOOP_VINFO_SLP_INSTANCES (vinfo).pop ();
    6255            0 :                   LOOP_VINFO_SLP_INSTANCES (vinfo)[i] = new_inst;
    6256              :                 }
    6257              :             }
    6258            0 :         }
    6259              : 
    6260              :   /* When we end up with load permutations that we cannot possibly handle,
    6261              :      like those requiring three vector inputs, lower them using interleaving
    6262              :      like schemes.  */
    6263      1195360 :   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
    6264              :     {
    6265       517993 :       vect_lower_load_permutations (loop_vinfo, bst_map, force_single_lane);
    6266       517993 :       if (dump_enabled_p ())
    6267              :         {
    6268        22128 :           dump_printf_loc (MSG_NOTE, vect_location,
    6269              :                            "SLP graph after lowering permutations:\n");
    6270        22128 :           hash_set<slp_tree> visited;
    6271        98387 :           FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
    6272        32030 :             vect_print_slp_graph (MSG_NOTE, vect_location,
    6273              :                                   SLP_INSTANCE_TREE (instance), visited);
    6274        22128 :         }
    6275              :     }
    6276              : 
    6277      1195360 :   release_scalar_stmts_to_slp_tree_map (bst_map);
    6278              : 
    6279      1195360 :   if (pattern_found && dump_enabled_p ())
    6280              :     {
    6281           20 :       dump_printf_loc (MSG_NOTE, vect_location,
    6282              :                        "Pattern matched SLP tree\n");
    6283           20 :       hash_set<slp_tree> visited;
    6284          101 :       FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
    6285           41 :         vect_print_slp_graph (MSG_NOTE, vect_location,
    6286              :                               SLP_INSTANCE_TREE (instance), visited);
    6287           20 :     }
    6288              : 
    6289      1195360 :   return opt_result::success ();
    6290      1195360 : }
    6291              : 
    6292              : /* Estimates the cost of inserting layout changes into the SLP graph.
    6293              :    It can also say that the insertion is impossible.  */
    6294              : 
    6295              : struct slpg_layout_cost
    6296              : {
    6297     11501233 :   slpg_layout_cost () = default;
    6298              :   slpg_layout_cost (sreal, bool);
    6299              : 
    6300       521518 :   static slpg_layout_cost impossible () { return { sreal::max (), 0 }; }
    6301      5589299 :   bool is_possible () const { return depth != sreal::max (); }
    6302              : 
    6303              :   bool operator== (const slpg_layout_cost &) const;
    6304              :   bool operator!= (const slpg_layout_cost &) const;
    6305              : 
    6306              :   bool is_better_than (const slpg_layout_cost &, bool) const;
    6307              : 
    6308              :   void add_parallel_cost (const slpg_layout_cost &);
    6309              :   void add_serial_cost (const slpg_layout_cost &);
    6310              :   void split (unsigned int);
    6311              : 
    6312              :   /* The longest sequence of layout changes needed during any traversal
    6313              :      of the partition dag, weighted by execution frequency.
    6314              : 
    6315              :      This is the most important metric when optimizing for speed, since
    6316              :      it helps to ensure that we keep the number of operations on
    6317              :      critical paths to a minimum.  */
    6318              :   sreal depth = 0;
    6319              : 
    6320              :   /* An estimate of the total number of operations needed.  It is weighted by
    6321              :      execution frequency when optimizing for speed but not when optimizing for
    6322              :      size.  In order to avoid double-counting, a node with a fanout of N will
    6323              :      distribute 1/N of its total cost to each successor.
    6324              : 
    6325              :      This is the most important metric when optimizing for size, since
    6326              :      it helps to keep the total number of operations to a minimum,  */
    6327              :   sreal total = 0;
    6328              : };
    6329              : 
    6330              : /* Construct costs for a node with weight WEIGHT.  A higher weight
    6331              :    indicates more frequent execution.  IS_FOR_SIZE is true if we are
    6332              :    optimizing for size rather than speed.  */
    6333              : 
    6334      1324397 : slpg_layout_cost::slpg_layout_cost (sreal weight, bool is_for_size)
    6335      1332428 :   : depth (weight), total (is_for_size && weight > 0 ? 1 : weight)
    6336              : {
    6337      1324397 : }
    6338              : 
    6339              : bool
    6340            0 : slpg_layout_cost::operator== (const slpg_layout_cost &other) const
    6341              : {
    6342            0 :   return depth == other.depth && total == other.total;
    6343              : }
    6344              : 
    6345              : bool
    6346            0 : slpg_layout_cost::operator!= (const slpg_layout_cost &other) const
    6347              : {
    6348            0 :   return !operator== (other);
    6349              : }
    6350              : 
    6351              : /* Return true if these costs are better than OTHER.  IS_FOR_SIZE is
    6352              :    true if we are optimizing for size rather than speed.  */
    6353              : 
    6354              : bool
    6355       321646 : slpg_layout_cost::is_better_than (const slpg_layout_cost &other,
    6356              :                                   bool is_for_size) const
    6357              : {
    6358       321646 :   if (is_for_size)
    6359              :     {
    6360         3477 :       if (total != other.total)
    6361         1720 :         return total < other.total;
    6362         1757 :       return depth < other.depth;
    6363              :     }
    6364              :   else
    6365              :     {
    6366       318169 :       if (depth != other.depth)
    6367       127265 :         return depth < other.depth;
    6368       190904 :       return total < other.total;
    6369              :     }
    6370              : }
    6371              : 
    6372              : /* Increase the costs to account for something with cost INPUT_COST
    6373              :    happening in parallel with the current costs.  */
    6374              : 
    6375              : void
    6376       385083 : slpg_layout_cost::add_parallel_cost (const slpg_layout_cost &input_cost)
    6377              : {
    6378       385083 :   depth = std::max (depth, input_cost.depth);
    6379       385083 :   total += input_cost.total;
    6380       385083 : }
    6381              : 
    6382              : /* Increase the costs to account for something with cost INPUT_COST
    6383              :    happening in series with the current costs.  */
    6384              : 
    6385              : void
    6386      1576581 : slpg_layout_cost::add_serial_cost (const slpg_layout_cost &other)
    6387              : {
    6388      1576581 :   depth += other.depth;
    6389      1576581 :   total += other.total;
    6390      1576581 : }
    6391              : 
    6392              : /* Split the total cost among TIMES successors or predecessors.  */
    6393              : 
    6394              : void
    6395      1261052 : slpg_layout_cost::split (unsigned int times)
    6396              : {
    6397      1261052 :   if (times > 1)
    6398       638429 :     total /= times;
    6399      1261052 : }
    6400              : 
    6401              : /* Information about one node in the SLP graph, for use during
    6402              :    vect_optimize_slp_pass.  */
    6403              : 
    6404              : struct slpg_vertex
    6405              : {
    6406     10738300 :   slpg_vertex (slp_tree node_) : node (node_) {}
    6407              : 
    6408              :   /* The node itself.  */
    6409              :   slp_tree node;
    6410              : 
    6411              :   /* Which partition the node belongs to, or -1 if none.  Nodes outside of
    6412              :      partitions are flexible; they can have whichever layout consumers
    6413              :      want them to have.  */
    6414              :   int partition = -1;
    6415              : 
    6416              :   /* The number of nodes that directly use the result of this one
    6417              :      (i.e. the number of nodes that count this one as a child).  */
    6418              :   unsigned int out_degree = 0;
    6419              : 
    6420              :   /* The execution frequency of the node.  */
    6421              :   sreal weight = 0;
    6422              : 
    6423              :   /* The total execution frequency of all nodes that directly use the
    6424              :      result of this one.  */
    6425              :   sreal out_weight = 0;
    6426              : };
    6427              : 
    6428              : /* Information about one partition of the SLP graph, for use during
    6429              :    vect_optimize_slp_pass.  */
    6430              : 
    6431              : struct slpg_partition_info
    6432              : {
    6433              :   /* The nodes in the partition occupy indices [NODE_BEGIN, NODE_END)
    6434              :      of m_partitioned_nodes.  */
    6435              :   unsigned int node_begin = 0;
    6436              :   unsigned int node_end = 0;
    6437              : 
    6438              :   /* Which layout we've chosen to use for this partition, or -1 if
    6439              :      we haven't picked one yet.  */
    6440              :   int layout = -1;
    6441              : 
    6442              :   /* The number of predecessors and successors in the partition dag.
    6443              :      The predecessors always have lower partition numbers and the
    6444              :      successors always have higher partition numbers.
    6445              : 
    6446              :      Note that the directions of these edges are not necessarily the
    6447              :      same as in the data flow graph.  For example, if an SCC has separate
    6448              :      partitions for an inner loop and an outer loop, the inner loop's
    6449              :      partition will have at least two incoming edges from the outer loop's
    6450              :      partition: one for a live-in value and one for a live-out value.
    6451              :      In data flow terms, one of these edges would also be from the outer loop
    6452              :      to the inner loop, but the other would be in the opposite direction.  */
    6453              :   unsigned int in_degree = 0;
    6454              :   unsigned int out_degree = 0;
    6455              : };
    6456              : 
    6457              : /* Information about the costs of using a particular layout for a
    6458              :    particular partition.  It can also say that the combination is
    6459              :    impossible.  */
    6460              : 
    6461              : struct slpg_partition_layout_costs
    6462              : {
    6463      1631269 :   bool is_possible () const { return internal_cost.is_possible (); }
    6464        63679 :   void mark_impossible () { internal_cost = slpg_layout_cost::impossible (); }
    6465              : 
    6466              :   /* The costs inherited from predecessor partitions.  */
    6467              :   slpg_layout_cost in_cost;
    6468              : 
    6469              :   /* The inherent cost of the layout within the node itself.  For example,
    6470              :      this is nonzero for a load if choosing a particular layout would require
    6471              :      the load to permute the loaded elements.  It is nonzero for a
    6472              :      VEC_PERM_EXPR if the permutation cannot be eliminated or converted
    6473              :      to full-vector moves.  */
    6474              :   slpg_layout_cost internal_cost;
    6475              : 
    6476              :   /* The costs inherited from successor partitions.  */
    6477              :   slpg_layout_cost out_cost;
    6478              : };
    6479              : 
    6480              : /* This class tries to optimize the layout of vectors in order to avoid
    6481              :    unnecessary shuffling.  At the moment, the set of possible layouts are
    6482              :    restricted to bijective permutations.
    6483              : 
    6484              :    The goal of the pass depends on whether we're optimizing for size or
    6485              :    for speed.  When optimizing for size, the goal is to reduce the overall
    6486              :    number of layout changes (including layout changes implied by things
    6487              :    like load permutations).  When optimizing for speed, the goal is to
    6488              :    reduce the maximum latency attributable to layout changes on any
    6489              :    non-cyclical path through the data flow graph.
    6490              : 
    6491              :    For example, when optimizing a loop nest for speed, we will prefer
    6492              :    to make layout changes outside of a loop rather than inside of a loop,
    6493              :    and will prefer to make layout changes in parallel rather than serially,
    6494              :    even if that increases the overall number of layout changes.
    6495              : 
    6496              :    The high-level procedure is:
    6497              : 
    6498              :    (1) Build a graph in which edges go from uses (parents) to definitions
    6499              :        (children).
    6500              : 
    6501              :    (2) Divide the graph into a dag of strongly-connected components (SCCs).
    6502              : 
    6503              :    (3) When optimizing for speed, partition the nodes in each SCC based
    6504              :        on their containing cfg loop.  When optimizing for size, treat
    6505              :        each SCC as a single partition.
    6506              : 
    6507              :        This gives us a dag of partitions.  The goal is now to assign a
    6508              :        layout to each partition.
    6509              : 
    6510              :    (4) Construct a set of vector layouts that are worth considering.
    6511              :        Record which nodes must keep their current layout.
    6512              : 
    6513              :    (5) Perform a forward walk over the partition dag (from loads to stores)
    6514              :        accumulating the "forward" cost of using each layout.  When visiting
    6515              :        each partition, assign a tentative choice of layout to the partition
    6516              :        and use that choice when calculating the cost of using a different
    6517              :        layout in successor partitions.
    6518              : 
    6519              :    (6) Perform a backward walk over the partition dag (from stores to loads),
    6520              :        accumulating the "backward" cost of using each layout.  When visiting
    6521              :        each partition, make a final choice of layout for that partition based
    6522              :        on the accumulated forward costs (from (5)) and backward costs
    6523              :        (from (6)).
    6524              : 
    6525              :    (7) Apply the chosen layouts to the SLP graph.
    6526              : 
    6527              :    For example, consider the SLP statements:
    6528              : 
    6529              :    S1:      a_1 = load
    6530              :        loop:
    6531              :    S2:      a_2 = PHI<a_1, a_3>
    6532              :    S3:      b_1 = load
    6533              :    S4:      a_3 = a_2 + b_1
    6534              :        exit:
    6535              :    S5:      a_4 = PHI<a_3>
    6536              :    S6:      store a_4
    6537              : 
    6538              :    S2 and S4 form an SCC and are part of the same loop.  Every other
    6539              :    statement is in a singleton SCC.  In this example there is a one-to-one
    6540              :    mapping between SCCs and partitions and the partition dag looks like this;
    6541              : 
    6542              :         S1     S3
    6543              :          \     /
    6544              :           S2+S4
    6545              :             |
    6546              :            S5
    6547              :             |
    6548              :            S6
    6549              : 
    6550              :    S2, S3 and S4 will have a higher execution frequency than the other
    6551              :    statements, so when optimizing for speed, the goal is to avoid any
    6552              :    layout changes:
    6553              : 
    6554              :    - within S3
    6555              :    - within S2+S4
    6556              :    - on the S3->S2+S4 edge
    6557              : 
    6558              :    For example, if S3 was originally a reversing load, the goal of the
    6559              :    pass is to make it an unreversed load and change the layout on the
    6560              :    S1->S2+S4 and S2+S4->S5 edges to compensate.  (Changing the layout
    6561              :    on S1->S2+S4 and S5->S6 would also be acceptable.)
    6562              : 
    6563              :    The difference between SCCs and partitions becomes important if we
    6564              :    add an outer loop:
    6565              : 
    6566              :    S1:      a_1 = ...
    6567              :        loop1:
    6568              :    S2:      a_2 = PHI<a_1, a_6>
    6569              :    S3:      b_1 = load
    6570              :    S4:      a_3 = a_2 + b_1
    6571              :        loop2:
    6572              :    S5:      a_4 = PHI<a_3, a_5>
    6573              :    S6:      c_1 = load
    6574              :    S7:      a_5 = a_4 + c_1
    6575              :        exit2:
    6576              :    S8:      a_6 = PHI<a_5>
    6577              :    S9:      store a_6
    6578              :        exit1:
    6579              : 
    6580              :    Here, S2, S4, S5, S7 and S8 form a single SCC.  However, when optimizing
    6581              :    for speed, we usually do not want restrictions in the outer loop to "infect"
    6582              :    the decision for the inner loop.  For example, if an outer-loop node
    6583              :    in the SCC contains a statement with a fixed layout, that should not
    6584              :    prevent the inner loop from using a different layout.  Conversely,
    6585              :    the inner loop should not dictate a layout to the outer loop: if the
    6586              :    outer loop does a lot of computation, then it may not be efficient to
    6587              :    do all of that computation in the inner loop's preferred layout.
    6588              : 
    6589              :    So when optimizing for speed, we partition the SCC into S2+S4+S8 (outer)
    6590              :    and S5+S7 (inner).  We also try to arrange partitions so that:
    6591              : 
    6592              :    - the partition for an outer loop comes before the partition for
    6593              :      an inner loop
    6594              : 
    6595              :    - if a sibling loop A dominates a sibling loop B, A's partition
    6596              :      comes before B's
    6597              : 
    6598              :    This gives the following partition dag for the example above:
    6599              : 
    6600              :         S1        S3
    6601              :          \        /
    6602              :           S2+S4+S8   S6
    6603              :            |   \\    /
    6604              :            |    S5+S7
    6605              :            |
    6606              :           S9
    6607              : 
    6608              :    There are two edges from S2+S4+S8 to S5+S7: one for the edge S4->S5 and
    6609              :    one for a reversal of the edge S7->S8.
    6610              : 
    6611              :    The backward walk picks a layout for S5+S7 before S2+S4+S8.  The choice
    6612              :    for S2+S4+S8 therefore has to balance the cost of using the outer loop's
    6613              :    preferred layout against the cost of changing the layout on entry to the
    6614              :    inner loop (S4->S5) and on exit from the inner loop (S7->S8 reversed).
    6615              : 
    6616              :    Although this works well when optimizing for speed, it has the downside
    6617              :    when optimizing for size that the choice of layout for S5+S7 is completely
    6618              :    independent of S9, which lessens the chance of reducing the overall number
    6619              :    of permutations.  We therefore do not partition SCCs when optimizing
    6620              :    for size.
    6621              : 
    6622              :    To give a concrete example of the difference between optimizing
    6623              :    for size and speed, consider:
    6624              : 
    6625              :    a[0] = (b[1] << c[3]) - d[1];
    6626              :    a[1] = (b[0] << c[2]) - d[0];
    6627              :    a[2] = (b[3] << c[1]) - d[3];
    6628              :    a[3] = (b[2] << c[0]) - d[2];
    6629              : 
    6630              :    There are three different layouts here: one for a, one for b and d,
    6631              :    and one for c.  When optimizing for speed it is better to permute each
    6632              :    of b, c and d into the order required by a, since those permutations
    6633              :    happen in parallel.  But when optimizing for size, it is better to:
    6634              : 
    6635              :    - permute c into the same order as b
    6636              :    - do the arithmetic
    6637              :    - permute the result into the order required by a
    6638              : 
    6639              :    This gives 2 permutations rather than 3.  */
    6640              : 
    6641              : class vect_optimize_slp_pass
    6642              : {
    6643              : public:
    6644       732178 :   vect_optimize_slp_pass (vec_info *vinfo) : m_vinfo (vinfo) {}
    6645              :   void run ();
    6646              : 
    6647              : private:
    6648              :   /* Graph building.  */
    6649              :   struct loop *containing_loop (slp_tree);
    6650              :   bool is_cfg_latch_edge (graph_edge *);
    6651              :   void build_vertices (hash_set<slp_tree> &, slp_tree);
    6652              :   void build_vertices ();
    6653              :   void build_graph ();
    6654              : 
    6655              :   /* Partitioning.  */
    6656              :   void create_partitions ();
    6657              :   template<typename T> void for_each_partition_edge (unsigned int, T);
    6658              : 
    6659              :   /* Layout selection.  */
    6660              :   bool is_compatible_layout (slp_tree, unsigned int);
    6661              :   bool is_compatible_layout (const slpg_partition_info &, unsigned int);
    6662              :   int change_layout_cost (slp_tree, unsigned int, unsigned int);
    6663              :   slpg_partition_layout_costs &partition_layout_costs (unsigned int,
    6664              :                                                        unsigned int);
    6665              :   void change_vec_perm_layout (slp_tree, lane_permutation_t &,
    6666              :                                int, unsigned int);
    6667              :   int internal_node_cost (slp_tree, int, unsigned int);
    6668              :   void start_choosing_layouts ();
    6669              :   bool legitimize ();
    6670              : 
    6671              :   /* Cost propagation.  */
    6672              :   slpg_layout_cost edge_layout_cost (graph_edge *, unsigned int,
    6673              :                                      unsigned int, unsigned int);
    6674              :   slpg_layout_cost total_in_cost (unsigned int);
    6675              :   slpg_layout_cost forward_cost (graph_edge *, unsigned int, unsigned int);
    6676              :   slpg_layout_cost backward_cost (graph_edge *, unsigned int, unsigned int);
    6677              :   void forward_pass ();
    6678              :   void backward_pass ();
    6679              : 
    6680              :   /* Rematerialization.  */
    6681              :   slp_tree get_result_with_layout (slp_tree, unsigned int);
    6682              :   void materialize ();
    6683              : 
    6684              :   /* Clean-up.  */
    6685              :   void remove_redundant_permutations ();
    6686              : 
    6687              :   /* Masked load lanes discovery.  */
    6688              :   void decide_masked_load_lanes ();
    6689              : 
    6690              :   void dump ();
    6691              : 
    6692              :   vec_info *m_vinfo;
    6693              : 
    6694              :   /* True if we should optimize the graph for size, false if we should
    6695              :      optimize it for speed.  (It wouldn't be easy to make this decision
    6696              :      more locally.)  */
    6697              :   bool m_optimize_size;
    6698              : 
    6699              :   /* A graph of all SLP nodes, with edges leading from uses to definitions.
    6700              :      In other words, a node's predecessors are its slp_tree parents and
    6701              :      a node's successors are its slp_tree children.  */
    6702              :   graph *m_slpg = nullptr;
    6703              : 
    6704              :   /* The vertices of M_SLPG, indexed by slp_tree::vertex.  */
    6705              :   auto_vec<slpg_vertex> m_vertices;
    6706              : 
    6707              :   /* The list of all leaves of M_SLPG. such as external definitions, constants,
    6708              :      and loads.  */
    6709              :   auto_vec<int> m_leafs;
    6710              : 
    6711              :   /* This array has one entry for every vector layout that we're considering.
    6712              :      Element 0 is null and indicates "no change".  Other entries describe
    6713              :      permutations that are inherent in the current graph and that we would
    6714              :      like to reverse if possible.
    6715              : 
    6716              :      For example, a permutation { 1, 2, 3, 0 } means that something has
    6717              :      effectively been permuted in that way, such as a load group
    6718              :      { a[1], a[2], a[3], a[0] } (viewed as a permutation of a[0:3]).
    6719              :      We'd then like to apply the reverse permutation { 3, 0, 1, 2 }
    6720              :      in order to put things "back" in order.  */
    6721              :   auto_vec<vec<unsigned> > m_perms;
    6722              : 
    6723              :   /* A partitioning of the nodes for which a layout must be chosen.
    6724              :      Each partition represents an <SCC, cfg loop> pair; that is,
    6725              :      nodes in different SCCs belong to different partitions, and nodes
    6726              :      within an SCC can be further partitioned according to a containing
    6727              :      cfg loop.  Partition <SCC1, L1> comes before <SCC2, L2> if:
    6728              : 
    6729              :      - SCC1 != SCC2 and SCC1 is a predecessor of SCC2 in a forward walk
    6730              :        from leaves (such as loads) to roots (such as stores).
    6731              : 
    6732              :      - SCC1 == SCC2 and L1's header strictly dominates L2's header.  */
    6733              :   auto_vec<slpg_partition_info> m_partitions;
    6734              : 
    6735              :   /* The list of all nodes for which a layout must be chosen.  Nodes for
    6736              :      partition P come before the nodes for partition P+1.  Nodes within a
    6737              :      partition are in reverse postorder.  */
    6738              :   auto_vec<unsigned int> m_partitioned_nodes;
    6739              : 
    6740              :   /* Index P * num-layouts + L contains the cost of using layout L
    6741              :      for partition P.  */
    6742              :   auto_vec<slpg_partition_layout_costs> m_partition_layout_costs;
    6743              : 
    6744              :   /* Index N * num-layouts + L, if nonnull, is a node that provides the
    6745              :      original output of node N adjusted to have layout L.  */
    6746              :   auto_vec<slp_tree> m_node_layouts;
    6747              : };
    6748              : 
    6749              : /* Fill the vertices and leafs vector with all nodes in the SLP graph.
    6750              :    Also record whether we should optimize anything for speed rather
    6751              :    than size.  */
    6752              : 
    6753              : void
    6754     11600796 : vect_optimize_slp_pass::build_vertices (hash_set<slp_tree> &visited,
    6755              :                                         slp_tree node)
    6756              : {
    6757     11600796 :   unsigned i;
    6758     11600796 :   slp_tree child;
    6759              : 
    6760     11600796 :   if (visited.add (node))
    6761     11600796 :     return;
    6762              : 
    6763     10738300 :   if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
    6764              :     {
    6765      8243192 :       basic_block bb = gimple_bb (vect_orig_stmt (rep)->stmt);
    6766      7248026 :       if (optimize_bb_for_speed_p (bb))
    6767      7118780 :         m_optimize_size = false;
    6768              :     }
    6769              : 
    6770     10738300 :   node->vertex = m_vertices.length ();
    6771     10738300 :   m_vertices.safe_push (slpg_vertex (node));
    6772              : 
    6773     10738300 :   bool leaf = true;
    6774     10738300 :   bool force_leaf = false;
    6775     20161236 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    6776      9422936 :     if (child)
    6777              :       {
    6778      8472028 :         leaf = false;
    6779      8472028 :         build_vertices (visited, child);
    6780              :       }
    6781              :     else
    6782              :       force_leaf = true;
    6783              :   /* Since SLP discovery works along use-def edges all cycles have an
    6784              :      entry - but there's the exception of cycles where we do not handle
    6785              :      the entry explicitly (but with a NULL SLP node), like some reductions
    6786              :      and inductions.  Force those SLP PHIs to act as leafs to make them
    6787              :      backwards reachable.  */
    6788     10738300 :   if (leaf || force_leaf)
    6789      5306898 :     m_leafs.safe_push (node->vertex);
    6790              : }
    6791              : 
    6792              : /* Fill the vertices and leafs vector with all nodes in the SLP graph.  */
    6793              : 
    6794              : void
    6795      1464356 : vect_optimize_slp_pass::build_vertices ()
    6796              : {
    6797      1464356 :   hash_set<slp_tree> visited;
    6798      1464356 :   unsigned i;
    6799      1464356 :   slp_instance instance;
    6800      1464356 :   m_vertices.truncate (0);
    6801      1464356 :   m_leafs.truncate (0);
    6802      7521836 :   FOR_EACH_VEC_ELT (m_vinfo->slp_instances, i, instance)
    6803      3128768 :     build_vertices (visited, SLP_INSTANCE_TREE (instance));
    6804      1464356 : }
    6805              : 
    6806              : /* Apply (reverse) bijectite PERM to VEC.  */
    6807              : 
    6808              : template <class T>
    6809              : static void
    6810       248360 : vect_slp_permute (vec<unsigned> perm,
    6811              :                   vec<T> &vec, bool reverse)
    6812              : {
    6813       248360 :   auto_vec<T, 64> saved;
    6814       248360 :   saved.create (vec.length ());
    6815       818672 :   for (unsigned i = 0; i < vec.length (); ++i)
    6816       570312 :     saved.quick_push (vec[i]);
    6817              : 
    6818       248360 :   if (reverse)
    6819              :     {
    6820      1617703 :       for (unsigned i = 0; i < vec.length (); ++i)
    6821       568192 :         vec[perm[i]] = saved[i];
    6822       815552 :       for (unsigned i = 0; i < vec.length (); ++i)
    6823       970541 :         gcc_assert (vec[perm[i]] == saved[i]);
    6824              :     }
    6825              :   else
    6826              :     {
    6827         6240 :       for (unsigned i = 0; i < vec.length (); ++i)
    6828         2120 :         vec[i] = saved[perm[i]];
    6829       250480 :       for (unsigned i = 0; i < vec.length (); ++i)
    6830         3180 :         gcc_assert (vec[i] == saved[perm[i]]);
    6831              :     }
    6832       248360 : }
    6833              : 
    6834              : /* Return the cfg loop that contains NODE.  */
    6835              : 
    6836              : struct loop *
    6837      4176324 : vect_optimize_slp_pass::containing_loop (slp_tree node)
    6838              : {
    6839      4176324 :   stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
    6840              :   /* ???  This is imprecise, VEC_PERM nodes do not have a representative
    6841              :      but are laid out close to their children.  */
    6842      4176324 :   if (!rep)
    6843       154289 :     return m_vinfo->bbs[0]->loop_father;
    6844      4533003 :   return gimple_bb (vect_orig_stmt (rep)->stmt)->loop_father;
    6845              : }
    6846              : 
    6847              : /* Return true if UD (an edge from a use to a definition) is associated
    6848              :    with a loop latch edge in the cfg.  */
    6849              : 
    6850              : bool
    6851      8472028 : vect_optimize_slp_pass::is_cfg_latch_edge (graph_edge *ud)
    6852              : {
    6853      8472028 :   slp_tree use = m_vertices[ud->src].node;
    6854      8472028 :   slp_tree def = m_vertices[ud->dest].node;
    6855      8472028 :   if ((SLP_TREE_DEF_TYPE (use) != vect_internal_def
    6856      8472028 :        || SLP_TREE_PERMUTE_P (use))
    6857      8129090 :       || SLP_TREE_DEF_TYPE (def) != vect_internal_def)
    6858              :     return false;
    6859              : 
    6860      4940280 :   stmt_vec_info use_rep = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (use));
    6861      4940280 :   return (is_a<gphi *> (use_rep->stmt)
    6862       385824 :           && bb_loop_header_p (gimple_bb (use_rep->stmt))
    6863      5157034 :           && containing_loop (def) == containing_loop (use));
    6864              : }
    6865              : 
    6866              : /* Build the graph.  Mark edges that correspond to cfg loop latch edges with
    6867              :    a nonnull data field.  */
    6868              : 
    6869              : void
    6870      1464356 : vect_optimize_slp_pass::build_graph ()
    6871              : {
    6872      1464356 :   m_optimize_size = true;
    6873      1464356 :   build_vertices ();
    6874              : 
    6875      2928712 :   m_slpg = new_graph (m_vertices.length ());
    6876     15131368 :   for (slpg_vertex &v : m_vertices)
    6877     32158348 :     for (slp_tree child : SLP_TREE_CHILDREN (v.node))
    6878      9422936 :       if (child)
    6879              :         {
    6880      8472028 :           graph_edge *ud = add_edge (m_slpg, v.node->vertex, child->vertex);
    6881      8472028 :           if (is_cfg_latch_edge (ud))
    6882       206806 :             ud->data = this;
    6883              :         }
    6884      1464356 : }
    6885              : 
    6886              : /* Return true if E corresponds to a loop latch edge in the cfg.  */
    6887              : 
    6888              : static bool
    6889      4338848 : skip_cfg_latch_edges (graph_edge *e)
    6890              : {
    6891      4338848 :   return e->data;
    6892              : }
    6893              : 
    6894              : /* Create the node partitions.  */
    6895              : 
    6896              : void
    6897       732178 : vect_optimize_slp_pass::create_partitions ()
    6898              : {
    6899              :   /* Calculate a postorder of the graph, ignoring edges that correspond
    6900              :      to natural latch edges in the cfg.  Reading the vector from the end
    6901              :      to the beginning gives the reverse postorder.  */
    6902       732178 :   auto_vec<int> initial_rpo;
    6903      1464356 :   graphds_dfs (m_slpg, &m_leafs[0], m_leafs.length (), &initial_rpo,
    6904              :                false, NULL, skip_cfg_latch_edges);
    6905      2196534 :   gcc_assert (initial_rpo.length () == m_vertices.length ());
    6906              : 
    6907              :   /* Calculate the strongly connected components of the graph.  */
    6908       732178 :   auto_vec<int> scc_grouping;
    6909       732178 :   unsigned int num_sccs = graphds_scc (m_slpg, NULL, NULL, &scc_grouping);
    6910              : 
    6911              :   /* Create a new index order in which all nodes from the same SCC are
    6912              :      consecutive.  Use scc_pos to record the index of the first node in
    6913              :      each SCC.  */
    6914       732178 :   auto_vec<unsigned int> scc_pos (num_sccs);
    6915       732178 :   int last_component = -1;
    6916       732178 :   unsigned int node_count = 0;
    6917      7565121 :   for (unsigned int node_i : scc_grouping)
    6918              :     {
    6919      5368587 :       if (last_component != m_slpg->vertices[node_i].component)
    6920              :         {
    6921      5235030 :           last_component = m_slpg->vertices[node_i].component;
    6922     10470060 :           gcc_assert (last_component == int (scc_pos.length ()));
    6923      5235030 :           scc_pos.quick_push (node_count);
    6924              :         }
    6925      5368587 :       node_count += 1;
    6926              :     }
    6927      1464356 :   gcc_assert (node_count == initial_rpo.length ()
    6928              :               && last_component + 1 == int (num_sccs));
    6929              : 
    6930              :   /* Use m_partitioned_nodes to group nodes into SCC order, with the nodes
    6931              :      inside each SCC following the RPO we calculated above.  The fact that
    6932              :      we ignored natural latch edges when calculating the RPO should ensure
    6933              :      that, for natural loop nests:
    6934              : 
    6935              :      - the first node that we encounter in a cfg loop is the loop header phi
    6936              :      - the loop header phis are in dominance order
    6937              : 
    6938              :      Arranging for this is an optimization (see below) rather than a
    6939              :      correctness issue.  Unnatural loops with a tangled mess of backedges
    6940              :      will still work correctly, but might give poorer results.
    6941              : 
    6942              :      Also update scc_pos so that it gives 1 + the index of the last node
    6943              :      in the SCC.  */
    6944       732178 :   m_partitioned_nodes.safe_grow (node_count);
    6945      6832943 :   for (unsigned int old_i = initial_rpo.length (); old_i-- > 0;)
    6946              :     {
    6947      5368587 :       unsigned int node_i = initial_rpo[old_i];
    6948      5368587 :       unsigned int new_i = scc_pos[m_slpg->vertices[node_i].component]++;
    6949      5368587 :       m_partitioned_nodes[new_i] = node_i;
    6950              :     }
    6951              : 
    6952              :   /* When optimizing for speed, partition each SCC based on the containing
    6953              :      cfg loop. The order we constructed above should ensure that, for natural
    6954              :      cfg loops, we'll create sub-SCC partitions for outer loops before
    6955              :      the corresponding sub-SCC partitions for inner loops.  Similarly,
    6956              :      when one sibling loop A dominates another sibling loop B, we should
    6957              :      create a sub-SCC partition for A before a sub-SCC partition for B.
    6958              : 
    6959              :      As above, nothing depends for correctness on whether this achieves
    6960              :      a natural nesting, but we should get better results when it does.  */
    6961      1464356 :   m_partitions.reserve (m_vertices.length ());
    6962       732178 :   unsigned int next_partition_i = 0;
    6963       732178 :   hash_map<struct loop *, int> loop_partitions;
    6964       732178 :   unsigned int rpo_begin = 0;
    6965       732178 :   unsigned int num_partitioned_nodes = 0;
    6966      7431564 :   for (unsigned int rpo_end : scc_pos)
    6967              :     {
    6968      5235030 :       loop_partitions.empty ();
    6969      5235030 :       unsigned int partition_i = next_partition_i;
    6970     10603617 :       for (unsigned int rpo_i = rpo_begin; rpo_i < rpo_end; ++rpo_i)
    6971              :         {
    6972              :           /* Handle externals and constants optimistically throughout.
    6973              :              But treat existing vectors as fixed since we do not handle
    6974              :              permuting them.  */
    6975      5368587 :           unsigned int node_i = m_partitioned_nodes[rpo_i];
    6976      5368587 :           auto &vertex = m_vertices[node_i];
    6977      5368587 :           if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
    6978       533050 :                && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
    6979      5371817 :               || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
    6980      1595615 :             vertex.partition = -1;
    6981              :           else
    6982              :             {
    6983      3772972 :               bool existed;
    6984      3772972 :               if (m_optimize_size)
    6985        30156 :                 existed = next_partition_i > partition_i;
    6986              :               else
    6987              :                 {
    6988      3742816 :                   struct loop *loop = containing_loop (vertex.node);
    6989      3742816 :                   auto &entry = loop_partitions.get_or_insert (loop, &existed);
    6990      3742816 :                   if (!existed)
    6991      3610500 :                     entry = next_partition_i;
    6992      3742816 :                   partition_i = entry;
    6993              :                 }
    6994      3772972 :               if (!existed)
    6995              :                 {
    6996      3640564 :                   m_partitions.quick_push (slpg_partition_info ());
    6997      3640564 :                   next_partition_i += 1;
    6998              :                 }
    6999      3772972 :               vertex.partition = partition_i;
    7000      3772972 :               num_partitioned_nodes += 1;
    7001      3772972 :               m_partitions[partition_i].node_end += 1;
    7002              :             }
    7003              :         }
    7004      5235030 :       rpo_begin = rpo_end;
    7005              :     }
    7006              : 
    7007              :   /* Assign ranges of consecutive node indices to each partition,
    7008              :      in partition order.  Start with node_end being the same as
    7009              :      node_begin so that the next loop can use it as a counter.  */
    7010       732178 :   unsigned int node_begin = 0;
    7011      5837098 :   for (auto &partition : m_partitions)
    7012              :     {
    7013      3640564 :       partition.node_begin = node_begin;
    7014      3640564 :       node_begin += partition.node_end;
    7015      3640564 :       partition.node_end = partition.node_begin;
    7016              :     }
    7017       732178 :   gcc_assert (node_begin == num_partitioned_nodes);
    7018              : 
    7019              :   /* Finally build the list of nodes in partition order.  */
    7020       732178 :   m_partitioned_nodes.truncate (num_partitioned_nodes);
    7021      6832943 :   for (unsigned int node_i = 0; node_i < m_vertices.length (); ++node_i)
    7022              :     {
    7023      5368587 :       int partition_i = m_vertices[node_i].partition;
    7024      5368587 :       if (partition_i >= 0)
    7025              :         {
    7026      3772972 :           unsigned int order_i = m_partitions[partition_i].node_end++;
    7027      3772972 :           m_partitioned_nodes[order_i] = node_i;
    7028              :         }
    7029              :     }
    7030       732178 : }
    7031              : 
    7032              : /* Look for edges from earlier partitions into node NODE_I and edges from
    7033              :    node NODE_I into later partitions.  Call:
    7034              : 
    7035              :       FN (ud, other_node_i)
    7036              : 
    7037              :    for each such use-to-def edge ud, where other_node_i is the node at the
    7038              :    other end of the edge.  */
    7039              : 
    7040              : template<typename T>
    7041              : void
    7042      4246650 : vect_optimize_slp_pass::for_each_partition_edge (unsigned int node_i, T fn)
    7043              : {
    7044      4246650 :   int partition_i = m_vertices[node_i].partition;
    7045      4246650 :   for (graph_edge *pred = m_slpg->vertices[node_i].pred;
    7046      7357288 :        pred; pred = pred->pred_next)
    7047              :     {
    7048      3110638 :       int src_partition_i = m_vertices[pred->src].partition;
    7049      3110638 :       if (src_partition_i >= 0 && src_partition_i != partition_i)
    7050      2759242 :         fn (pred, pred->src);
    7051              :     }
    7052      4246650 :   for (graph_edge *succ = m_slpg->vertices[node_i].succ;
    7053      9133155 :        succ; succ = succ->succ_next)
    7054              :     {
    7055      4886505 :       int dest_partition_i = m_vertices[succ->dest].partition;
    7056      4886505 :       if (dest_partition_i >= 0 && dest_partition_i != partition_i)
    7057      2803005 :         fn (succ, succ->dest);
    7058              :     }
    7059      4246650 : }
    7060              : 
    7061              : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
    7062              :    that NODE would operate on.  This test is independent of NODE's actual
    7063              :    operation.  */
    7064              : 
    7065              : bool
    7066      1783211 : vect_optimize_slp_pass::is_compatible_layout (slp_tree node,
    7067              :                                               unsigned int layout_i)
    7068              : {
    7069      1783211 :   if (layout_i == 0)
    7070              :     return true;
    7071              : 
    7072      1087766 :   if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
    7073        11080 :     return false;
    7074              : 
    7075              :   return true;
    7076              : }
    7077              : 
    7078              : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
    7079              :    that NODE would operate on for each NODE in PARTITION.
    7080              :    This test is independent of NODE's actual operations.  */
    7081              : 
    7082              : bool
    7083        25822 : vect_optimize_slp_pass::is_compatible_layout (const slpg_partition_info
    7084              :                                                 &partition,
    7085              :                                               unsigned int layout_i)
    7086              : {
    7087        52077 :   for (unsigned int order_i = partition.node_begin;
    7088        52077 :        order_i < partition.node_end; ++order_i)
    7089              :     {
    7090        26357 :       unsigned int node_i = m_partitioned_nodes[order_i];
    7091        26357 :       auto &vertex = m_vertices[node_i];
    7092              : 
    7093              :       /* The layout is incompatible if it is individually incompatible
    7094              :          with any node in the partition.  */
    7095        26357 :       if (!is_compatible_layout (vertex.node, layout_i))
    7096              :         return false;
    7097              :     }
    7098              :   return true;
    7099              : }
    7100              : 
    7101              : /* Return the cost (in arbitrary units) of going from layout FROM_LAYOUT_I
    7102              :    to layout TO_LAYOUT_I for a node like NODE.  Return -1 if either of the
    7103              :    layouts is incompatible with NODE or if the change is not possible for
    7104              :    some other reason.
    7105              : 
    7106              :    The properties taken from NODE include the number of lanes and the
    7107              :    vector type.  The actual operation doesn't matter.  */
    7108              : 
    7109              : int
    7110       752488 : vect_optimize_slp_pass::change_layout_cost (slp_tree node,
    7111              :                                             unsigned int from_layout_i,
    7112              :                                             unsigned int to_layout_i)
    7113              : {
    7114       752488 :   if (!is_compatible_layout (node, from_layout_i)
    7115       752488 :       || !is_compatible_layout (node, to_layout_i))
    7116              :     return -1;
    7117              : 
    7118       751807 :   if (from_layout_i == to_layout_i)
    7119              :     return 0;
    7120              : 
    7121       329683 :   auto_vec<slp_tree, 1> children (1);
    7122       329683 :   children.quick_push (node);
    7123       329683 :   auto_lane_permutation_t perm (SLP_TREE_LANES (node));
    7124       329683 :   if (from_layout_i > 0)
    7125       905950 :     for (unsigned int i : m_perms[from_layout_i])
    7126       400897 :       perm.quick_push ({ 0, i });
    7127              :   else
    7128       525543 :     for (unsigned int i = 0; i < SLP_TREE_LANES (node); ++i)
    7129       364211 :       perm.quick_push ({ 0, i });
    7130       329683 :   if (to_layout_i > 0)
    7131       162081 :     vect_slp_permute (m_perms[to_layout_i], perm, true);
    7132       329683 :   auto count = vectorizable_slp_permutation_1 (m_vinfo, nullptr, node, perm,
    7133              :                                                children, false);
    7134       329683 :   if (count >= 0)
    7135       324338 :     return MAX (count, 1);
    7136              : 
    7137              :   /* ??? In principle we could try changing via layout 0, giving two
    7138              :      layout changes rather than 1.  Doing that would require
    7139              :      corresponding support in get_result_with_layout.  */
    7140              :   return -1;
    7141       329683 : }
    7142              : 
    7143              : /* Return the costs of assigning layout LAYOUT_I to partition PARTITION_I.  */
    7144              : 
    7145              : inline slpg_partition_layout_costs &
    7146      1114787 : vect_optimize_slp_pass::partition_layout_costs (unsigned int partition_i,
    7147              :                                                 unsigned int layout_i)
    7148              : {
    7149      2229574 :   return m_partition_layout_costs[partition_i * m_perms.length () + layout_i];
    7150              : }
    7151              : 
    7152              : /* Change PERM in one of two ways:
    7153              : 
    7154              :    - if IN_LAYOUT_I < 0, accept input operand I in the layout that has been
    7155              :      chosen for child I of NODE.
    7156              : 
    7157              :    - if IN_LAYOUT >= 0, accept all inputs operands with that layout.
    7158              : 
    7159              :    In both cases, arrange for the output to have layout OUT_LAYOUT_I  */
    7160              : 
    7161              : void
    7162        43878 : vect_optimize_slp_pass::
    7163              : change_vec_perm_layout (slp_tree node, lane_permutation_t &perm,
    7164              :                         int in_layout_i, unsigned int out_layout_i)
    7165              : {
    7166       252696 :   for (auto &entry : perm)
    7167              :     {
    7168       121062 :       int this_in_layout_i = in_layout_i;
    7169       121062 :       if (this_in_layout_i < 0)
    7170              :         {
    7171        78867 :           slp_tree in_node = SLP_TREE_CHILDREN (node)[entry.first];
    7172        78867 :           unsigned int in_partition_i = m_vertices[in_node->vertex].partition;
    7173        78867 :           if (in_partition_i == -1u)
    7174          329 :             continue;
    7175        78538 :           this_in_layout_i = m_partitions[in_partition_i].layout;
    7176              :         }
    7177       120733 :       if (this_in_layout_i > 0)
    7178        28645 :         entry.second = m_perms[this_in_layout_i][entry.second];
    7179              :     }
    7180        43878 :   if (out_layout_i > 0)
    7181        12531 :     vect_slp_permute (m_perms[out_layout_i], perm, true);
    7182        43878 : }
    7183              : 
    7184              : /* Check whether the target allows NODE to be rearranged so that the node's
    7185              :    output has layout OUT_LAYOUT_I.  Return the cost of the change if so,
    7186              :    in the same arbitrary units as for change_layout_cost.  Return -1 otherwise.
    7187              : 
    7188              :    If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I < 0, also check whether
    7189              :    NODE can adapt to the layout changes that have (perhaps provisionally)
    7190              :    been chosen for NODE's children, so that no extra permutations are
    7191              :    needed on either the input or the output of NODE.
    7192              : 
    7193              :    If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I >= 0, instead assume
    7194              :    that all inputs will be forced into layout IN_LAYOUT_I beforehand.
    7195              : 
    7196              :    IN_LAYOUT_I has no meaning for other types of node.
    7197              : 
    7198              :    Keeping the node as-is is always valid.  If the target doesn't appear
    7199              :    to support the node as-is, but might realistically support other layouts,
    7200              :    then layout 0 instead has the cost of a worst-case permutation.  On the
    7201              :    one hand, this ensures that every node has at least one valid layout,
    7202              :    avoiding what would otherwise be an awkward special case.  On the other,
    7203              :    it still encourages the pass to change an invalid pre-existing layout
    7204              :    choice into a valid one.  */
    7205              : 
    7206              : int
    7207       253121 : vect_optimize_slp_pass::internal_node_cost (slp_tree node, int in_layout_i,
    7208              :                                             unsigned int out_layout_i)
    7209              : {
    7210       253121 :   const int fallback_cost = 1;
    7211              : 
    7212       253121 :   if (SLP_TREE_PERMUTE_P (node))
    7213              :     {
    7214        35564 :       auto_lane_permutation_t tmp_perm;
    7215        35564 :       tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
    7216              : 
    7217              :       /* Check that the child nodes support the chosen layout.  Checking
    7218              :          the first child is enough, since any second child would have the
    7219              :          same shape.  */
    7220        35564 :       auto first_child = SLP_TREE_CHILDREN (node)[0];
    7221        35564 :       if (in_layout_i > 0
    7222        35564 :           && !is_compatible_layout (first_child, in_layout_i))
    7223              :         return -1;
    7224              : 
    7225        34888 :       change_vec_perm_layout (node, tmp_perm, in_layout_i, out_layout_i);
    7226        69776 :       int count = vectorizable_slp_permutation_1 (m_vinfo, nullptr,
    7227              :                                                   node, tmp_perm,
    7228        34888 :                                                   SLP_TREE_CHILDREN (node),
    7229              :                                                   false);
    7230        34888 :       if (count < 0)
    7231              :         {
    7232         2585 :           if (in_layout_i == 0 && out_layout_i == 0)
    7233              :             {
    7234              :               /* Use the fallback cost if the node could in principle support
    7235              :                  some nonzero layout for both the inputs and the outputs.
    7236              :                  Otherwise assume that the node will be rejected later
    7237              :                  and rebuilt from scalars.  */
    7238          644 :               if (SLP_TREE_LANES (node) == SLP_TREE_LANES (first_child))
    7239              :                 return fallback_cost;
    7240          355 :               return 0;
    7241              :             }
    7242              :           return -1;
    7243              :         }
    7244              : 
    7245              :       /* We currently have no way of telling whether the new layout is cheaper
    7246              :          or more expensive than the old one.  But at least in principle,
    7247              :          it should be worth making zero permutations (whole-vector shuffles)
    7248              :          cheaper than real permutations, in case the pass is able to remove
    7249              :          the latter.  */
    7250        32303 :       return count == 0 ? 0 : 1;
    7251        35564 :     }
    7252              : 
    7253       217557 :   stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
    7254       217557 :   if (rep
    7255       215640 :       && STMT_VINFO_DATA_REF (rep)
    7256        72998 :       && DR_IS_READ (STMT_VINFO_DATA_REF (rep))
    7257       264348 :       && SLP_TREE_LOAD_PERMUTATION (node).exists ())
    7258              :     {
    7259        38354 :       auto_load_permutation_t tmp_perm;
    7260        38354 :       tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
    7261        38354 :       if (out_layout_i > 0)
    7262        15037 :         vect_slp_permute (m_perms[out_layout_i], tmp_perm, true);
    7263              : 
    7264        38354 :       poly_uint64 vf = 1;
    7265        38354 :       if (auto loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
    7266        11360 :         vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    7267        38354 :       unsigned int n_perms;
    7268        38354 :       if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
    7269              :                                            nullptr, vf, true, false, &n_perms))
    7270              :         {
    7271         1998 :           auto rep = SLP_TREE_REPRESENTATIVE (node);
    7272         1998 :           if (out_layout_i == 0)
    7273              :             {
    7274              :               /* Use the fallback cost if the load is an N-to-N permutation.
    7275              :                  Otherwise assume that the node will be rejected later
    7276              :                  and rebuilt from scalars.  */
    7277         1472 :               if (STMT_VINFO_GROUPED_ACCESS (rep)
    7278         2944 :                   && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
    7279         1472 :                       == SLP_TREE_LANES (node)))
    7280          668 :                 return fallback_cost;
    7281              :               return 0;
    7282              :             }
    7283              :           return -1;
    7284              :         }
    7285              : 
    7286              :       /* See the comment above the corresponding VEC_PERM_EXPR handling.  */
    7287        36356 :       return n_perms == 0 ? 0 : 1;
    7288        38354 :     }
    7289              : 
    7290              :   return 0;
    7291              : }
    7292              : 
    7293              : /* Decide which element layouts we should consider using.  Calculate the
    7294              :    weights associated with inserting layout changes on partition edges.
    7295              :    Also mark partitions that cannot change layout, by setting their
    7296              :    layout to zero.  */
    7297              : 
    7298              : void
    7299       732178 : vect_optimize_slp_pass::start_choosing_layouts ()
    7300              : {
    7301              :   /* Used to assign unique permutation indices.  */
    7302       732178 :   using perm_hash = unbounded_hashmap_traits<
    7303              :     vec_free_hash_base<int_hash_base<unsigned>>,
    7304              :     int_hash<int, -1, -2>
    7305              :   >;
    7306       732178 :   hash_map<vec<unsigned>, int, perm_hash> layout_ids;
    7307              : 
    7308              :   /* Layout 0 is "no change".  */
    7309       732178 :   m_perms.safe_push (vNULL);
    7310              : 
    7311              :   /* Create layouts from existing permutations.  */
    7312       732178 :   auto_load_permutation_t tmp_perm;
    7313      5969506 :   for (unsigned int node_i : m_partitioned_nodes)
    7314              :     {
    7315              :       /* Leafs also double as entries to the reverse graph.  Allow the
    7316              :          layout of those to be changed.  */
    7317      3772972 :       auto &vertex = m_vertices[node_i];
    7318      3772972 :       auto &partition = m_partitions[vertex.partition];
    7319      3772972 :       if (!m_slpg->vertices[node_i].succ)
    7320       966134 :         partition.layout = 0;
    7321              : 
    7322              :       /* Loads and VEC_PERM_EXPRs are the only things generating permutes.  */
    7323      3772972 :       slp_tree node = vertex.node;
    7324      3772972 :       stmt_vec_info dr_stmt = SLP_TREE_REPRESENTATIVE (node);
    7325      3772972 :       slp_tree child;
    7326      3772972 :       unsigned HOST_WIDE_INT imin, imax = 0;
    7327      3772972 :       bool any_permute = false;
    7328      3772972 :       tmp_perm.truncate (0);
    7329      3772972 :       if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
    7330              :         {
    7331              :           /* If splitting out a SLP_TREE_LANE_PERMUTATION can make the node
    7332              :              unpermuted, record a layout that reverses this permutation.
    7333              : 
    7334              :              We would need more work to cope with loads that are internally
    7335              :              permuted and also have inputs (such as masks for
    7336              :              IFN_MASK_LOADs).  */
    7337       650230 :           gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
    7338       650230 :           if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
    7339              :             {
    7340       462971 :               partition.layout = -1;
    7341      3750645 :               continue;
    7342              :             }
    7343       187259 :           dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
    7344       187259 :           imin = DR_GROUP_SIZE (dr_stmt) + 1;
    7345       187259 :           tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
    7346              :         }
    7347      6124936 :       else if (SLP_TREE_PERMUTE_P (node)
    7348       145724 :                && SLP_TREE_CHILDREN (node).length () == 1
    7349       120548 :                && (child = SLP_TREE_CHILDREN (node)[0])
    7350      3268466 :                && (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (child))
    7351       120548 :                    .is_constant (&imin)))
    7352              :         {
    7353              :           /* If the child has the same vector size as this node,
    7354              :              reversing the permutation can make the permutation a no-op.
    7355              :              In other cases it can change a true permutation into a
    7356              :              full-vector extract.  */
    7357       120548 :           tmp_perm.reserve (SLP_TREE_LANES (node));
    7358       449998 :           for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
    7359       208902 :             tmp_perm.quick_push (SLP_TREE_LANE_PERMUTATION (node)[j].second);
    7360              :         }
    7361              :       else
    7362      3002194 :         continue;
    7363              : 
    7364       830748 :       for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
    7365              :         {
    7366       522941 :           unsigned idx = tmp_perm[j];
    7367       522941 :           imin = MIN (imin, idx);
    7368       522941 :           imax = MAX (imax, idx);
    7369       522941 :           if (idx - tmp_perm[0] != j)
    7370       170097 :             any_permute = true;
    7371              :         }
    7372              :       /* If the span doesn't match we'd disrupt VF computation, avoid
    7373              :          that for now.  */
    7374       307807 :       if (imax - imin + 1 != SLP_TREE_LANES (node))
    7375        93770 :         continue;
    7376              :       /* If there's no permute no need to split one out.  In this case
    7377              :          we can consider turning a load into a permuted load, if that
    7378              :          turns out to be cheaper than alternatives.  */
    7379       214037 :       if (!any_permute)
    7380              :         {
    7381       191552 :           partition.layout = -1;
    7382       191552 :           continue;
    7383              :         }
    7384              : 
    7385              :       /* For now only handle true permutes, like
    7386              :          vect_attempt_slp_rearrange_stmts did.  This allows us to be lazy
    7387              :          when permuting constants and invariants keeping the permute
    7388              :          bijective.  */
    7389        22485 :       auto_sbitmap load_index (SLP_TREE_LANES (node));
    7390        22485 :       bitmap_clear (load_index);
    7391       112915 :       for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
    7392        67945 :         bitmap_set_bit (load_index, tmp_perm[j] - imin);
    7393              :       unsigned j;
    7394        89574 :       for (j = 0; j < SLP_TREE_LANES (node); ++j)
    7395        67247 :         if (!bitmap_bit_p (load_index, j))
    7396              :           break;
    7397        22485 :       if (j != SLP_TREE_LANES (node))
    7398          158 :         continue;
    7399              : 
    7400        22327 :       vec<unsigned> perm = vNULL;
    7401        22327 :       perm.safe_grow (SLP_TREE_LANES (node), true);
    7402       111616 :       for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
    7403        66962 :         perm[j] = tmp_perm[j] - imin;
    7404              : 
    7405        44654 :       if (int (m_perms.length ()) >= param_vect_max_layout_candidates)
    7406              :         {
    7407              :           /* Continue to use existing layouts, but don't add any more.  */
    7408            0 :           int *entry = layout_ids.get (perm);
    7409            0 :           partition.layout = entry ? *entry : 0;
    7410            0 :           perm.release ();
    7411              :         }
    7412              :       else
    7413              :         {
    7414        22327 :           bool existed;
    7415        22327 :           int &layout_i = layout_ids.get_or_insert (perm, &existed);
    7416        22327 :           if (existed)
    7417         7882 :             perm.release ();
    7418              :           else
    7419              :             {
    7420        14445 :               layout_i = m_perms.length ();
    7421        14445 :               m_perms.safe_push (perm);
    7422              :             }
    7423        22327 :           partition.layout = layout_i;
    7424              :         }
    7425        22485 :     }
    7426              : 
    7427              :   /* Initially assume that every layout is possible and has zero cost
    7428              :      in every partition.  */
    7429       732178 :   m_partition_layout_costs.safe_grow_cleared (m_partitions.length ()
    7430      1464356 :                                               * m_perms.length ());
    7431              : 
    7432              :   /* We have to mark outgoing permutations facing non-associating-reduction
    7433              :      graph entries that are not represented as to be materialized.
    7434              :      slp_inst_kind_bb_reduc currently only covers associatable reductions.  */
    7435      3760918 :   for (slp_instance instance : m_vinfo->slp_instances)
    7436      1564384 :     if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor)
    7437              :       {
    7438         7515 :         unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
    7439         7515 :         m_partitions[m_vertices[node_i].partition].layout = 0;
    7440              :       }
    7441      1556869 :     else if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_reduc_chain)
    7442              :       {
    7443          582 :         stmt_vec_info stmt_info
    7444          582 :           = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (instance));
    7445          582 :         vect_reduc_info reduc_info
    7446          582 :           = info_for_reduction (as_a <loop_vec_info> (m_vinfo),
    7447              :                                 SLP_INSTANCE_TREE (instance));
    7448          582 :         if (needs_fold_left_reduction_p (TREE_TYPE
    7449              :                                            (gimple_get_lhs (stmt_info->stmt)),
    7450              :                                          VECT_REDUC_INFO_CODE (reduc_info)))
    7451              :           {
    7452          124 :             unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
    7453          124 :             m_partitions[m_vertices[node_i].partition].layout = 0;
    7454              :           }
    7455              :       }
    7456              : 
    7457              :   /* Check which layouts each node and partition can handle.  Calculate the
    7458              :      weights associated with inserting layout changes on edges.  */
    7459      5969506 :   for (unsigned int node_i : m_partitioned_nodes)
    7460              :     {
    7461      3772972 :       auto &vertex = m_vertices[node_i];
    7462      3772972 :       auto &partition = m_partitions[vertex.partition];
    7463      3772972 :       slp_tree node = vertex.node;
    7464              : 
    7465      3772972 :       vertex.weight = vect_slp_node_weight (m_vinfo, node);
    7466              : 
    7467      3772972 :       if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
    7468              :         {
    7469              :           /* We do not handle stores with a permutation, so all
    7470              :              incoming permutations must have been materialized.
    7471              : 
    7472              :              We also don't handle masked grouped loads, which lack a
    7473              :              permutation vector.  In this case the memory locations
    7474              :              form an implicit second input to the loads, on top of the
    7475              :              explicit mask input, and the memory input's layout cannot
    7476              :              be changed.
    7477              : 
    7478              :              On the other hand, we do support permuting gather loads and
    7479              :              masked gather loads, where each scalar load is independent
    7480              :              of the others.  This can be useful if the address/index input
    7481              :              benefits from permutation.  */
    7482      3624013 :           if (STMT_VINFO_DATA_REF (rep)
    7483      1733113 :               && STMT_VINFO_GROUPED_ACCESS (rep)
    7484      4634454 :               && !SLP_TREE_LOAD_PERMUTATION (node).exists ())
    7485       823182 :             partition.layout = 0;
    7486              : 
    7487              :           /* We cannot change the layout of an operation that is
    7488              :              not independent on lanes.  Note this is an explicit
    7489              :              negative list since that's much shorter than the respective
    7490              :              positive one but it's critical to keep maintaining it.  */
    7491      3624013 :           if (is_gimple_call (STMT_VINFO_STMT (rep)))
    7492        34462 :             switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
    7493              :               {
    7494         1098 :               case CFN_COMPLEX_ADD_ROT90:
    7495         1098 :               case CFN_COMPLEX_ADD_ROT270:
    7496         1098 :               case CFN_COMPLEX_MUL:
    7497         1098 :               case CFN_COMPLEX_MUL_CONJ:
    7498         1098 :               case CFN_VEC_ADDSUB:
    7499         1098 :               case CFN_VEC_FMADDSUB:
    7500         1098 :               case CFN_VEC_FMSUBADD:
    7501         1098 :                 partition.layout = 0;
    7502              :               default:;
    7503              :               }
    7504              :         }
    7505              : 
    7506      8550846 :       auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
    7507              :         {
    7508      4777874 :           auto &other_vertex = m_vertices[other_node_i];
    7509              : 
    7510              :           /* Count the number of edges from earlier partitions and the number
    7511              :              of edges to later partitions.  */
    7512      4777874 :           if (other_vertex.partition < vertex.partition)
    7513      2388937 :             partition.in_degree += 1;
    7514              :           else
    7515      2388937 :             partition.out_degree += 1;
    7516              : 
    7517              :           /* If the current node uses the result of OTHER_NODE_I, accumulate
    7518              :              the effects of that.  */
    7519      4777874 :           if (ud->src == int (node_i))
    7520              :             {
    7521      2388937 :               other_vertex.out_weight += vertex.weight;
    7522      2388937 :               other_vertex.out_degree += 1;
    7523              :             }
    7524      8550846 :         };
    7525      3772972 :       for_each_partition_edge (node_i, process_edge);
    7526              :     }
    7527       732178 : }
    7528              : 
    7529              : /* Return the incoming costs for node NODE_I, assuming that each input keeps
    7530              :    its current (provisional) choice of layout.  The inputs do not necessarily
    7531              :    have the same layout as each other.  */
    7532              : 
    7533              : slpg_layout_cost
    7534         4379 : vect_optimize_slp_pass::total_in_cost (unsigned int node_i)
    7535              : {
    7536         4379 :   auto &vertex = m_vertices[node_i];
    7537         4379 :   slpg_layout_cost cost;
    7538        14497 :   auto add_cost = [&](graph_edge *, unsigned int other_node_i)
    7539              :     {
    7540        10118 :       auto &other_vertex = m_vertices[other_node_i];
    7541        10118 :       if (other_vertex.partition < vertex.partition)
    7542              :         {
    7543         6772 :           auto &other_partition = m_partitions[other_vertex.partition];
    7544        13544 :           auto &other_costs = partition_layout_costs (other_vertex.partition,
    7545         6772 :                                                       other_partition.layout);
    7546         6772 :           slpg_layout_cost this_cost = other_costs.in_cost;
    7547         6772 :           this_cost.add_serial_cost (other_costs.internal_cost);
    7548         6772 :           this_cost.split (other_partition.out_degree);
    7549         6772 :           cost.add_parallel_cost (this_cost);
    7550              :         }
    7551        14497 :     };
    7552         4379 :   for_each_partition_edge (node_i, add_cost);
    7553         4379 :   return cost;
    7554              : }
    7555              : 
    7556              : /* Return the cost of switching between layout LAYOUT1_I (at node NODE1_I)
    7557              :    and layout LAYOUT2_I on cross-partition use-to-def edge UD.  Return
    7558              :    slpg_layout_cost::impossible () if the change isn't possible.  */
    7559              : 
    7560              : slpg_layout_cost
    7561       752488 : vect_optimize_slp_pass::
    7562              : edge_layout_cost (graph_edge *ud, unsigned int node1_i, unsigned int layout1_i,
    7563              :                   unsigned int layout2_i)
    7564              : {
    7565       752488 :   auto &def_vertex = m_vertices[ud->dest];
    7566       752488 :   auto &use_vertex = m_vertices[ud->src];
    7567       752488 :   auto def_layout_i = ud->dest == int (node1_i) ? layout1_i : layout2_i;
    7568       752488 :   auto use_layout_i = ud->dest == int (node1_i) ? layout2_i : layout1_i;
    7569       752488 :   auto factor = change_layout_cost (def_vertex.node, def_layout_i,
    7570              :                                     use_layout_i);
    7571       752488 :   if (factor < 0)
    7572         6026 :     return slpg_layout_cost::impossible ();
    7573              : 
    7574              :   /* We have a choice of putting the layout change at the site of the
    7575              :      definition or at the site of the use.  Prefer the former when
    7576              :      optimizing for size or when the execution frequency of the
    7577              :      definition is no greater than the combined execution frequencies of
    7578              :      the uses.  When putting the layout change at the site of the definition,
    7579              :      divvy up the cost among all consumers.  */
    7580       746462 :   if (m_optimize_size || def_vertex.weight <= def_vertex.out_weight)
    7581              :     {
    7582       704108 :       slpg_layout_cost cost = { def_vertex.weight * factor, m_optimize_size };
    7583       704108 :       cost.split (def_vertex.out_degree);
    7584       704108 :       return cost;
    7585              :     }
    7586        42354 :   return { use_vertex.weight * factor, m_optimize_size };
    7587              : }
    7588              : 
    7589              : /* UD represents a use-def link between FROM_NODE_I and a node in a later
    7590              :    partition; FROM_NODE_I could be the definition node or the use node.
    7591              :    The node at the other end of the link wants to use layout TO_LAYOUT_I.
    7592              :    Return the cost of any necessary fix-ups on edge UD, or return
    7593              :    slpg_layout_cost::impossible () if the change isn't possible.
    7594              : 
    7595              :    At this point, FROM_NODE_I's partition has chosen the cheapest
    7596              :    layout based on the information available so far, but this choice
    7597              :    is only provisional.  */
    7598              : 
    7599              : slpg_layout_cost
    7600       202677 : vect_optimize_slp_pass::forward_cost (graph_edge *ud, unsigned int from_node_i,
    7601              :                                       unsigned int to_layout_i)
    7602              : {
    7603       202677 :   auto &from_vertex = m_vertices[from_node_i];
    7604       202677 :   unsigned int from_partition_i = from_vertex.partition;
    7605       202677 :   slpg_partition_info &from_partition = m_partitions[from_partition_i];
    7606       202677 :   gcc_assert (from_partition.layout >= 0);
    7607              : 
    7608              :   /* First calculate the cost on the assumption that FROM_PARTITION sticks
    7609              :      with its current layout preference.  */
    7610       202677 :   slpg_layout_cost cost = slpg_layout_cost::impossible ();
    7611       202677 :   auto edge_cost = edge_layout_cost (ud, from_node_i,
    7612       202677 :                                      from_partition.layout, to_layout_i);
    7613       202677 :   if (edge_cost.is_possible ())
    7614              :     {
    7615       398990 :       auto &from_costs = partition_layout_costs (from_partition_i,
    7616       199495 :                                                  from_partition.layout);
    7617       199495 :       cost = from_costs.in_cost;
    7618       199495 :       cost.add_serial_cost (from_costs.internal_cost);
    7619       199495 :       cost.split (from_partition.out_degree);
    7620       199495 :       cost.add_serial_cost (edge_cost);
    7621              :     }
    7622         3182 :   else if (from_partition.layout == 0)
    7623              :     /* We must allow the source partition to have layout 0 as a fallback,
    7624              :        in case all other options turn out to be impossible.  */
    7625         3182 :     return cost;
    7626              : 
    7627              :   /* Take the minimum of that cost and the cost that applies if
    7628              :      FROM_PARTITION instead switches to TO_LAYOUT_I.  */
    7629       199495 :   auto &direct_layout_costs = partition_layout_costs (from_partition_i,
    7630              :                                                       to_layout_i);
    7631       199495 :   if (direct_layout_costs.is_possible ())
    7632              :     {
    7633       171861 :       slpg_layout_cost direct_cost = direct_layout_costs.in_cost;
    7634       171861 :       direct_cost.add_serial_cost (direct_layout_costs.internal_cost);
    7635       171861 :       direct_cost.split (from_partition.out_degree);
    7636       171861 :       if (!cost.is_possible ()
    7637       171861 :           || direct_cost.is_better_than (cost, m_optimize_size))
    7638        35494 :         cost = direct_cost;
    7639              :     }
    7640              : 
    7641       199495 :   return cost;
    7642              : }
    7643              : 
    7644              : /* UD represents a use-def link between TO_NODE_I and a node in an earlier
    7645              :    partition; TO_NODE_I could be the definition node or the use node.
    7646              :    The node at the other end of the link wants to use layout FROM_LAYOUT_I;
    7647              :    return the cost of any necessary fix-ups on edge UD, or
    7648              :    slpg_layout_cost::impossible () if the choice cannot be made.
    7649              : 
    7650              :    At this point, TO_NODE_I's partition has a fixed choice of layout.  */
    7651              : 
    7652              : slpg_layout_cost
    7653       178816 : vect_optimize_slp_pass::backward_cost (graph_edge *ud, unsigned int to_node_i,
    7654              :                                        unsigned int from_layout_i)
    7655              : {
    7656       178816 :   auto &to_vertex = m_vertices[to_node_i];
    7657       178816 :   unsigned int to_partition_i = to_vertex.partition;
    7658       178816 :   slpg_partition_info &to_partition = m_partitions[to_partition_i];
    7659       178816 :   gcc_assert (to_partition.layout >= 0);
    7660              : 
    7661              :   /* If TO_NODE_I is a VEC_PERM_EXPR consumer, see whether it can be
    7662              :      adjusted for this input having layout FROM_LAYOUT_I.  Assume that
    7663              :      any other inputs keep their current choice of layout.  */
    7664       178816 :   auto &to_costs = partition_layout_costs (to_partition_i,
    7665              :                                            to_partition.layout);
    7666       178816 :   if (ud->src == int (to_node_i)
    7667       178534 :       && SLP_TREE_PERMUTE_P (to_vertex.node))
    7668              :     {
    7669        11436 :       auto &from_partition = m_partitions[m_vertices[ud->dest].partition];
    7670        11436 :       auto old_layout = from_partition.layout;
    7671        11436 :       from_partition.layout = from_layout_i;
    7672        22872 :       int factor = internal_node_cost (to_vertex.node, -1,
    7673        11436 :                                        to_partition.layout);
    7674        11436 :       from_partition.layout = old_layout;
    7675        11436 :       if (factor >= 0)
    7676              :         {
    7677        10529 :           slpg_layout_cost cost = to_costs.out_cost;
    7678        10529 :           cost.add_serial_cost ({ to_vertex.weight * factor,
    7679              :                                   m_optimize_size });
    7680        10529 :           cost.split (to_partition.in_degree);
    7681        10529 :           return cost;
    7682              :         }
    7683              :     }
    7684              : 
    7685              :   /* Compute the cost if we insert any necessary layout change on edge UD.  */
    7686       168287 :   auto edge_cost = edge_layout_cost (ud, to_node_i,
    7687       168287 :                                      to_partition.layout, from_layout_i);
    7688       168287 :   if (edge_cost.is_possible ())
    7689              :     {
    7690       168287 :       slpg_layout_cost cost = to_costs.out_cost;
    7691       168287 :       cost.add_serial_cost (to_costs.internal_cost);
    7692       168287 :       cost.split (to_partition.in_degree);
    7693       168287 :       cost.add_serial_cost (edge_cost);
    7694       168287 :       return cost;
    7695              :     }
    7696              : 
    7697            0 :   return slpg_layout_cost::impossible ();
    7698              : }
    7699              : 
    7700              : /* Make a forward pass through the partitions, accumulating input costs.
    7701              :    Make a tentative (provisional) choice of layout for each partition,
    7702              :    ensuring that this choice still allows later partitions to keep
    7703              :    their original layout.  */
    7704              : 
    7705              : void
    7706         6616 : vect_optimize_slp_pass::forward_pass ()
    7707              : {
    7708       131184 :   for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
    7709              :        ++partition_i)
    7710              :     {
    7711       124568 :       auto &partition = m_partitions[partition_i];
    7712              : 
    7713              :       /* If the partition consists of a single VEC_PERM_EXPR, precompute
    7714              :          the incoming cost that would apply if every predecessor partition
    7715              :          keeps its current layout.  This is used within the loop below.  */
    7716       124568 :       slpg_layout_cost in_cost;
    7717       124568 :       slp_tree single_node = nullptr;
    7718       124568 :       if (partition.node_end == partition.node_begin + 1)
    7719              :         {
    7720       119919 :           unsigned int node_i = m_partitioned_nodes[partition.node_begin];
    7721       119919 :           single_node = m_vertices[node_i].node;
    7722       119919 :           if (SLP_TREE_PERMUTE_P (single_node))
    7723         4379 :             in_cost = total_in_cost (node_i);
    7724              :         }
    7725              : 
    7726              :       /* Go through the possible layouts.  Decide which ones are valid
    7727              :          for this partition and record which of the valid layouts has
    7728              :          the lowest cost.  */
    7729       124568 :       unsigned int min_layout_i = 0;
    7730       124568 :       slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
    7731       384268 :       for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
    7732              :         {
    7733       259700 :           auto &layout_costs = partition_layout_costs (partition_i, layout_i);
    7734       259700 :           if (!layout_costs.is_possible ())
    7735        63679 :             continue;
    7736              : 
    7737              :           /* If the recorded layout is already 0 then the layout cannot
    7738              :              change.  */
    7739       259700 :           if (partition.layout == 0 && layout_i != 0)
    7740              :             {
    7741        49678 :               layout_costs.mark_impossible ();
    7742        49678 :               continue;
    7743              :             }
    7744              : 
    7745       210022 :           bool is_possible = true;
    7746       442592 :           for (unsigned int order_i = partition.node_begin;
    7747       442592 :                order_i < partition.node_end; ++order_i)
    7748              :             {
    7749       243701 :               unsigned int node_i = m_partitioned_nodes[order_i];
    7750       243701 :               auto &vertex = m_vertices[node_i];
    7751              : 
    7752              :               /* Reject the layout if it is individually incompatible
    7753              :                  with any node in the partition.  */
    7754       243701 :               if (!is_compatible_layout (vertex.node, layout_i))
    7755              :                 {
    7756         9621 :                   is_possible = false;
    7757        11131 :                   break;
    7758              :                 }
    7759              : 
    7760       619273 :               auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
    7761              :                 {
    7762       385193 :                   auto &other_vertex = m_vertices[other_node_i];
    7763       385193 :                   if (other_vertex.partition < vertex.partition)
    7764              :                     {
    7765              :                       /* Accumulate the incoming costs from earlier
    7766              :                          partitions, plus the cost of any layout changes
    7767              :                          on UD itself.  */
    7768       202677 :                       auto cost = forward_cost (ud, other_node_i, layout_i);
    7769       202677 :                       if (!cost.is_possible ())
    7770         3182 :                         is_possible = false;
    7771              :                       else
    7772       199495 :                         layout_costs.in_cost.add_parallel_cost (cost);
    7773              :                     }
    7774              :                   else
    7775              :                     /* Reject the layout if it would make layout 0 impossible
    7776              :                        for later partitions.  This amounts to testing that the
    7777              :                        target supports reversing the layout change on edges
    7778              :                        to later partitions.
    7779              : 
    7780              :                        In principle, it might be possible to push a layout
    7781              :                        change all the way down a graph, so that it never
    7782              :                        needs to be reversed and so that the target doesn't
    7783              :                        need to support the reverse operation.  But it would
    7784              :                        be awkward to bail out if we hit a partition that
    7785              :                        does not support the new layout, especially since
    7786              :                        we are not dealing with a lattice.  */
    7787       182516 :                     is_possible &= edge_layout_cost (ud, other_node_i, 0,
    7788       182516 :                                                      layout_i).is_possible ();
    7789       619273 :                 };
    7790       234080 :               for_each_partition_edge (node_i, add_cost);
    7791              : 
    7792              :               /* Accumulate the cost of using LAYOUT_I within NODE,
    7793              :                  both for the inputs and the outputs.  */
    7794       234080 :               int factor = internal_node_cost (vertex.node, layout_i,
    7795              :                                                layout_i);
    7796       234080 :               if (factor < 0)
    7797              :                 {
    7798         1510 :                   is_possible = false;
    7799         1510 :                   break;
    7800              :                 }
    7801       232570 :               else if (factor)
    7802        39009 :                 layout_costs.internal_cost.add_serial_cost
    7803        39009 :                   ({ vertex.weight * factor, m_optimize_size });
    7804              :             }
    7805       210022 :           if (!is_possible)
    7806              :             {
    7807        14001 :               layout_costs.mark_impossible ();
    7808        14001 :               continue;
    7809              :             }
    7810              : 
    7811              :           /* Combine the incoming and partition-internal costs.  */
    7812       196021 :           slpg_layout_cost combined_cost = layout_costs.in_cost;
    7813       196021 :           combined_cost.add_serial_cost (layout_costs.internal_cost);
    7814              : 
    7815              :           /* If this partition consists of a single VEC_PERM_EXPR, see
    7816              :              if the VEC_PERM_EXPR can be changed to support output layout
    7817              :              LAYOUT_I while keeping all the provisional choices of input
    7818              :              layout.  */
    7819       196021 :           if (single_node && SLP_TREE_PERMUTE_P (single_node))
    7820              :             {
    7821         7605 :               int factor = internal_node_cost (single_node, -1, layout_i);
    7822         7605 :               if (factor >= 0)
    7823              :                 {
    7824         6879 :                   auto weight = m_vertices[single_node->vertex].weight;
    7825         6879 :                   slpg_layout_cost internal_cost
    7826         6879 :                     = { weight * factor, m_optimize_size };
    7827              : 
    7828         6879 :                   slpg_layout_cost alt_cost = in_cost;
    7829         6879 :                   alt_cost.add_serial_cost (internal_cost);
    7830         6879 :                   if (alt_cost.is_better_than (combined_cost, m_optimize_size))
    7831              :                     {
    7832         1670 :                       combined_cost = alt_cost;
    7833         1670 :                       layout_costs.in_cost = in_cost;
    7834         1670 :                       layout_costs.internal_cost = internal_cost;
    7835              :                     }
    7836              :                 }
    7837              :             }
    7838              : 
    7839              :           /* Record the layout with the lowest cost.  Prefer layout 0 in
    7840              :              the event of a tie between it and another layout.  */
    7841       196021 :           if (!min_layout_cost.is_possible ()
    7842        71453 :               || combined_cost.is_better_than (min_layout_cost,
    7843              :                                                m_optimize_size))
    7844              :             {
    7845       142025 :               min_layout_i = layout_i;
    7846       142025 :               min_layout_cost = combined_cost;
    7847              :             }
    7848              :         }
    7849              : 
    7850              :       /* This loop's handling of earlier partitions should ensure that
    7851              :          choosing the original layout for the current partition is no
    7852              :          less valid than it was in the original graph, even with the
    7853              :          provisional layout choices for those earlier partitions.  */
    7854       124568 :       gcc_assert (min_layout_cost.is_possible ());
    7855       124568 :       partition.layout = min_layout_i;
    7856              :     }
    7857         6616 : }
    7858              : 
    7859              : /* Make a backward pass through the partitions, accumulating output costs.
    7860              :    Make a final choice of layout for each partition.  */
    7861              : 
    7862              : void
    7863         6616 : vect_optimize_slp_pass::backward_pass ()
    7864              : {
    7865       137800 :   for (unsigned int partition_i = m_partitions.length (); partition_i-- > 0;)
    7866              :     {
    7867       124568 :       auto &partition = m_partitions[partition_i];
    7868              : 
    7869       124568 :       unsigned int min_layout_i = 0;
    7870       124568 :       slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
    7871       384268 :       for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
    7872              :         {
    7873       259700 :           auto &layout_costs = partition_layout_costs (partition_i, layout_i);
    7874       259700 :           if (!layout_costs.is_possible ())
    7875        63679 :             continue;
    7876              : 
    7877              :           /* Accumulate the costs from successor partitions.  */
    7878       196021 :           bool is_possible = true;
    7879       425696 :           for (unsigned int order_i = partition.node_begin;
    7880       425696 :                order_i < partition.node_end; ++order_i)
    7881              :             {
    7882       229675 :               unsigned int node_i = m_partitioned_nodes[order_i];
    7883       229675 :               auto &vertex = m_vertices[node_i];
    7884       607499 :               auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
    7885              :                 {
    7886       377824 :                   auto &other_vertex = m_vertices[other_node_i];
    7887       377824 :                   auto &other_partition = m_partitions[other_vertex.partition];
    7888       377824 :                   if (other_vertex.partition > vertex.partition)
    7889              :                     {
    7890              :                       /* Accumulate the incoming costs from later
    7891              :                          partitions, plus the cost of any layout changes
    7892              :                          on UD itself.  */
    7893       178816 :                       auto cost = backward_cost (ud, other_node_i, layout_i);
    7894       178816 :                       if (!cost.is_possible ())
    7895            0 :                         is_possible = false;
    7896              :                       else
    7897       178816 :                         layout_costs.out_cost.add_parallel_cost (cost);
    7898              :                     }
    7899              :                   else
    7900              :                     /* Make sure that earlier partitions can (if necessary
    7901              :                        or beneficial) keep the layout that they chose in
    7902              :                        the forward pass.  This ensures that there is at
    7903              :                        least one valid choice of layout.  */
    7904       199008 :                     is_possible &= edge_layout_cost (ud, other_node_i,
    7905       199008 :                                                      other_partition.layout,
    7906       199008 :                                                      layout_i).is_possible ();
    7907       607499 :                 };
    7908       229675 :               for_each_partition_edge (node_i, add_cost);
    7909              :             }
    7910       196021 :           if (!is_possible)
    7911              :             {
    7912            0 :               layout_costs.mark_impossible ();
    7913            0 :               continue;
    7914              :             }
    7915              : 
    7916              :           /* Locally combine the costs from the forward and backward passes.
    7917              :              (This combined cost is not passed on, since that would lead
    7918              :              to double counting.)  */
    7919       196021 :           slpg_layout_cost combined_cost = layout_costs.in_cost;
    7920       196021 :           combined_cost.add_serial_cost (layout_costs.internal_cost);
    7921       196021 :           combined_cost.add_serial_cost (layout_costs.out_cost);
    7922              : 
    7923              :           /* Record the layout with the lowest cost.  Prefer layout 0 in
    7924              :              the event of a tie between it and another layout.  */
    7925       196021 :           if (!min_layout_cost.is_possible ()
    7926        71453 :               || combined_cost.is_better_than (min_layout_cost,
    7927              :                                                m_optimize_size))
    7928              :             {
    7929       135607 :               min_layout_i = layout_i;
    7930       135607 :               min_layout_cost = combined_cost;
    7931              :             }
    7932              :         }
    7933              : 
    7934       124568 :       gcc_assert (min_layout_cost.is_possible ());
    7935       124568 :       partition.layout = min_layout_i;
    7936              :     }
    7937         6616 : }
    7938              : 
    7939              : /* Return a node that applies layout TO_LAYOUT_I to the original form of NODE.
    7940              :    NODE already has the layout that was selected for its partition.  */
    7941              : 
    7942              : slp_tree
    7943       175417 : vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
    7944              :                                                 unsigned int to_layout_i)
    7945              : {
    7946       175417 :   unsigned int result_i = node->vertex * m_perms.length () + to_layout_i;
    7947       175417 :   slp_tree result = m_node_layouts[result_i];
    7948       175417 :   if (result)
    7949              :     return result;
    7950              : 
    7951       174500 :   if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
    7952       174500 :       || (SLP_TREE_DEF_TYPE (node) == vect_external_def
    7953              :           /* We can't permute vector defs in place.  */
    7954        22151 :           && SLP_TREE_VEC_DEFS (node).is_empty ()))
    7955              :     {
    7956              :       /* If the vector is uniform or unchanged, there's nothing to do.  */
    7957        47100 :       if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
    7958              :         result = node;
    7959              :       else
    7960              :         {
    7961         3677 :           auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
    7962         3677 :           result = vect_create_new_slp_node (scalar_ops);
    7963         3677 :           vect_slp_permute (m_perms[to_layout_i], scalar_ops, true);
    7964              :         }
    7965              :     }
    7966              :   else
    7967              :     {
    7968       127400 :       unsigned int partition_i = m_vertices[node->vertex].partition;
    7969       127400 :       unsigned int from_layout_i = m_partitions[partition_i].layout;
    7970       127400 :       if (from_layout_i == to_layout_i)
    7971       126183 :         return node;
    7972              : 
    7973              :       /* If NODE is itself a VEC_PERM_EXPR, try to create a parallel
    7974              :          permutation instead of a serial one.  Leave the new permutation
    7975              :          in TMP_PERM on success.  */
    7976         1217 :       auto_lane_permutation_t tmp_perm;
    7977         1217 :       unsigned int num_inputs = 1;
    7978         1217 :       if (SLP_TREE_PERMUTE_P (node))
    7979              :         {
    7980          103 :           tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
    7981          103 :           if (from_layout_i != 0)
    7982           43 :             vect_slp_permute (m_perms[from_layout_i], tmp_perm, false);
    7983          103 :           if (to_layout_i != 0)
    7984           64 :             vect_slp_permute (m_perms[to_layout_i], tmp_perm, true);
    7985          103 :           if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
    7986              :                                               tmp_perm,
    7987          103 :                                               SLP_TREE_CHILDREN (node),
    7988              :                                               false) >= 0)
    7989          103 :             num_inputs = SLP_TREE_CHILDREN (node).length ();
    7990              :           else
    7991            0 :             tmp_perm.truncate (0);
    7992              :         }
    7993              : 
    7994         1217 :       if (dump_enabled_p ())
    7995              :         {
    7996           71 :           if (tmp_perm.length () > 0)
    7997            6 :             dump_printf_loc (MSG_NOTE, vect_location,
    7998              :                              "duplicating permutation node %p with"
    7999              :                              " layout %d\n",
    8000              :                              (void *) node, to_layout_i);
    8001              :           else
    8002           65 :             dump_printf_loc (MSG_NOTE, vect_location,
    8003              :                              "inserting permutation node in place of %p\n",
    8004              :                              (void *) node);
    8005              :         }
    8006              : 
    8007         1217 :       unsigned int num_lanes = SLP_TREE_LANES (node);
    8008         1217 :       result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
    8009         1217 :       if (SLP_TREE_SCALAR_STMTS (node).exists ())
    8010              :         {
    8011         1217 :           auto &stmts = SLP_TREE_SCALAR_STMTS (result);
    8012         1217 :           stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
    8013         1217 :           if (from_layout_i != 0)
    8014          500 :             vect_slp_permute (m_perms[from_layout_i], stmts, false);
    8015         1217 :           if (to_layout_i != 0)
    8016          721 :             vect_slp_permute (m_perms[to_layout_i], stmts, true);
    8017              :         }
    8018         1217 :       SLP_TREE_LANES (result) = num_lanes;
    8019         1217 :       SLP_TREE_VECTYPE (result) = SLP_TREE_VECTYPE (node);
    8020         1217 :       result->vertex = -1;
    8021              : 
    8022         1217 :       auto &lane_perm = SLP_TREE_LANE_PERMUTATION (result);
    8023         1217 :       if (tmp_perm.length ())
    8024              :         {
    8025          103 :           lane_perm.safe_splice (tmp_perm);
    8026          103 :           SLP_TREE_CHILDREN (result).safe_splice (SLP_TREE_CHILDREN (node));
    8027              :         }
    8028              :       else
    8029              :         {
    8030         1114 :           lane_perm.create (num_lanes);
    8031         4538 :           for (unsigned j = 0; j < num_lanes; ++j)
    8032         2310 :             lane_perm.quick_push ({ 0, j });
    8033         1114 :           if (from_layout_i != 0)
    8034          457 :             vect_slp_permute (m_perms[from_layout_i], lane_perm, false);
    8035         1114 :           if (to_layout_i != 0)
    8036          657 :             vect_slp_permute (m_perms[to_layout_i], lane_perm, true);
    8037         1114 :           SLP_TREE_CHILDREN (result).safe_push (node);
    8038              :         }
    8039         4940 :       for (slp_tree child : SLP_TREE_CHILDREN (result))
    8040         1289 :         child->refcnt++;
    8041         1217 :     }
    8042        48317 :   m_node_layouts[result_i] = result;
    8043        48317 :   return result;
    8044              : }
    8045              : 
    8046              : /* Apply the chosen vector layouts to the SLP graph.  */
    8047              : 
    8048              : void
    8049        13871 : vect_optimize_slp_pass::materialize ()
    8050              : {
    8051              :   /* We no longer need the costs, so avoid having two O(N * P) arrays
    8052              :      live at the same time.  */
    8053        13871 :   m_partition_layout_costs.release ();
    8054        41613 :   m_node_layouts.safe_grow_cleared (m_vertices.length () * m_perms.length ());
    8055              : 
    8056        27742 :   auto_sbitmap fully_folded (m_vertices.length ());
    8057        13871 :   bitmap_clear (fully_folded);
    8058       198586 :   for (unsigned int node_i : m_partitioned_nodes)
    8059              :     {
    8060       156973 :       auto &vertex = m_vertices[node_i];
    8061       156973 :       slp_tree node = vertex.node;
    8062       156973 :       int layout_i = m_partitions[vertex.partition].layout;
    8063       156973 :       gcc_assert (layout_i >= 0);
    8064              : 
    8065              :       /* Rearrange the scalar statements to match the chosen layout.  */
    8066       156973 :       if (layout_i > 0)
    8067        26718 :         vect_slp_permute (m_perms[layout_i],
    8068        26718 :                           SLP_TREE_SCALAR_STMTS (node), true);
    8069              : 
    8070              :       /* Update load and lane permutations.  */
    8071       156973 :       if (SLP_TREE_PERMUTE_P (node))
    8072              :         {
    8073              :           /* First try to absorb the input vector layouts.  If that fails,
    8074              :              force the inputs to have layout LAYOUT_I too.  We checked that
    8075              :              that was possible before deciding to use nonzero output layouts.
    8076              :              (Note that at this stage we don't really have any guarantee that
    8077              :              the target supports the original VEC_PERM_EXPR.)  */
    8078         8346 :           auto &perm = SLP_TREE_LANE_PERMUTATION (node);
    8079         8346 :           auto_lane_permutation_t tmp_perm;
    8080         8346 :           tmp_perm.safe_splice (perm);
    8081         8346 :           change_vec_perm_layout (node, tmp_perm, -1, layout_i);
    8082         8346 :           if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
    8083              :                                               tmp_perm,
    8084         8346 :                                               SLP_TREE_CHILDREN (node),
    8085              :                                               false) >= 0)
    8086              :             {
    8087         7702 :               if (dump_enabled_p ()
    8088         8630 :                   && !std::equal (tmp_perm.begin (), tmp_perm.end (),
    8089              :                                   perm.begin ()))
    8090           58 :                 dump_printf_loc (MSG_NOTE, vect_location,
    8091              :                                  "absorbing input layouts into %p\n",
    8092              :                                  (void *) node);
    8093        42423 :               std::copy (tmp_perm.begin (), tmp_perm.end (), perm.begin ());
    8094         7702 :               bitmap_set_bit (fully_folded, node_i);
    8095              :             }
    8096              :           else
    8097              :             {
    8098              :               /* Not MSG_MISSED because it would make no sense to users.  */
    8099          644 :               if (dump_enabled_p ())
    8100           46 :                 dump_printf_loc (MSG_NOTE, vect_location,
    8101              :                                  "failed to absorb input layouts into %p\n",
    8102              :                                  (void *) node);
    8103          644 :               change_vec_perm_layout (nullptr, perm, layout_i, layout_i);
    8104              :             }
    8105         8346 :         }
    8106              :       else
    8107              :         {
    8108       148627 :           gcc_assert (!SLP_TREE_LANE_PERMUTATION (node).exists ());
    8109       148627 :           auto &load_perm = SLP_TREE_LOAD_PERMUTATION (node);
    8110       148627 :           if (layout_i > 0)
    8111              :             /* ???  When we handle non-bijective permutes the idea
    8112              :                is that we can force the load-permutation to be
    8113              :                { min, min + 1, min + 2, ... max }.  But then the
    8114              :                scalar defs might no longer match the lane content
    8115              :                which means wrong-code with live lane vectorization.
    8116              :                So we possibly have to have NULL entries for those.  */
    8117        25874 :             vect_slp_permute (m_perms[layout_i], load_perm, true);
    8118              :         }
    8119              :     }
    8120              : 
    8121              :   /* Do this before any nodes disappear, since it involves a walk
    8122              :      over the leaves.  */
    8123        13871 :   remove_redundant_permutations ();
    8124              : 
    8125              :   /* Replace each child with a correctly laid-out version.  */
    8126       198586 :   for (unsigned int node_i : m_partitioned_nodes)
    8127              :     {
    8128              :       /* Skip nodes that have already been handled above.  */
    8129       156973 :       if (bitmap_bit_p (fully_folded, node_i))
    8130         7702 :         continue;
    8131              : 
    8132       149271 :       auto &vertex = m_vertices[node_i];
    8133       149271 :       int in_layout_i = m_partitions[vertex.partition].layout;
    8134       149271 :       gcc_assert (in_layout_i >= 0);
    8135              : 
    8136              :       unsigned j;
    8137              :       slp_tree child;
    8138       443965 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (vertex.node), j, child)
    8139              :         {
    8140       179699 :           if (!child)
    8141         4282 :             continue;
    8142              : 
    8143       175417 :           slp_tree new_child = get_result_with_layout (child, in_layout_i);
    8144       175417 :           if (new_child != child)
    8145              :             {
    8146         5478 :               vect_free_slp_tree (child);
    8147         5478 :               SLP_TREE_CHILDREN (vertex.node)[j] = new_child;
    8148         5478 :               new_child->refcnt += 1;
    8149              :             }
    8150              :         }
    8151              :     }
    8152        13871 : }
    8153              : 
    8154              : /* Elide load permutations that are not necessary.  Such permutations might
    8155              :    be pre-existing, rather than created by the layout optimizations.  */
    8156              : 
    8157              : void
    8158       732178 : vect_optimize_slp_pass::remove_redundant_permutations ()
    8159              : {
    8160      4849983 :   for (unsigned int node_i : m_leafs)
    8161              :     {
    8162      2653449 :       slp_tree node = m_vertices[node_i].node;
    8163      2653449 :       if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
    8164      2003219 :         continue;
    8165              : 
    8166              :       /* In basic block vectorization we allow any subchain of an interleaving
    8167              :          chain.
    8168              :          FORNOW: not in loop SLP because of realignment complications.  */
    8169       650230 :       if (is_a <bb_vec_info> (m_vinfo))
    8170              :         {
    8171       193411 :           bool subchain_p = true;
    8172              :           stmt_vec_info next_load_info = NULL;
    8173              :           stmt_vec_info load_info;
    8174              :           unsigned j;
    8175       193411 :           FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
    8176              :             {
    8177       159766 :               if (j != 0
    8178       159766 :                   && (next_load_info != load_info
    8179        70941 :                       || ! load_info
    8180        70941 :                       || DR_GROUP_GAP (load_info) != 1))
    8181              :                 {
    8182              :                   subchain_p = false;
    8183              :                   break;
    8184              :                 }
    8185       127966 :               next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
    8186              :             }
    8187        65445 :           if (subchain_p)
    8188              :             {
    8189        33645 :               SLP_TREE_LOAD_PERMUTATION (node).release ();
    8190        33645 :               continue;
    8191              :             }
    8192              :         }
    8193              :       else
    8194              :         {
    8195       584785 :           loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
    8196       584785 :           bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
    8197              :           /* When this isn't a grouped access we know it's single element
    8198              :              and contiguous.  */
    8199       584785 :           if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
    8200              :             {
    8201       462971 :               if (!this_load_permuted
    8202       462971 :                   && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
    8203       462183 :                       || SLP_TREE_LANES (node) == 1))
    8204       462201 :                 SLP_TREE_LOAD_PERMUTATION (node).release ();
    8205       462971 :               continue;
    8206              :             }
    8207       121814 :           stmt_vec_info first_stmt_info
    8208       121814 :             = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
    8209       122347 :           if (!this_load_permuted
    8210              :               /* The load requires permutation when unrolling exposes
    8211              :                  a gap either because the group is larger than the SLP
    8212              :                  group-size or because there is a gap between the groups.  */
    8213       121814 :               && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
    8214        98842 :                   || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
    8215          147 :                       && DR_GROUP_GAP (first_stmt_info) == 0)))
    8216              :             {
    8217          533 :               SLP_TREE_LOAD_PERMUTATION (node).release ();
    8218          533 :               continue;
    8219              :             }
    8220              :         }
    8221              :     }
    8222       732178 : }
    8223              : 
    8224              : /* Print the partition graph and layout information to the dump file.  */
    8225              : 
    8226              : void
    8227          691 : vect_optimize_slp_pass::dump ()
    8228              : {
    8229          691 :   dump_printf_loc (MSG_NOTE, vect_location,
    8230              :                    "SLP optimize permutations:\n");
    8231         1395 :   for (unsigned int layout_i = 1; layout_i < m_perms.length (); ++layout_i)
    8232              :     {
    8233          704 :       dump_printf_loc (MSG_NOTE, vect_location, "  %d: { ", layout_i);
    8234          704 :       const char *sep = "";
    8235         5976 :       for (unsigned int idx : m_perms[layout_i])
    8236              :         {
    8237         3864 :           dump_printf (MSG_NOTE, "%s%d", sep, idx);
    8238         3864 :           sep = ", ";
    8239              :         }
    8240          704 :       dump_printf (MSG_NOTE, " }\n");
    8241              :     }
    8242          691 :   dump_printf_loc (MSG_NOTE, vect_location,
    8243              :                    "SLP optimize partitions:\n");
    8244         5996 :   for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
    8245              :        ++partition_i)
    8246              :     {
    8247         5305 :       auto &partition = m_partitions[partition_i];
    8248         5305 :       dump_printf_loc (MSG_NOTE, vect_location,  "  -------------\n");
    8249         5305 :       dump_printf_loc (MSG_NOTE, vect_location,
    8250              :                        "  partition %d (layout %d):\n",
    8251              :                        partition_i, partition.layout);
    8252         5305 :       dump_printf_loc (MSG_NOTE, vect_location, "    nodes:\n");
    8253        10849 :       for (unsigned int order_i = partition.node_begin;
    8254        10849 :            order_i < partition.node_end; ++order_i)
    8255              :         {
    8256         5544 :           auto &vertex = m_vertices[m_partitioned_nodes[order_i]];
    8257        11088 :           dump_printf_loc (MSG_NOTE, vect_location, "      - %p:\n",
    8258         5544 :                            (void *) vertex.node);
    8259         5544 :           dump_printf_loc (MSG_NOTE, vect_location,
    8260              :                            "          weight: %f\n",
    8261              :                            vertex.weight.to_double ());
    8262         5544 :           if (vertex.out_degree)
    8263         4344 :             dump_printf_loc (MSG_NOTE, vect_location,
    8264              :                              "          out weight: %f (degree %d)\n",
    8265              :                              vertex.out_weight.to_double (),
    8266              :                              vertex.out_degree);
    8267         5544 :           if (SLP_TREE_PERMUTE_P (vertex.node))
    8268          510 :             dump_printf_loc (MSG_NOTE, vect_location,
    8269              :                              "          op: VEC_PERM_EXPR\n");
    8270         5034 :           else if (auto rep = SLP_TREE_REPRESENTATIVE (vertex.node))
    8271         5016 :             dump_printf_loc (MSG_NOTE, vect_location,
    8272              :                              "          op template: %G", rep->stmt);
    8273              :         }
    8274         5305 :       dump_printf_loc (MSG_NOTE, vect_location, "    edges:\n");
    8275        10849 :       for (unsigned int order_i = partition.node_begin;
    8276        10849 :            order_i < partition.node_end; ++order_i)
    8277              :         {
    8278         5544 :           unsigned int node_i = m_partitioned_nodes[order_i];
    8279         5544 :           auto &vertex = m_vertices[node_i];
    8280        16782 :           auto print_edge = [&](graph_edge *, unsigned int other_node_i)
    8281              :             {
    8282        11238 :               auto &other_vertex = m_vertices[other_node_i];
    8283        11238 :               if (other_vertex.partition < vertex.partition)
    8284         5619 :                 dump_printf_loc (MSG_NOTE, vect_location,
    8285              :                                  "      - %p [%d] --> %p\n",
    8286         5619 :                                  (void *) other_vertex.node,
    8287              :                                  other_vertex.partition,
    8288         5619 :                                  (void *) vertex.node);
    8289              :               else
    8290         5619 :                 dump_printf_loc (MSG_NOTE, vect_location,
    8291              :                                  "      - %p --> [%d] %p\n",
    8292         5619 :                                  (void *) vertex.node,
    8293              :                                  other_vertex.partition,
    8294         5619 :                                  (void *) other_vertex.node);
    8295        16782 :             };
    8296         5544 :           for_each_partition_edge (node_i, print_edge);
    8297              :         }
    8298              : 
    8299        16114 :       for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
    8300              :         {
    8301        10809 :           auto &layout_costs = partition_layout_costs (partition_i, layout_i);
    8302        10809 :           if (layout_costs.is_possible ())
    8303              :             {
    8304         8952 :               dump_printf_loc (MSG_NOTE, vect_location,
    8305              :                                "    layout %d:%s\n", layout_i,
    8306         8952 :                                partition.layout == int (layout_i)
    8307              :                                ? " (*)" : "");
    8308         8952 :               slpg_layout_cost combined_cost = layout_costs.in_cost;
    8309         8952 :               combined_cost.add_serial_cost (layout_costs.internal_cost);
    8310         8952 :               combined_cost.add_serial_cost (layout_costs.out_cost);
    8311              : #define TEMPLATE "{depth: %f, total: %f}"
    8312         8952 :               dump_printf_loc (MSG_NOTE, vect_location,
    8313              :                                "        " TEMPLATE "\n",
    8314              :                                layout_costs.in_cost.depth.to_double (),
    8315              :                                layout_costs.in_cost.total.to_double ());
    8316         8952 :               dump_printf_loc (MSG_NOTE, vect_location,
    8317              :                                "      + " TEMPLATE "\n",
    8318              :                                layout_costs.internal_cost.depth.to_double (),
    8319              :                                layout_costs.internal_cost.total.to_double ());
    8320         8952 :               dump_printf_loc (MSG_NOTE, vect_location,
    8321              :                                "      + " TEMPLATE "\n",
    8322              :                                layout_costs.out_cost.depth.to_double (),
    8323              :                                layout_costs.out_cost.total.to_double ());
    8324         8952 :               dump_printf_loc (MSG_NOTE, vect_location,
    8325              :                                "      = " TEMPLATE "\n",
    8326              :                                combined_cost.depth.to_double (),
    8327              :                                combined_cost.total.to_double ());
    8328              : #undef TEMPLATE
    8329              :             }
    8330              :           else
    8331         1857 :             dump_printf_loc (MSG_NOTE, vect_location,
    8332              :                              "    layout %d: rejected\n", layout_i);
    8333              :         }
    8334              :     }
    8335          691 : }
    8336              : 
    8337              : /* Masked load lanes discovery.  */
    8338              : 
    8339              : void
    8340       732178 : vect_optimize_slp_pass::decide_masked_load_lanes ()
    8341              : {
    8342      7566247 :   for (auto v : m_vertices)
    8343              :     {
    8344      5369713 :       slp_tree node = v.node;
    8345      5369713 :       if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
    8346      3770863 :           || SLP_TREE_PERMUTE_P (node))
    8347      1745700 :         continue;
    8348      3624013 :       stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
    8349      1733113 :       if (! STMT_VINFO_GROUPED_ACCESS (stmt_info)
    8350              :           /* The mask has to be uniform.  */
    8351      1010441 :           || STMT_VINFO_SLP_VECT_ONLY (stmt_info)
    8352      1010363 :           || ! is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
    8353      3624098 :           || ! gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
    8354              :                                        IFN_MASK_LOAD))
    8355      3623980 :         continue;
    8356           33 :       stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
    8357           66 :       if (STMT_VINFO_STRIDED_P (stmt_info)
    8358           33 :           || compare_step_with_zero (m_vinfo, stmt_info) <= 0
    8359           63 :           || vect_load_lanes_supported (SLP_TREE_VECTYPE (node),
    8360           30 :                                         DR_GROUP_SIZE (stmt_info),
    8361              :                                         true) == IFN_LAST)
    8362           33 :         continue;
    8363              : 
    8364              :       /* Uniform masks need to be suitably represented.  */
    8365            0 :       slp_tree mask = SLP_TREE_CHILDREN (node)[0];
    8366            0 :       if (!SLP_TREE_PERMUTE_P (mask)
    8367            0 :           || SLP_TREE_CHILDREN (mask).length () != 1)
    8368            0 :         continue;
    8369            0 :       bool match = true;
    8370            0 :       for (auto perm : SLP_TREE_LANE_PERMUTATION (mask))
    8371            0 :         if (perm.first != 0 || perm.second != 0)
    8372              :           {
    8373              :             match = false;
    8374              :             break;
    8375              :           }
    8376            0 :       if (!match)
    8377            0 :         continue;
    8378              : 
    8379              :       /* Now see if the consumer side matches.  */
    8380            0 :       for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
    8381            0 :            pred; pred = pred->pred_next)
    8382              :         {
    8383            0 :           slp_tree pred_node = m_vertices[pred->src].node;
    8384              :           /* All consumers should be a permute with a single outgoing lane.  */
    8385            0 :           if (!SLP_TREE_PERMUTE_P (pred_node)
    8386            0 :               || SLP_TREE_LANES (pred_node) != 1)
    8387              :             {
    8388              :               match = false;
    8389              :               break;
    8390              :             }
    8391            0 :           gcc_assert (SLP_TREE_CHILDREN (pred_node).length () == 1);
    8392              :         }
    8393            0 :       if (!match)
    8394            0 :         continue;
    8395              :       /* Now we can mark the nodes as to use load lanes.  */
    8396            0 :       node->ldst_lanes = true;
    8397            0 :       for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
    8398            0 :            pred; pred = pred->pred_next)
    8399            0 :         m_vertices[pred->src].node->ldst_lanes = true;
    8400              :       /* The catch is we have to massage the mask.  We have arranged
    8401              :          analyzed uniform masks to be represented by a splat VEC_PERM
    8402              :          which we can now simply elide as we cannot easily re-do SLP
    8403              :          discovery here.  */
    8404            0 :       slp_tree new_mask = SLP_TREE_CHILDREN (mask)[0];
    8405            0 :       SLP_TREE_REF_COUNT (new_mask)++;
    8406            0 :       SLP_TREE_CHILDREN (node)[0] = new_mask;
    8407            0 :       vect_free_slp_tree (mask);
    8408              :     }
    8409       732178 : }
    8410              : 
    8411              : /* Perform legitimizing attempts.  This is intended to improve the
    8412              :    situation when layout 0 is not valid which is a situation the cost
    8413              :    based propagation does not handle well.
    8414              :    Return true if further layout optimization is possible, false if
    8415              :    the layout configuration should be considered final.  */
    8416              : 
    8417              : bool
    8418        13871 : vect_optimize_slp_pass::legitimize ()
    8419              : {
    8420              :   /* Perform a very simple legitimizing attempt by attempting to choose
    8421              :      a single layout for all partitions that will make all permutations
    8422              :      a noop.  That should also be the optimal layout choice in case
    8423              :      layout zero is legitimate.
    8424              :      ???  Disconnected components of the SLP graph could have distinct
    8425              :      single layouts.  */
    8426        13871 :   int single_layout_i = -1;
    8427        13871 :   unsigned deferred_up_to = -1U;
    8428        42698 :   for (unsigned partition_i = 0; partition_i < m_partitions.length ();
    8429              :        ++partition_i)
    8430              :     {
    8431        35426 :       auto &partition = m_partitions[partition_i];
    8432        35426 :       if (single_layout_i == -1)
    8433              :         {
    8434        17567 :           single_layout_i = partition.layout;
    8435        17567 :           deferred_up_to = partition_i;
    8436              :         }
    8437        17859 :       else if (partition.layout == single_layout_i || partition.layout == -1)
    8438              :         ;
    8439              :       else
    8440              :         single_layout_i = 0;
    8441        32313 :       if (single_layout_i == 0)
    8442              :         return true;
    8443              : 
    8444        28912 :       if (single_layout_i != -1
    8445        28912 :           && !is_compatible_layout (partition, single_layout_i))
    8446              :         return true;
    8447              :     }
    8448              : 
    8449         7272 :   if (single_layout_i <= 0)
    8450              :     return true;
    8451              : 
    8452         7861 :   for (unsigned partition_i = 0; partition_i < deferred_up_to; ++partition_i)
    8453          606 :     if (!is_compatible_layout (m_partitions[partition_i],
    8454              :                                single_layout_i))
    8455              :       return true;
    8456              : 
    8457        22321 :   for (unsigned partition_i = 0; partition_i < m_partitions.length ();
    8458              :        ++partition_i)
    8459              :     {
    8460        15066 :       auto &partition = m_partitions[partition_i];
    8461        15066 :       partition.layout = single_layout_i;
    8462              :     }
    8463              : 
    8464              :   return false;
    8465              : }
    8466              : 
    8467              : /* Main entry point for the SLP graph optimization pass.  */
    8468              : 
    8469              : void
    8470       732178 : vect_optimize_slp_pass::run ()
    8471              : {
    8472       732178 :   build_graph ();
    8473       732178 :   create_partitions ();
    8474       732178 :   start_choosing_layouts ();
    8475       732178 :   if (m_perms.length () > 1)
    8476              :     {
    8477        13871 :       if (legitimize ())
    8478              :         {
    8479         6616 :           forward_pass ();
    8480         6616 :           backward_pass ();
    8481              :         }
    8482        13871 :       if (dump_enabled_p ())
    8483          691 :         dump ();
    8484        13871 :       materialize ();
    8485        56058 :       while (!m_perms.is_empty ())
    8486        28316 :         m_perms.pop ().release ();
    8487              :     }
    8488              :   else
    8489       718307 :     remove_redundant_permutations ();
    8490       732178 :   free_graph (m_slpg);
    8491       732178 :   build_graph ();
    8492       732178 :   decide_masked_load_lanes ();
    8493       732178 :   free_graph (m_slpg);
    8494       732178 : }
    8495              : 
    8496              : /* Apply CSE to NODE and its children using BST_MAP.  */
    8497              : 
    8498              : static void
    8499      5796717 : vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree& node)
    8500              : {
    8501      5796717 :   bool put_p = false;
    8502      5796717 :   if (SLP_TREE_DEF_TYPE (node) == vect_internal_def
    8503              :       /* Besides some VEC_PERM_EXPR, two-operator nodes also
    8504              :          lack scalar stmts and thus CSE doesn't work via bst_map.  Ideally
    8505              :          we'd have sth that works for all internal and external nodes.  */
    8506      5796717 :       && !SLP_TREE_SCALAR_STMTS (node).is_empty ())
    8507              :     {
    8508      4162453 :       slp_tree *leader = bst_map->get (SLP_TREE_SCALAR_STMTS (node));
    8509      4162453 :       if (leader)
    8510              :         {
    8511              :           /* We've visited this node already.  */
    8512       430304 :           if (!*leader || *leader == node)
    8513              :             return;
    8514              : 
    8515         4362 :           if (dump_enabled_p ())
    8516          919 :             dump_printf_loc (MSG_NOTE, vect_location,
    8517              :                              "re-using SLP tree %p for %p\n",
    8518              :                              (void *)*leader, (void *)node);
    8519         4362 :           vect_free_slp_tree (node);
    8520         4362 :           (*leader)->refcnt += 1;
    8521         4362 :           node = *leader;
    8522         4362 :           return;
    8523              :         }
    8524              : 
    8525              :       /* Avoid creating a cycle by populating the map only after recursion.  */
    8526      3732149 :       bst_map->put (SLP_TREE_SCALAR_STMTS (node).copy (), nullptr);
    8527      3732149 :       node->refcnt += 1;
    8528      3732149 :       put_p = true;
    8529              :       /* And recurse.  */
    8530              :     }
    8531              : 
    8532     16065552 :   for (slp_tree &child : SLP_TREE_CHILDREN (node))
    8533      4707781 :     if (child)
    8534      4232333 :       vect_cse_slp_nodes (bst_map, child);
    8535              : 
    8536              :   /* Now record the node for CSE in other siblings.  */
    8537      5366413 :   if (put_p)
    8538      3732149 :     *bst_map->get (SLP_TREE_SCALAR_STMTS (node)) = node;
    8539              : }
    8540              : 
    8541              : /* Associate stmts with possible starts of a subset of lanes of NODE
    8542              :    in PART_STARTS.  */
    8543              : 
    8544              : static void
    8545      2149175 : vect_cse_gather_part_starts (hash_set<slp_tree> &visited,
    8546              :                              vec<vec<slp_tree>> part_starts, slp_tree node)
    8547              : {
    8548              :   /* CSEing external nodes complicates scheduling since we materialize
    8549              :      those at the latest position, so avoid that.  */
    8550      2149175 :   if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
    8551      2149175 :       || visited.add (node))
    8552              :     return;
    8553              : 
    8554              :   /* Besides some VEC_PERM_EXPR, two-operator nodes also lack scalar stmts
    8555              :      and thus CSE doesn't work.  For now gather two-lane aligned starts
    8556              :      of nodes with a multiple of two number of lanes.  */
    8557      1156500 :   if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
    8558      1149092 :       && SLP_TREE_LANES (node) > 2
    8559       152211 :       && (SLP_TREE_LANES (node) & 1) == 0)
    8560              :     {
    8561       152211 :       auto_vec<unsigned, 8> uids;
    8562       631173 :       for (unsigned i = 0; i < SLP_TREE_LANES (node); i += 2)
    8563              :         {
    8564       478962 :           stmt_vec_info s = SLP_TREE_SCALAR_STMTS (node)[i];
    8565       478962 :           if (!s)
    8566            0 :             continue;
    8567       478962 :           unsigned uid = gimple_uid (s->stmt);
    8568       478962 :           if (!uids.contains (uid))
    8569              :             {
    8570       474944 :               uids.safe_push (uid);
    8571       474944 :               part_starts[uid].safe_push (node);
    8572              :             }
    8573              :         }
    8574       152211 :     }
    8575              : 
    8576      4492662 :   for (slp_tree &child : SLP_TREE_CHILDREN (node))
    8577      1314694 :     if (child)
    8578      1314694 :       vect_cse_gather_part_starts (visited, part_starts, child);
    8579              : }
    8580              : 
    8581              : /* Apply CSE to NODE and its children using lowparts of nodes in BST_MAP.  */
    8582              : 
    8583              : static void
    8584      2145483 : vect_cse_slp_node_parts (hash_set<slp_tree> &visited,
    8585              :                          const vec<vec<slp_tree>> part_starts,
    8586              :                          vec<slp_tree> &drops, slp_tree node)
    8587              : {
    8588      2145483 :   if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
    8589      2145483 :       || visited.add (node))
    8590              :     return;
    8591              : 
    8592              :   /* Besides some VEC_PERM_EXPR, two-operator nodes also
    8593              :      lack scalar stmts and thus CSE doesn't work.  */
    8594      1154803 :   unsigned HOST_WIDE_INT c;
    8595      1154803 :   if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
    8596      1147399 :       && SLP_TREE_SCALAR_STMTS (node)[0]
    8597              :       /* Avoid touching loads which need care with load permutations
    8598              :          and specialities like load-lane representations.  */
    8599      1147399 :       && (SLP_TREE_PERMUTE_P (node)
    8600      1139762 :           || !STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (node))))
    8601        37927 :     for (slp_tree cand
    8602       431452 :          : part_starts[gimple_uid (SLP_TREE_SCALAR_STMTS (node)[0]->stmt)])
    8603              :       /* ???  There is a possible ordering/optimality problem in that
    8604              :          the CSE then can keep a wider feeding live even though it itself
    8605              :          becomes dead by means of CSE.  Which might be solvable by doing
    8606              :          the CSE in a wide-to-narrow order.  */
    8607        38394 :       if (SLP_TREE_LANES (cand) > SLP_TREE_LANES (node)
    8608              :           /* We can do high/lo extracts and full vector copies.  */
    8609              :           && constant_multiple_p
    8610         4619 :                (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (cand)),
    8611        38886 :                 TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)), &c)
    8612        43013 :           && c <= 2)
    8613              :         {
    8614         4127 :           unsigned HOST_WIDE_INT s;
    8615         4127 :           bool const_p
    8616         4127 :             = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)).is_constant (&s);
    8617         4127 :           unsigned i;
    8618        14819 :           for (i = 0; i <= SLP_TREE_LANES (cand) - SLP_TREE_LANES (node);)
    8619              :             {
    8620              :               unsigned j;
    8621        15786 :               for (j = 0; j < SLP_TREE_LANES (node); ++j)
    8622        15319 :                 if (!SLP_TREE_SCALAR_STMTS (node)[j]
    8623        15319 :                     || (SLP_TREE_SCALAR_STMTS (cand)[i+j]
    8624        15319 :                         != SLP_TREE_SCALAR_STMTS (node)[j]))
    8625              :                   break;
    8626        11159 :               if (j == SLP_TREE_LANES (node))
    8627              :                 break;
    8628        10692 :               if (!const_p)
    8629              :                 {
    8630              :                   i = SLP_TREE_LANES (cand);
    8631              :                   break;
    8632              :                 }
    8633              :               /* We can extract only aligned on node vector type boundary.  */
    8634        10692 :               i += s;
    8635              :             }
    8636         4127 :           if (i > SLP_TREE_LANES (cand) - SLP_TREE_LANES (node))
    8637         3660 :             continue;
    8638              :           /* Found node within cand at i.  Put a permute in place
    8639              :              of it, selecting the subset from cand.  */
    8640          467 :           if (dump_enabled_p ())
    8641           41 :             dump_printf (MSG_NOTE, "CSEd node %p as %spart of node %p\n",
    8642              :                          (void *)node, i == 0 ? "low" : "high", (void *)cand);
    8643         2302 :           for (slp_tree child : SLP_TREE_CHILDREN (node))
    8644              :             /* Delay SLP tree release since we might still reference a node
    8645              :                from the part_starts map.  */
    8646          901 :             drops.safe_push (child);
    8647          467 :           SLP_TREE_CHILDREN (node).truncate (1);
    8648          467 :           SLP_TREE_REF_COUNT (cand)++;
    8649          467 :           SLP_TREE_CHILDREN (node)[0] = cand;
    8650          467 :           SLP_TREE_CODE (node) = VEC_PERM_EXPR;
    8651          467 :           SLP_TREE_REPRESENTATIVE (node) = NULL;
    8652          467 :           SLP_TREE_LANE_PERMUTATION (node).create (SLP_TREE_LANES (node));
    8653         2000 :           for (unsigned j = i; j < i + SLP_TREE_LANES (node); ++j)
    8654         1066 :             SLP_TREE_LANE_PERMUTATION (node).quick_push (std::make_pair (0, j));
    8655      2145483 :           return;
    8656              :         }
    8657              : 
    8658      4483052 :   for (slp_tree &child : SLP_TREE_CHILDREN (node))
    8659      1311002 :     if (child)
    8660      1311002 :       vect_cse_slp_node_parts (visited, part_starts, drops, child);
    8661              : }
    8662              : 
    8663              : /* Optimize the SLP graph of VINFO.  */
    8664              : 
    8665              : void
    8666      1127020 : vect_optimize_slp (vec_info *vinfo)
    8667              : {
    8668      1127020 :   if (vinfo->slp_instances.is_empty ())
    8669       844495 :     return;
    8670       732178 :   vect_optimize_slp_pass (vinfo).run ();
    8671              : 
    8672              :   /* Apply CSE again to nodes after permute optimization.  */
    8673       732178 :   scalar_stmts_to_slp_tree_map_t *bst_map
    8674       732178 :     = new scalar_stmts_to_slp_tree_map_t ();
    8675              : 
    8676      3760918 :   for (auto inst : vinfo->slp_instances)
    8677      1564384 :     vect_cse_slp_nodes (bst_map, SLP_INSTANCE_TREE (inst));
    8678              : 
    8679       732178 :   release_scalar_stmts_to_slp_tree_map (bst_map);
    8680              : 
    8681       732178 :   if (!is_a <bb_vec_info> (vinfo))
    8682              :     return;
    8683              : 
    8684              :   /* Attempt to merge SLP sub-graphs that intersect in low or highparts of
    8685              :      each other.  Build the reverse mapping from stmt to SLP node for
    8686              :      lanes starting at the low or high part.
    8687              :      ???  In the future we can extend this to do a two-step permute
    8688              :      and extract or extract and permute to put the high/low part in
    8689              :      place on the original vector or permute the hogh/low part to
    8690              :      match up the target lane order.  */
    8691       282525 :   hash_set<slp_tree> visited;
    8692       282525 :   vec<vec<slp_tree>> start_for_part;
    8693       565050 :   start_for_part.create (vinfo->stmt_vec_infos.length () + 1);
    8694       565050 :   start_for_part.quick_grow_cleared (vinfo->stmt_vec_infos.length () + 1);
    8695      1682056 :   for (auto inst : vinfo->slp_instances)
    8696       834481 :     vect_cse_gather_part_starts (visited,
    8697              :                                  start_for_part, SLP_INSTANCE_TREE (inst));
    8698              : 
    8699              :   /* Now replace low/highpart copies with extracting permutes.  */
    8700       282525 :   auto_vec<slp_tree> drops;
    8701       282525 :   visited.empty ();
    8702      1682056 :   for (auto inst : vinfo->slp_instances)
    8703       834481 :     vect_cse_slp_node_parts (visited, start_for_part, drops,
    8704              :                              SLP_INSTANCE_TREE (inst));
    8705              : 
    8706              :   /* Now perform delayed releases of nodes.  */
    8707       283920 :   for (slp_tree node : drops)
    8708          901 :     vect_free_slp_tree (node);
    8709              : 
    8710     24239811 :   for (auto v : start_for_part)
    8711     23392236 :     v.release ();
    8712       282525 :   start_for_part.release ();
    8713       282525 : }
    8714              : 
    8715              : /* Gather loads reachable from the individual SLP graph entries.  */
    8716              : 
    8717              : void
    8718      1127020 : vect_gather_slp_loads (vec_info *vinfo)
    8719              : {
    8720      1127020 :   unsigned i;
    8721      1127020 :   slp_instance instance;
    8722      2691404 :   FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
    8723              :     {
    8724      1564384 :       hash_set<slp_tree> visited;
    8725      1564384 :       vect_gather_slp_loads (SLP_INSTANCE_LOADS (instance),
    8726              :                              SLP_INSTANCE_TREE (instance), visited);
    8727      1564384 :     }
    8728      1127020 : }
    8729              : 
    8730              : /* For NODE update VF based on the number of lanes and the vector types
    8731              :    used.  */
    8732              : 
    8733              : static void
    8734      4659079 : vect_update_slp_vf_for_node (slp_tree node, poly_uint64 &vf,
    8735              :                              hash_set<slp_tree> &visited)
    8736              : {
    8737      4659079 :   if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
    8738      1689362 :     return;
    8739      3355859 :   if (visited.add (node))
    8740              :     return;
    8741              : 
    8742     11322038 :   for (slp_tree child : SLP_TREE_CHILDREN (node))
    8743      3833179 :     vect_update_slp_vf_for_node (child, vf, visited);
    8744              : 
    8745              :   /* We do not visit SLP nodes for constants or externals - those neither
    8746              :      have a vector type set yet (vectorizable_* does this).
    8747              :      Note that when we stop using fixed size vectors externs and constants
    8748              :      shouldn't influence the (minimum) vectorization factor, instead
    8749              :      vectorizable_* should honor the vectorization factor when trying to
    8750              :      assign vector types to constants and externals and cause iteration
    8751              :      to a higher vectorization factor when required.  */
    8752      2969747 :   tree vectype = SLP_TREE_VECTYPE (node);
    8753      2969747 :   if (!vectype)
    8754              :     /* OMP SIMD calls w/o LHS have no SLP_TREE_VECTYPE set.  */
    8755              :     return;
    8756      2969717 :   poly_uint64 node_vf
    8757      2969717 :     = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
    8758              :                                   SLP_TREE_LANES (node));
    8759      2969717 :   vf = force_common_multiple (vf, node_vf);
    8760              : 
    8761              :   /* For permute nodes that are fed from externs or constants we have to
    8762              :      consider their number of lanes as well.  Likewise for store-lanes.  */
    8763      2969717 :   if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
    8764       745645 :     for (slp_tree child : SLP_TREE_CHILDREN (node))
    8765       204460 :       if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
    8766              :         {
    8767         3684 :           poly_uint64 child_vf
    8768         3684 :             = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
    8769              :                                           SLP_TREE_LANES (child));
    8770         3684 :           vf = force_common_multiple (vf, child_vf);
    8771              :         }
    8772              : }
    8773              : 
    8774              : /* For each possible SLP instance decide whether to SLP it and calculate overall
    8775              :    unrolling factor needed to SLP the loop.  Return TRUE if decided to SLP at
    8776              :    least one instance.  */
    8777              : 
    8778              : bool
    8779       517993 : vect_make_slp_decision (loop_vec_info loop_vinfo)
    8780              : {
    8781       517993 :   unsigned int i;
    8782       517993 :   poly_uint64 unrolling_factor = 1;
    8783       517993 :   const vec<slp_instance> &slp_instances
    8784              :     = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
    8785       517993 :   slp_instance instance;
    8786       517993 :   int decided_to_slp = 0;
    8787              : 
    8788       517993 :   DUMP_VECT_SCOPE ("vect_make_slp_decision");
    8789              : 
    8790       517993 :   hash_set<slp_tree> visited;
    8791      1861886 :   FOR_EACH_VEC_ELT (slp_instances, i, instance)
    8792              :     {
    8793       825900 :       slp_tree root = SLP_INSTANCE_TREE (instance);
    8794              : 
    8795              :       /* All unroll factors have the form:
    8796              : 
    8797              :            GET_MODE_SIZE (vinfo->vector_mode) * X
    8798              : 
    8799              :          for some rational X, so they must have a common multiple.  */
    8800       825900 :       vect_update_slp_vf_for_node (root, unrolling_factor, visited);
    8801              : 
    8802              :       /* If all instances ended up with vector(1) T roots make sure to
    8803              :          not vectorize.  RVV for example relies on loop vectorization
    8804              :          when some instances are essentially kept scalar.  See PR121048.  */
    8805       825900 :       if (SLP_TREE_VECTYPE (root)
    8806       825900 :           && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
    8807       669498 :         decided_to_slp++;
    8808              :     }
    8809              : 
    8810       517993 :   LOOP_VINFO_VECT_FACTOR (loop_vinfo) = unrolling_factor;
    8811              : 
    8812       517993 :   if (decided_to_slp && dump_enabled_p ())
    8813              :     {
    8814        20930 :       dump_printf_loc (MSG_NOTE, vect_location,
    8815              :                        "Decided to SLP %d instances. Unrolling factor ",
    8816              :                        decided_to_slp);
    8817        20930 :       dump_dec (MSG_NOTE, unrolling_factor);
    8818        20930 :       dump_printf (MSG_NOTE, "\n");
    8819              :     }
    8820              : 
    8821       517993 :   return (decided_to_slp > 0);
    8822       517993 : }
    8823              : 
    8824              : /* Initialize a bb_vec_info struct for the statements in BBS basic blocks.  */
    8825              : 
    8826      2312925 : _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
    8827              :   : vec_info (vec_info::bb, shared),
    8828      2312925 :     roots (vNULL)
    8829              : {
    8830              :   /* The region we are operating on.  bbs[0] is the entry, excluding
    8831              :      its PHI nodes.  In the future we might want to track an explicit
    8832              :      entry edge to cover bbs[0] PHI nodes and have a region entry
    8833              :      insert location.  */
    8834      2312925 :   bbs = _bbs.address ();
    8835      2312925 :   nbbs = _bbs.length ();
    8836              : 
    8837     18469388 :   for (unsigned i = 0; i < nbbs; ++i)
    8838              :     {
    8839     16156463 :       if (i != 0)
    8840     21066647 :         for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
    8841      7223109 :              gsi_next (&si))
    8842              :           {
    8843      7223109 :             gphi *phi = si.phi ();
    8844      7223109 :             gimple_set_uid (phi, 0);
    8845      7223109 :             add_stmt (phi);
    8846              :           }
    8847     32312926 :       for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
    8848    147734531 :            !gsi_end_p (gsi); gsi_next (&gsi))
    8849              :         {
    8850    131578068 :           gimple *stmt = gsi_stmt (gsi);
    8851    131578068 :           gimple_set_uid (stmt, 0);
    8852    131578068 :           if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
    8853     84458367 :             continue;
    8854     47119701 :           add_stmt (stmt);
    8855              :         }
    8856              :     }
    8857      2312925 : }
    8858              : 
    8859              : 
    8860              : /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
    8861              :    stmts in the basic block.  */
    8862              : 
    8863      2312925 : _bb_vec_info::~_bb_vec_info ()
    8864              : {
    8865              :   /* Reset region marker.  */
    8866     18469388 :   for (unsigned i = 0; i < nbbs; ++i)
    8867              :     {
    8868     16156463 :       if (i != 0)
    8869     21080644 :         for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
    8870      7237106 :              gsi_next (&si))
    8871              :           {
    8872      7237106 :             gphi *phi = si.phi ();
    8873      7237106 :             gimple_set_uid (phi, -1);
    8874              :           }
    8875     32312926 :       for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
    8876    147608310 :            !gsi_end_p (gsi); gsi_next (&gsi))
    8877              :         {
    8878    131451847 :           gimple *stmt = gsi_stmt (gsi);
    8879    131451847 :           gimple_set_uid (stmt, -1);
    8880              :         }
    8881              :     }
    8882              : 
    8883      3694508 :   for (unsigned i = 0; i < roots.length (); ++i)
    8884              :     {
    8885      1381583 :       roots[i].stmts.release ();
    8886      1381583 :       roots[i].roots.release ();
    8887      1381583 :       roots[i].remain.release ();
    8888              :     }
    8889      2312925 :   roots.release ();
    8890      2312925 : }
    8891              : 
    8892              : /* Subroutine of vect_slp_analyze_node_operations.  Handle the root of NODE,
    8893              :    given then that child nodes have already been processed, and that
    8894              :    their def types currently match their SLP node's def type.  */
    8895              : 
    8896              : static bool
    8897      2757127 : vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
    8898              :                                     slp_instance node_instance,
    8899              :                                     stmt_vector_for_cost *cost_vec)
    8900              : {
    8901              :   /* Handle purely internal nodes.  */
    8902      2757127 :   if (SLP_TREE_PERMUTE_P (node))
    8903              :     {
    8904        83167 :       gcc_checking_assert (!SLP_TREE_REPRESENTATIVE (node));
    8905        83167 :       if (!vectorizable_slp_permutation (vinfo, NULL, node, cost_vec))
    8906              :         return false;
    8907              : 
    8908              :       stmt_vec_info slp_stmt_info;
    8909              :       unsigned int i;
    8910       205178 :       FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, slp_stmt_info)
    8911              :         {
    8912       123632 :           if (slp_stmt_info
    8913       117554 :               && STMT_VINFO_LIVE_P (slp_stmt_info)
    8914       123632 :               && !vectorizable_live_operation (vinfo, slp_stmt_info, node,
    8915              :                                                node_instance, i,
    8916              :                                                false, cost_vec))
    8917              :             return false;
    8918              :         }
    8919        81546 :       SLP_TREE_TYPE (node) = permute_info_type;
    8920        81546 :       return true;
    8921              :     }
    8922              : 
    8923      2673960 :   return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
    8924              : }
    8925              : 
    8926              : static int
    8927       913493 : sort_ints (const void *a_, const void *b_)
    8928              : {
    8929       913493 :   int a = *(const int *)a_;
    8930       913493 :   int b = *(const int *)b_;
    8931       913493 :   return a - b;
    8932              : }
    8933              : 
    8934              : /* Verify if we can externalize a set of internal defs.  */
    8935              : 
    8936              : static bool
    8937       412541 : vect_slp_can_convert_to_external (const vec<stmt_vec_info> &stmts)
    8938              : {
    8939              :   /* Constant generation uses get_later_stmt which can only handle
    8940              :      defs from the same BB or a set of defs that can be ordered
    8941              :      with a dominance query.  */
    8942       412541 :   basic_block bb = NULL;
    8943       412541 :   bool all_same = true;
    8944       412541 :   auto_vec<int> bbs;
    8945       825082 :   bbs.reserve_exact (stmts.length ());
    8946      2175175 :   for (stmt_vec_info stmt : stmts)
    8947              :     {
    8948       937552 :       if (!stmt)
    8949              :         return false;
    8950       937552 :       else if (!bb)
    8951       412541 :         bb = gimple_bb (stmt->stmt);
    8952       525011 :       else if (gimple_bb (stmt->stmt) != bb)
    8953       157408 :         all_same = false;
    8954       937552 :       bbs.quick_push (gimple_bb (stmt->stmt)->index);
    8955              :     }
    8956       412541 :   if (all_same)
    8957              :     return true;
    8958              : 
    8959              :   /* Produce a vector of unique BB indexes for the defs.  */
    8960       140513 :   bbs.qsort (sort_ints);
    8961       140513 :   unsigned i, j;
    8962       304164 :   for (i = 1, j = 1; i < bbs.length (); ++i)
    8963       163651 :     if (bbs[i] != bbs[j-1])
    8964       144424 :       bbs[j++] = bbs[i];
    8965       140513 :   gcc_assert (j >= 2);
    8966       140513 :   bbs.truncate (j);
    8967              : 
    8968       281026 :   if (bbs.length () == 2)
    8969       138815 :     return (dominated_by_p (CDI_DOMINATORS,
    8970       138815 :                             BASIC_BLOCK_FOR_FN (cfun, bbs[0]),
    8971       138815 :                             BASIC_BLOCK_FOR_FN (cfun, bbs[1]))
    8972       271161 :             || dominated_by_p (CDI_DOMINATORS,
    8973       132346 :                                BASIC_BLOCK_FOR_FN (cfun, bbs[1]),
    8974       132346 :                                BASIC_BLOCK_FOR_FN (cfun, bbs[0])));
    8975              : 
    8976              :   /* ???  For more than two BBs we can sort the vector and verify the
    8977              :      result is a total order.  But we can't use vec::qsort with a
    8978              :      compare function using a dominance query since there's no way to
    8979              :      signal failure and any fallback for an unordered pair would
    8980              :      fail qsort_chk later.
    8981              :      For now simply hope that ordering after BB index provides the
    8982              :      best candidate total order.  If required we can implement our
    8983              :      own mergesort or export an entry without checking.  */
    8984       419790 :   for (unsigned i = 1; i < bbs.length (); ++i)
    8985         5585 :     if (!dominated_by_p (CDI_DOMINATORS,
    8986         5585 :                          BASIC_BLOCK_FOR_FN (cfun, bbs[i]),
    8987         5585 :                          BASIC_BLOCK_FOR_FN (cfun, bbs[i-1])))
    8988              :       return false;
    8989              : 
    8990              :   return true;
    8991       412541 : }
    8992              : 
    8993              : /* Try to build NODE from scalars, returning true on success.
    8994              :    NODE_INSTANCE is the SLP instance that contains NODE.  */
    8995              : 
    8996              : static bool
    8997       675829 : vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
    8998              :                               slp_instance node_instance)
    8999              : {
    9000       675829 :   stmt_vec_info stmt_info;
    9001       675829 :   unsigned int i;
    9002              : 
    9003       675829 :   if (!is_a <bb_vec_info> (vinfo)
    9004        83418 :       || node == SLP_INSTANCE_TREE (node_instance)
    9005        25051 :       || !SLP_TREE_SCALAR_STMTS (node).exists ()
    9006        25010 :       || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node))
    9007              :       /* Force the mask use to be built from scalars instead.  */
    9008        22953 :       || VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node))
    9009       698533 :       || !vect_slp_can_convert_to_external (SLP_TREE_SCALAR_STMTS (node)))
    9010              :     return false;
    9011              : 
    9012        22704 :   if (dump_enabled_p ())
    9013          100 :     dump_printf_loc (MSG_NOTE, vect_location,
    9014              :                      "Building vector operands of %p from scalars instead\n",
    9015              :                      (void *) node);
    9016              : 
    9017              :   /* Don't remove and free the child nodes here, since they could be
    9018              :      referenced by other structures.  The analysis and scheduling phases
    9019              :      (need to) ignore child nodes of anything that isn't vect_internal_def.  */
    9020        22704 :   unsigned int group_size = SLP_TREE_LANES (node);
    9021        22704 :   SLP_TREE_DEF_TYPE (node) = vect_external_def;
    9022              :   /* Invariants get their vector type from the uses.  */
    9023        22704 :   SLP_TREE_VECTYPE (node) = NULL_TREE;
    9024        22704 :   SLP_TREE_SCALAR_OPS (node).safe_grow (group_size, true);
    9025        22704 :   SLP_TREE_LOAD_PERMUTATION (node).release ();
    9026       103142 :   FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
    9027              :     {
    9028        57734 :       tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
    9029        57734 :       SLP_TREE_SCALAR_OPS (node)[i] = lhs;
    9030              :     }
    9031              :   return true;
    9032              : }
    9033              : 
    9034              : /* Return true if all elements of the slice are the same.  */
    9035              : bool
    9036       451335 : vect_scalar_ops_slice::all_same_p () const
    9037              : {
    9038       503208 :   for (unsigned int i = 1; i < length; ++i)
    9039       415704 :     if (!operand_equal_p (op (0), op (i)))
    9040              :       return false;
    9041              :   return true;
    9042              : }
    9043              : 
    9044              : hashval_t
    9045       426283 : vect_scalar_ops_slice_hash::hash (const value_type &s)
    9046              : {
    9047       426283 :   hashval_t hash = 0;
    9048      1630612 :   for (unsigned i = 0; i < s.length; ++i)
    9049      1204329 :     hash = iterative_hash_expr (s.op (i), hash);
    9050       426283 :   return hash;
    9051              : }
    9052              : 
    9053              : bool
    9054       234126 : vect_scalar_ops_slice_hash::equal (const value_type &s1,
    9055              :                                    const compare_type &s2)
    9056              : {
    9057       234126 :   if (s1.length != s2.length)
    9058              :     return false;
    9059       401295 :   for (unsigned i = 0; i < s1.length; ++i)
    9060       350751 :     if (!operand_equal_p (s1.op (i), s2.op (i)))
    9061              :       return false;
    9062              :   return true;
    9063              : }
    9064              : 
    9065              : /* Like vect_get_num_copies but N copies of the vector might have
    9066              :    excess elements in the last vector.  Stores the number of excess
    9067              :    elements in the last vector in *EXCESS_ELTS.
    9068              :    Returns false if *NVECTORS or *EXCESS_ELTS cannot be computed.  */
    9069              : 
    9070              : bool
    9071      1608656 : vect_get_num_copies_for_invariant (vec_info *vinfo, slp_tree node,
    9072              :                                    unsigned *nvectors,
    9073              :                                    unsigned *excess_elts)
    9074              : {
    9075      1608656 :   poly_uint64 vf;
    9076              : 
    9077      1608656 :   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
    9078       385506 :     vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    9079              :   else
    9080      1223150 :     vf = 1;
    9081      1608656 :   vf *= SLP_TREE_LANES (node);
    9082              : 
    9083      1608656 :   tree vectype = SLP_TREE_VECTYPE (node);
    9084      1608656 :   uint64_t rem;
    9085      1608656 :   if (can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
    9086              :                                 nvectors)
    9087      1608656 :       && (TYPE_VECTOR_SUBPARTS (vectype) * *nvectors - vf) .is_constant (&rem))
    9088              :     {
    9089      1608656 :       *excess_elts = rem;
    9090      1608656 :       return true;
    9091              :     }
    9092              :   return false;
    9093              : }
    9094              : 
    9095              : /* Compute the prologue cost for invariant or constant operands represented
    9096              :    by NODE.  */
    9097              : 
    9098              : static void
    9099      1104857 : vect_prologue_cost_for_slp (slp_tree node, unsigned nvectors,
    9100              :                             stmt_vector_for_cost *cost_vec)
    9101              : {
    9102              :   /* There's a special case of an existing vector, that costs nothing.  */
    9103      1104857 :   if (SLP_TREE_SCALAR_OPS (node).length () == 0
    9104      1104857 :       && !SLP_TREE_VEC_DEFS (node).is_empty ())
    9105         2314 :     return;
    9106              :   /* Without looking at the actual initializer a vector of
    9107              :      constants can be implemented as load from the constant pool.
    9108              :      When all elements are the same we can use a splat.  */
    9109      1102543 :   tree vectype = SLP_TREE_VECTYPE (node);
    9110      1102543 :   unsigned group_size = SLP_TREE_LANES (node);
    9111      1102543 :   unsigned HOST_WIDE_INT const_nunits;
    9112      1102543 :   unsigned nelt_limit;
    9113      1102543 :   auto ops = &SLP_TREE_SCALAR_OPS (node);
    9114      1102543 :   auto_vec<unsigned int> starts (nvectors);
    9115      1102543 :   if (TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
    9116      1102543 :       && ! multiple_p (const_nunits, group_size))
    9117              :     {
    9118        67685 :       nelt_limit = const_nunits;
    9119        67685 :       hash_set<vect_scalar_ops_slice_hash> vector_ops;
    9120       347006 :       for (unsigned int i = 0; i < nvectors; ++i)
    9121       211636 :         if (!vector_ops.add ({ ops, i * nelt_limit, nelt_limit }))
    9122       161092 :           starts.quick_push (i * nelt_limit);
    9123        67685 :     }
    9124              :   else
    9125              :     {
    9126              :       /* If either the vector has variable length or the vectors
    9127              :          are composed of repeated whole groups we only need to
    9128              :          cost construction once.  All vectors will be the same.  */
    9129      1034858 :       nelt_limit = group_size;
    9130      1034858 :       starts.quick_push (0);
    9131              :     }
    9132              :   /* ???  We're just tracking whether vectors in a single node are the same.
    9133              :      Ideally we'd do something more global.  */
    9134      4503579 :   for (unsigned int start : starts)
    9135              :     {
    9136      1195950 :       vect_cost_for_stmt kind;
    9137      1195950 :       if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
    9138              :         kind = vector_load;
    9139       451335 :       else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
    9140              :         kind = scalar_to_vec;
    9141              :       else
    9142       363831 :         kind = vec_construct;
    9143      1195950 :       record_stmt_cost (cost_vec, 1, kind, nullptr, node,
    9144              :                         vectype, 0, vect_prologue);
    9145              :     }
    9146      1102543 : }
    9147              : 
    9148              : /* Analyze statements contained in SLP tree NODE after recursively analyzing
    9149              :    the subtree.  NODE_INSTANCE contains NODE and VINFO contains INSTANCE.
    9150              : 
    9151              :    Return true if the operations are supported.  */
    9152              : 
    9153              : static bool
    9154      5138347 : vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
    9155              :                                   slp_instance node_instance,
    9156              :                                   hash_set<slp_tree> &visited_set,
    9157              :                                   vec<slp_tree> &visited_vec,
    9158              :                                   stmt_vector_for_cost *cost_vec)
    9159              : {
    9160      5138347 :   int i, j;
    9161      5138347 :   slp_tree child;
    9162              : 
    9163              :   /* Costing and analysis of invariants is delayed.  */
    9164      5138347 :   if (!node
    9165      4729037 :       || SLP_TREE_DEF_TYPE (node) == vect_constant_def
    9166      3925623 :       || SLP_TREE_DEF_TYPE (node) == vect_external_def)
    9167              :     return true;
    9168              : 
    9169      3438647 :   if (SLP_TREE_DEF_TYPE (node) == vect_uninitialized_def)
    9170              :     {
    9171            5 :       if (dump_enabled_p ())
    9172            0 :         dump_printf_loc (MSG_NOTE, vect_location,
    9173              :                          "Failed cyclic SLP reference in %p\n", (void *) node);
    9174              :       return false;
    9175              :     }
    9176      3438642 :   gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_internal_def);
    9177              : 
    9178              :   /* If we already analyzed the exact same set of scalar stmts we're done.
    9179              :      We share the generated vector stmts for those.  */
    9180      3438642 :   if (visited_set.add (node))
    9181              :     return true;
    9182      3145298 :   visited_vec.safe_push (node);
    9183              : 
    9184      3145298 :   bool res = true;
    9185      3145298 :   unsigned visited_rec_start = visited_vec.length ();
    9186      3145298 :   unsigned cost_vec_rec_start = cost_vec->length ();
    9187      3145298 :   bool seen_non_constant_child = false;
    9188      9077024 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    9189              :     {
    9190      3749287 :       res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
    9191              :                                               visited_set, visited_vec,
    9192              :                                               cost_vec);
    9193      3749287 :       if (!res)
    9194              :         break;
    9195      3361334 :       if (child && SLP_TREE_DEF_TYPE (child) != vect_constant_def)
    9196      3361334 :         seen_non_constant_child = true;
    9197              :     }
    9198              :   /* We're having difficulties scheduling nodes with just constant
    9199              :      operands and no scalar stmts since we then cannot compute a stmt
    9200              :      insertion place.  */
    9201      3145298 :   if (res
    9202      3145298 :       && !seen_non_constant_child
    9203      3145298 :       && SLP_TREE_SCALAR_STMTS (node).is_empty ())
    9204              :     {
    9205          218 :       if (dump_enabled_p ())
    9206            6 :         dump_printf_loc (MSG_NOTE, vect_location,
    9207              :                          "Cannot vectorize all-constant op node %p\n",
    9208              :                          (void *) node);
    9209              :       res = false;
    9210              :     }
    9211              : 
    9212      3145086 :   if (res)
    9213      2757127 :     res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
    9214              :                                               cost_vec);
    9215              :   /* If analysis failed we have to pop all recursive visited nodes
    9216              :      plus ourselves.  */
    9217      3145298 :   if (!res)
    9218              :     {
    9219      3297876 :       while (visited_vec.length () >= visited_rec_start)
    9220       973109 :         visited_set.remove (visited_vec.pop ());
    9221       675829 :       cost_vec->truncate (cost_vec_rec_start);
    9222              :     }
    9223              : 
    9224              :   /* When the node can be vectorized cost invariant nodes it references.
    9225              :      This is not done in DFS order to allow the referring node
    9226              :      vectorizable_* calls to nail down the invariant nodes vector type
    9227              :      and possibly unshare it if it needs a different vector type than
    9228              :      other referrers.  */
    9229      3145298 :   if (res)
    9230      9488049 :     FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
    9231      2996880 :       if (child
    9232      2671924 :           && (SLP_TREE_DEF_TYPE (child) == vect_constant_def
    9233      2671924 :               || SLP_TREE_DEF_TYPE (child) == vect_external_def)
    9234              :           /* Perform usual caching, note code-generation still
    9235              :              code-gens these nodes multiple times but we expect
    9236              :              to CSE them later.  */
    9237      4147323 :           && !visited_set.add (child))
    9238              :         {
    9239      1147394 :           visited_vec.safe_push (child);
    9240              :           /* ???  After auditing more code paths make a "default"
    9241              :              and push the vector type from NODE to all children
    9242              :              if it is not already set.  */
    9243              :           /* Compute the number of vectors to be generated.  */
    9244      1147394 :           tree vector_type = SLP_TREE_VECTYPE (child);
    9245      1147394 :           if (!vector_type)
    9246              :             {
    9247              :               /* Masked loads can have an undefined (default SSA definition)
    9248              :                  else operand.  We do not need to cost it.  */
    9249        42537 :               vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
    9250        43978 :               if (SLP_TREE_TYPE (node) == load_vec_info_type
    9251        43978 :                   && ((ops.length ()
    9252         1441 :                        && TREE_CODE (ops[0]) == SSA_NAME
    9253            0 :                        && SSA_NAME_IS_DEFAULT_DEF (ops[0])
    9254            0 :                        && VAR_P (SSA_NAME_VAR (ops[0])))
    9255         1441 :                       || SLP_TREE_DEF_TYPE (child) == vect_constant_def))
    9256         1441 :                 continue;
    9257              : 
    9258              :               /* For shifts with a scalar argument we don't need
    9259              :                  to cost or code-generate anything.
    9260              :                  ???  Represent this more explicitly.  */
    9261        41096 :               gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
    9262              :                           && j == 1);
    9263        41096 :               continue;
    9264        41096 :             }
    9265              : 
    9266              :           /* Make sure we can generate them and then cost them.  */
    9267      1104857 :           unsigned nvectors, excess_elts;
    9268      1104857 :           if (!vect_get_num_copies_for_invariant (vinfo, child, &nvectors,
    9269              :                                                   &excess_elts))
    9270            0 :             return false;
    9271      1104857 :           vect_prologue_cost_for_slp (child, nvectors, cost_vec);
    9272              :         }
    9273              : 
    9274              :   /* If this node or any of its children can't be vectorized, try pruning
    9275              :      the tree here rather than felling the whole thing.  */
    9276       675829 :   if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
    9277              :     {
    9278              :       /* We'll need to revisit this for invariant costing and number
    9279              :          of vectorized stmt setting.   */
    9280              :       res = true;
    9281              :     }
    9282              : 
    9283              :   return res;
    9284              : }
    9285              : 
    9286              : /* Mark lanes of NODE that are live outside of the basic-block vectorized
    9287              :    region and that can be vectorized using vectorizable_live_operation
    9288              :    with STMT_VINFO_LIVE_P.  Not handled live operations will cause the
    9289              :    scalar code computing it to be retained.  */
    9290              : 
    9291              : static void
    9292      1011752 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node,
    9293              :                              slp_instance instance,
    9294              :                              stmt_vector_for_cost *cost_vec,
    9295              :                              hash_set<stmt_vec_info> &svisited,
    9296              :                              hash_set<slp_tree> &visited)
    9297              : {
    9298      1011752 :   if (visited.add (node))
    9299        58901 :     return;
    9300              : 
    9301              :   unsigned i;
    9302              :   stmt_vec_info stmt_info;
    9303              :   gimple *last_stmt = NULL;
    9304      3416365 :   FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
    9305              :     {
    9306      2463514 :       if (!stmt_info || svisited.contains (stmt_info))
    9307        96614 :         continue;
    9308      2427254 :       stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
    9309      2427254 :       if (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)
    9310        41182 :           && STMT_VINFO_RELATED_STMT (orig_stmt_info) != stmt_info)
    9311              :         /* Only the pattern root stmt computes the original scalar value.  */
    9312        30457 :         continue;
    9313      2396797 :       if (!PURE_SLP_STMT (orig_stmt_info))
    9314              :         /* Iff the stmt is not part of the vector coverage because it or
    9315              :            uses of it are used by SLP graph leafs as extern input there is
    9316              :            no point in trying to live code-generate from a vector stmt as
    9317              :            the scalar stmt will survive anyway.  */
    9318        29897 :         continue;
    9319      2366900 :       bool mark_visited = true;
    9320      2366900 :       gimple *orig_stmt = orig_stmt_info->stmt;
    9321      2366900 :       ssa_op_iter op_iter;
    9322      2366900 :       def_operand_p def_p;
    9323      5323530 :       FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_stmt, op_iter, SSA_OP_DEF)
    9324              :         {
    9325              :           /* We have to verify whether we can insert the lane extract
    9326              :              before all uses.  The following is a conservative approximation.
    9327              :              We cannot put this into vectorizable_live_operation because
    9328              :              iterating over all use stmts from inside a FOR_EACH_IMM_USE_STMT
    9329              :              doesn't work.
    9330              :              Note that while the fact that we emit code for loads at the
    9331              :              first load should make this a non-problem leafs we construct
    9332              :              from scalars are vectorized after the last scalar def.
    9333              :              ???  If we'd actually compute the insert location during
    9334              :              analysis we could use sth less conservative than the last
    9335              :              scalar stmt in the node for the dominance check.  */
    9336              :           /* ???  What remains is "live" uses in vector CTORs in the same
    9337              :              SLP graph which is where those uses can end up code-generated
    9338              :              right after their definition instead of close to their original
    9339              :              use.  But that would restrict us to code-generate lane-extracts
    9340              :              from the latest stmt in a node.  So we compensate for this
    9341              :              during code-generation, simply not replacing uses for those
    9342              :              hopefully rare cases.  */
    9343       589730 :           imm_use_iterator use_iter;
    9344              : 
    9345       589730 :           bool live_p = false;
    9346       589730 :           bool can_insert = true;
    9347       589730 :           use_operand_p use_p;
    9348      1761626 :           FOR_EACH_IMM_USE_FAST (use_p, use_iter, DEF_FROM_PTR (def_p))
    9349              :             {
    9350      1180005 :               gimple *use_stmt = USE_STMT (use_p);
    9351      1180005 :               stmt_vec_info use_stmt_info;
    9352      1180005 :               if (!(!is_gimple_debug (use_stmt)
    9353       912285 :                     && (!(use_stmt_info = bb_vinfo->lookup_stmt (use_stmt))
    9354       900800 :                         || !PURE_SLP_STMT (use_stmt_info))))
    9355       961835 :                 continue;
    9356       218170 :               live_p = true;
    9357       218170 :               if (!last_stmt)
    9358        67758 :                 last_stmt
    9359        68024 :                   = (node->si ? node->si
    9360          266 :                      : vect_find_last_scalar_stmt_in_slp (node)->stmt);
    9361       218170 :               if (is_a <gphi *> (use_stmt))
    9362              :                 {
    9363        80416 :                   if (!dominated_by_p (CDI_DOMINATORS,
    9364        40208 :                                        phi_arg_edge_from_use (use_p)->src,
    9365        40208 :                                        gimple_bb (last_stmt)))
    9366              :                     can_insert = false;
    9367              :                 }
    9368              :               /* As we instert after last_stmt it may not be the use_stmt
    9369              :                  itself.  */
    9370       177962 :               else if (last_stmt == use_stmt
    9371       177962 :                        || !vect_stmt_dominates_stmt_p (last_stmt, use_stmt))
    9372              :                 can_insert = false;
    9373              :               if (!can_insert)
    9374              :                 {
    9375         8109 :                   if (dump_enabled_p ())
    9376           82 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9377              :                                      "Cannot determine insertion place for "
    9378              :                                      "lane extract of %T at node %p\n",
    9379              :                                      DEF_FROM_PTR (def_p), (void *)node);
    9380              :                   can_insert = false;
    9381              :                   break;
    9382              :                 }
    9383       589730 :             }
    9384       589730 :           if (live_p && can_insert)
    9385              :             {
    9386              :               /* Only record a live stmt when we can replace all uses.  We
    9387              :                  record from which SLP tree we vectorize the uses, so we'll
    9388              :                  cost once and can deal with the case that not all SLP nodes
    9389              :                  may be suitable for code-generation of all live uses.
    9390              :                  ???  But we never split up the work between multiple SLP
    9391              :                  nodes.  */
    9392       109070 :               STMT_VINFO_LIVE_P (stmt_info) = true;
    9393       109070 :               if (!vectorizable_live_operation (bb_vinfo, stmt_info, node,
    9394              :                                                 instance, i, false, cost_vec))
    9395              :                 {
    9396            0 :                   STMT_VINFO_LIVE_P (stmt_info) = false;
    9397            0 :                   mark_visited = false;
    9398              :                 }
    9399              :             }
    9400              :         }
    9401      2366900 :       if (mark_visited)
    9402      2366900 :         svisited.add (stmt_info);
    9403              :     }
    9404              : 
    9405              :   slp_tree child;
    9406      2782251 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
    9407       995275 :     if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
    9408       289257 :       vect_bb_slp_mark_live_stmts (bb_vinfo, child, instance, cost_vec,
    9409              :                                    svisited, visited);
    9410              : }
    9411              : 
    9412              : /* Traverse all slp instances of BB_VINFO, and mark lanes of every node that
    9413              :    are live outside of the basic-block vectorized region and that can be
    9414              :    vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P.  */
    9415              : 
    9416              : static void
    9417       248546 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
    9418              : {
    9419       248546 :   if (bb_vinfo->slp_instances.is_empty ())
    9420            0 :     return;
    9421              : 
    9422       248546 :   hash_set<slp_tree> visited;
    9423       248546 :   hash_set<stmt_vec_info> svisited;
    9424      1468133 :   for (slp_instance instance : bb_vinfo->slp_instances)
    9425              :     {
    9426       722495 :       if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
    9427        55191 :         STMT_VINFO_LIVE_P (SLP_INSTANCE_ROOT_STMTS (instance)[0]) = true;
    9428       722495 :       vect_location = instance->location ();
    9429       722495 :       vect_bb_slp_mark_live_stmts (bb_vinfo, SLP_INSTANCE_TREE (instance),
    9430              :                                    instance, &instance->cost_vec,
    9431              :                                    svisited, visited);
    9432              :     }
    9433       248546 : }
    9434              : 
    9435              : /* Determine whether we can vectorize the reduction epilogue for INSTANCE.  */
    9436              : 
    9437              : static bool
    9438        93278 : vectorizable_bb_reduc_epilogue (slp_instance instance,
    9439              :                                 stmt_vector_for_cost *cost_vec)
    9440              : {
    9441        93278 :   gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
    9442        93278 :   enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
    9443        93278 :   if (reduc_code == MINUS_EXPR)
    9444            0 :     reduc_code = PLUS_EXPR;
    9445        93278 :   internal_fn reduc_fn;
    9446        93278 :   tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
    9447        93278 :   if (!vectype
    9448        93266 :       || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
    9449        93266 :                                      TREE_TYPE (vectype))
    9450        56166 :       || (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u)
    9451         1006 :           && (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
    9452         1006 :               || reduc_fn == IFN_LAST
    9453         1006 :               || !direct_internal_fn_supported_p (reduc_fn, vectype,
    9454              :                                                   OPTIMIZE_FOR_BOTH)))
    9455              :       /* Two-element reductions do not need special-handling for fold-left,
    9456              :          other cases are not yet implemented.  remain_defs also have to
    9457              :          be included here.  */
    9458       186592 :       || (needs_fold_left_reduction_p (TREE_TYPE (vectype), reduc_code)
    9459         6020 :           && (!instance->remain_defs.is_empty ()
    9460         1886 :               || SLP_TREE_LANES (SLP_INSTANCE_TREE (instance)) != 2)))
    9461              :     {
    9462        39229 :       if (dump_enabled_p ())
    9463           54 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9464              :                          "not vectorized: basic block reduction epilogue "
    9465              :                          "operation unsupported.\n");
    9466              :       return false;
    9467              :     }
    9468              : 
    9469              :   /* There's no way to cost a horizontal vector reduction via REDUC_FN so
    9470              :      cost log2 vector operations plus shuffles and one extraction.  */
    9471        54049 :   unsigned steps = floor_log2 (vect_nunits_for_cost (vectype));
    9472        54049 :   record_stmt_cost (cost_vec, steps, vector_stmt, instance->root_stmts[0],
    9473              :                     vectype, 0, vect_body);
    9474        54049 :   record_stmt_cost (cost_vec, steps, vec_perm, instance->root_stmts[0],
    9475              :                     vectype, 0, vect_body);
    9476        54049 :   record_stmt_cost (cost_vec, 1, vec_to_scalar, instance->root_stmts[0],
    9477              :                     vectype, 0, vect_body);
    9478              : 
    9479              :   /* Since we replace all stmts of a possibly longer scalar reduction
    9480              :      chain account for the extra scalar stmts for that.  */
    9481        54049 :   if (!instance->remain_defs.is_empty ())
    9482        36818 :     record_stmt_cost (cost_vec, instance->remain_defs.length (), scalar_stmt,
    9483        18409 :                       instance->root_stmts[0], 0, vect_body);
    9484              :   return true;
    9485              : }
    9486              : 
    9487              : /* Prune from ROOTS all stmts that are computed as part of lanes of NODE
    9488              :    and recurse to children.  */
    9489              : 
    9490              : static void
    9491       387989 : vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
    9492              :                               hash_set<slp_tree> &visited)
    9493              : {
    9494       387989 :   if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
    9495       387989 :       || visited.add (node))
    9496              :     return;
    9497              : 
    9498      1116833 :   for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
    9499       499694 :     if (stmt)
    9500       536530 :       roots.remove (vect_orig_stmt (stmt));
    9501              : 
    9502       824329 :   for (auto child : SLP_TREE_CHILDREN (node))
    9503       274000 :     if (child)
    9504       272394 :       vect_slp_prune_covered_roots (child, roots, visited);
    9505              : }
    9506              : 
    9507              : /* Hand over COST_VEC to the target COSTS grouped by SLP node.  */
    9508              : 
    9509              : static void
    9510      1041555 : add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
    9511              : {
    9512      3876960 :   for (unsigned start = 0; start < cost_vec.length ();)
    9513              :     {
    9514      2835405 :       unsigned end = start + 1;
    9515      3566457 :       while (end < cost_vec.length ()
    9516      6101101 :              && cost_vec[start].node == cost_vec[end].node)
    9517       731052 :         end++;
    9518      2835405 :       if (cost_vec[start].node)
    9519      2790380 :         costs->add_slp_cost (cost_vec[start].node,
    9520      2790380 :                              array_slice<stmt_info_for_cost>
    9521      2790380 :                                (cost_vec.begin () + start, end - start));
    9522              :       else
    9523        45025 :         costs->vector_costs::add_slp_cost (cost_vec[start].node,
    9524        45025 :                              array_slice<stmt_info_for_cost>
    9525        45025 :                                (cost_vec.begin () + start, end - start));
    9526      2835405 :       start = end;
    9527              :     }
    9528      1041555 : }
    9529              : 
    9530              : /* Analyze statements in SLP instances of VINFO.  Return true if the
    9531              :    operations are supported. */
    9532              : 
    9533              : bool
    9534       712375 : vect_slp_analyze_operations (vec_info *vinfo)
    9535              : {
    9536       712375 :   slp_instance instance;
    9537       712375 :   int i;
    9538              : 
    9539       712375 :   DUMP_VECT_SCOPE ("vect_slp_analyze_operations");
    9540              : 
    9541       712375 :   hash_set<slp_tree> visited;
    9542      2542558 :   for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
    9543              :     {
    9544      1389060 :       auto_vec<slp_tree> visited_vec;
    9545      1389060 :       stmt_vector_for_cost cost_vec;
    9546      1389060 :       cost_vec.create (2);
    9547      1389060 :       if (is_a <bb_vec_info> (vinfo))
    9548       825682 :         vect_location = instance->location ();
    9549      1389060 :       if (!vect_slp_analyze_node_operations (vinfo,
    9550              :                                              SLP_INSTANCE_TREE (instance),
    9551              :                                              instance, visited, visited_vec,
    9552              :                                              &cost_vec)
    9553              :           /* CTOR instances require vectorized defs for the SLP tree root.  */
    9554      1123883 :           || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor
    9555         6725 :               && (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance))
    9556              :                   != vect_internal_def
    9557              :                   /* Make sure we vectorized with the expected type.  */
    9558         6725 :                   || !useless_type_conversion_p
    9559         6725 :                         (TREE_TYPE (TREE_TYPE (gimple_assign_rhs1
    9560              :                                               (instance->root_stmts[0]->stmt))),
    9561         6725 :                          TREE_TYPE (SLP_TREE_VECTYPE
    9562              :                                             (SLP_INSTANCE_TREE (instance))))))
    9563              :           /* Check we can vectorize the reduction.  */
    9564      1123868 :           || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_bb_reduc
    9565        93278 :               && !vectorizable_bb_reduc_epilogue (instance, &cost_vec))
    9566              :           /* Check we can vectorize the gcond.  */
    9567      2473699 :           || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
    9568        67203 :               && !vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
    9569        67203 :                                            SLP_INSTANCE_ROOT_STMTS (instance)[0],
    9570              :                                            NULL,
    9571              :                                            SLP_INSTANCE_TREE (instance),
    9572              :                                            &cost_vec)))
    9573              :         {
    9574       368863 :           cost_vec.release ();
    9575       368863 :           slp_tree node = SLP_INSTANCE_TREE (instance);
    9576       368863 :           stmt_vec_info stmt_info;
    9577       368863 :           if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
    9578       271889 :             stmt_info = SLP_INSTANCE_ROOT_STMTS (instance)[0];
    9579        96974 :           else if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
    9580        96974 :                    && SLP_TREE_SCALAR_STMTS (node)[0])
    9581              :             stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
    9582              :           else
    9583            3 :             stmt_info = SLP_TREE_REPRESENTATIVE (node);
    9584       368863 :           if (is_a <loop_vec_info> (vinfo))
    9585              :             {
    9586       271252 :               if (dump_enabled_p ())
    9587         7654 :                 dump_printf_loc (MSG_NOTE, vect_location,
    9588              :                                  "unsupported SLP instance starting from: %G",
    9589              :                                  stmt_info->stmt);
    9590       271252 :               return false;
    9591              :             }
    9592        97611 :           if (dump_enabled_p ())
    9593          114 :             dump_printf_loc (MSG_NOTE, vect_location,
    9594              :                              "removing SLP instance operations starting from: %G",
    9595              :                              stmt_info->stmt);
    9596       234903 :           while (!visited_vec.is_empty ())
    9597              :             {
    9598       137292 :               slp_tree node = visited_vec.pop ();
    9599       137292 :               SLP_TREE_TYPE (node) = undef_vec_info_type;
    9600       137292 :               if (node->data)
    9601              :                 {
    9602         8525 :                   delete node->data;
    9603         8525 :                   node->data = nullptr;
    9604              :                 }
    9605       137292 :               visited.remove (node);
    9606              :             }
    9607        97611 :           vect_free_slp_instance (instance);
    9608        97611 :           vinfo->slp_instances.ordered_remove (i);
    9609              :         }
    9610              :       else
    9611              :         {
    9612      1020197 :           i++;
    9613      1020197 :           if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
    9614              :             {
    9615       292126 :               add_slp_costs (loop_vinfo->vector_costs, cost_vec);
    9616       292126 :               cost_vec.release ();
    9617              :             }
    9618              :           else
    9619              :             /* For BB vectorization remember the SLP graph entry
    9620              :                cost for later.  */
    9621       728071 :             instance->cost_vec = cost_vec;
    9622              :         }
    9623      1389060 :     }
    9624              : 
    9625              :   /* Now look for SLP instances with a root that are covered by other
    9626              :      instances and remove them.  */
    9627       441123 :   hash_set<stmt_vec_info> roots;
    9628      1830836 :   for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
    9629      1012098 :     if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
    9630        63508 :       roots.add (SLP_INSTANCE_ROOT_STMTS (instance)[0]);
    9631       441123 :   if (!roots.is_empty ())
    9632              :     {
    9633        26712 :       visited.empty ();
    9634       142307 :       for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
    9635       115595 :         vect_slp_prune_covered_roots (SLP_INSTANCE_TREE (instance), roots,
    9636              :                                       visited);
    9637       142307 :       for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
    9638       115595 :         if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ()
    9639        63508 :             && !roots.contains (SLP_INSTANCE_ROOT_STMTS (instance)[0]))
    9640              :           {
    9641         5538 :             stmt_vec_info root = SLP_INSTANCE_ROOT_STMTS (instance)[0];
    9642         5538 :             if (dump_enabled_p ())
    9643          287 :               dump_printf_loc (MSG_NOTE, vect_location,
    9644              :                                "removing SLP instance operations starting "
    9645              :                                "from: %G", root->stmt);
    9646         5538 :             vect_free_slp_instance (instance);
    9647         5538 :             vinfo->slp_instances.ordered_remove (i);
    9648              :           }
    9649              :         else
    9650       110057 :           ++i;
    9651              :     }
    9652              : 
    9653       882246 :   return !vinfo->slp_instances.is_empty ();
    9654      1153498 : }
    9655              : 
    9656              : /* Get the SLP instance leader from INSTANCE_LEADER thereby transitively
    9657              :    closing the eventual chain.  */
    9658              : 
    9659              : static slp_instance
    9660       814880 : get_ultimate_leader (slp_instance instance,
    9661              :                      hash_map<slp_instance, slp_instance> &instance_leader)
    9662              : {
    9663       814880 :   auto_vec<slp_instance *, 8> chain;
    9664       814880 :   slp_instance *tem;
    9665       927661 :   while (*(tem = instance_leader.get (instance)) != instance)
    9666              :     {
    9667       112781 :       chain.safe_push (tem);
    9668       112781 :       instance = *tem;
    9669              :     }
    9670       927661 :   while (!chain.is_empty ())
    9671       112781 :     *chain.pop () = instance;
    9672       814880 :   return instance;
    9673       814880 : }
    9674              : 
    9675              : namespace {
    9676              : /* Subroutine of vect_bb_partition_graph_r.  Map KEY to INSTANCE in
    9677              :    KEY_TO_INSTANCE, making INSTANCE the leader of any previous mapping
    9678              :    for KEY.  Return true if KEY was already in KEY_TO_INSTANCE.
    9679              : 
    9680              :    INSTANCE_LEADER is as for get_ultimate_leader.  */
    9681              : 
    9682              : template<typename T>
    9683              : bool
    9684      3614304 : vect_map_to_instance (slp_instance instance, T key,
    9685              :                       hash_map<T, slp_instance> &key_to_instance,
    9686              :                       hash_map<slp_instance, slp_instance> &instance_leader)
    9687              : {
    9688              :   bool existed_p;
    9689      3614304 :   slp_instance &key_instance = key_to_instance.get_or_insert (key, &existed_p);
    9690      3614304 :   if (!existed_p)
    9691              :     ;
    9692       247407 :   else if (key_instance != instance)
    9693              :     {
    9694              :       /* If we're running into a previously marked key make us the
    9695              :          leader of the current ultimate leader.  This keeps the
    9696              :          leader chain acyclic and works even when the current instance
    9697              :          connects two previously independent graph parts.  */
    9698        92385 :       slp_instance key_leader
    9699        92385 :         = get_ultimate_leader (key_instance, instance_leader);
    9700        92385 :       if (key_leader != instance)
    9701        27619 :         instance_leader.put (key_leader, instance);
    9702              :     }
    9703      3614304 :   key_instance = instance;
    9704      3614304 :   return existed_p;
    9705              : }
    9706              : }
    9707              : 
    9708              : /* Worker of vect_bb_partition_graph, recurse on NODE.  */
    9709              : 
    9710              : static void
    9711      1011752 : vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
    9712              :                            slp_instance instance, slp_tree node,
    9713              :                            hash_map<stmt_vec_info, slp_instance> &stmt_to_instance,
    9714              :                            hash_map<slp_tree, slp_instance> &node_to_instance,
    9715              :                            hash_map<slp_instance, slp_instance> &instance_leader)
    9716              : {
    9717      5626676 :   for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
    9718      2602552 :     if (stmt_info)
    9719      2602552 :       vect_map_to_instance (instance, stmt_info, stmt_to_instance,
    9720              :                             instance_leader);
    9721              : 
    9722      1011752 :   if (vect_map_to_instance (instance, node, node_to_instance,
    9723              :                             instance_leader))
    9724              :     return;
    9725              : 
    9726      3616376 :   for (auto child : SLP_TREE_CHILDREN (node))
    9727       995275 :     if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
    9728       289257 :       vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
    9729              :                                  node_to_instance, instance_leader);
    9730              : }
    9731              : 
    9732              : /* Partition the SLP graph into pieces that can be costed independently.  */
    9733              : 
    9734              : static void
    9735       248546 : vect_bb_partition_graph (bb_vec_info bb_vinfo)
    9736              : {
    9737       248546 :   DUMP_VECT_SCOPE ("vect_bb_partition_graph");
    9738              : 
    9739              :   /* First walk the SLP graph assigning each involved scalar stmt a
    9740              :      corresponding SLP graph entry and upon visiting a previously
    9741              :      marked stmt, make the stmts leader the current SLP graph entry.  */
    9742       248546 :   hash_map<stmt_vec_info, slp_instance> stmt_to_instance;
    9743       248546 :   hash_map<slp_tree, slp_instance> node_to_instance;
    9744       248546 :   hash_map<slp_instance, slp_instance> instance_leader;
    9745       248546 :   slp_instance instance;
    9746      1219587 :   for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
    9747              :     {
    9748       722495 :       instance_leader.put (instance, instance);
    9749       722495 :       vect_bb_partition_graph_r (bb_vinfo,
    9750              :                                  instance, SLP_INSTANCE_TREE (instance),
    9751              :                                  stmt_to_instance, node_to_instance,
    9752              :                                  instance_leader);
    9753              :     }
    9754              : 
    9755              :   /* Then collect entries to each independent subgraph.  */
    9756      1219587 :   for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
    9757              :     {
    9758       722495 :       slp_instance leader = get_ultimate_leader (instance, instance_leader);
    9759       722495 :       leader->subgraph_entries.safe_push (instance);
    9760       722495 :       if (dump_enabled_p ()
    9761       722495 :           && leader != instance)
    9762          106 :         dump_printf_loc (MSG_NOTE, vect_location,
    9763              :                          "instance %p is leader of %p\n",
    9764              :                          (void *) leader, (void *) instance);
    9765              :     }
    9766       248546 : }
    9767              : 
    9768              : /* Compute the scalar cost of the SLP node NODE and its children
    9769              :    and return it.  Do not account defs that are marked in LIFE and
    9770              :    update LIFE according to uses of NODE.  */
    9771              : 
    9772              : static void
    9773       718201 : vect_bb_slp_scalar_cost (bb_vec_info vinfo,
    9774              :                          vec<stmt_vec_info> &worklist,
    9775              :                          stmt_vector_for_cost *cost_vec,
    9776              :                          hash_set<stmt_vec_info> &visited)
    9777              : {
    9778      3359350 :   while (!worklist.is_empty ())
    9779              :     {
    9780      2641149 :       stmt_vec_info stmt = worklist.pop ();
    9781      2967914 :       if (!PURE_SLP_STMT (stmt))
    9782       335405 :         continue;
    9783              : 
    9784              :       /* When the stmt is live but not actually vectorized we have
    9785              :          to keep the feeding scalar defs.  */
    9786      2324732 :       if (!STMT_VINFO_LIVE_P (vect_stmt_to_vectorize (stmt)))
    9787              :         {
    9788      2207723 :           bool live_p = false;
    9789      2207723 :           ssa_op_iter op_iter;
    9790      2207723 :           def_operand_p def_p;
    9791      4869441 :           FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt->stmt, op_iter, SSA_OP_DEF)
    9792              :             {
    9793       453995 :               imm_use_iterator use_iter;
    9794       453995 :               gimple *use_stmt;
    9795      1157711 :               FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
    9796       703716 :                 if (!is_gimple_debug (use_stmt))
    9797              :                   {
    9798       532747 :                     stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
    9799       532747 :                     if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
    9800              :                       {
    9801        15859 :                         if (dump_enabled_p ())
    9802              :                           {
    9803           92 :                             dump_printf_loc (MSG_NOTE, vect_location,
    9804              :                                              "stmt considered live: %G",
    9805              :                                              stmt->stmt);
    9806           92 :                             dump_printf_loc (MSG_NOTE, vect_location,
    9807              :                                              "because of use in: %G",
    9808              :                                              use_stmt);
    9809              :                           }
    9810              :                         live_p = true;
    9811              :                       }
    9812       453995 :                   }
    9813              :             }
    9814      2207723 :           if (live_p)
    9815         8640 :             continue;
    9816              :         }
    9817              : 
    9818              :       /* The following assert verifies that vect_bb_partition_graph
    9819              :          partitions the SLP graph in a way that each scalar stmt of
    9820              :          the coverage of the SLP graph belongs to exactly one subgraph.
    9821              :          ???  This is currently not guaranteed since the function
    9822              :          works purely on SLP_TREE_SCALAR_STMTS, resulting in the assert
    9823              :          tripping or scalar stmts costed multiple times, making vectorization
    9824              :          more profitable than it really is.  */
    9825              :       /* gcc_checking_assert (!gimple_visited_p (stmt->stmt)); */
    9826              : 
    9827      2305744 :       if (vect_nop_conversion_p (stmt))
    9828              :         ;
    9829              :       /* For single-argument PHIs assume coalescing which means zero
    9830              :          cost for the scalar and the vector PHIs.  This avoids
    9831              :          artificially favoring the vector path (but may pessimize it
    9832              :          in some cases).  */
    9833      2274260 :       else if (is_a <gphi *> (stmt->stmt)
    9834      2274260 :                && gimple_phi_num_args (as_a <gphi *> (stmt->stmt)) == 1)
    9835              :         ;
    9836              :       else
    9837              :         {
    9838      2263762 :           vect_cost_for_stmt kind;
    9839      2263762 :           if (STMT_VINFO_DATA_REF (stmt))
    9840              :             {
    9841      2026686 :               data_reference_p dr = STMT_VINFO_DATA_REF (stmt);
    9842      2026686 :               tree base = get_base_address (DR_REF (dr));
    9843              :               /* When the scalar access is to a non-global not
    9844              :                  address-taken decl that is not BLKmode assume we can
    9845              :                  access it with a single non-load/store instruction.  */
    9846      2026686 :               if (DECL_P (base)
    9847      1550348 :                   && !is_global_var (base)
    9848      1471897 :                   && !TREE_ADDRESSABLE (base)
    9849      2586645 :                   && DECL_MODE (base) != BLKmode)
    9850              :                 kind = scalar_stmt;
    9851      1881680 :               else if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt)))
    9852              :                 kind = scalar_load;
    9853              :               else
    9854      1625706 :                 kind = scalar_store;
    9855              :             }
    9856              :           else
    9857              :             kind = scalar_stmt;
    9858              :           /* Cost each scalar stmt only once.  */
    9859      2263762 :           gimple_set_visited (stmt->stmt, true);
    9860      2263762 :           record_stmt_cost (cost_vec, 1, kind, stmt, NULL_TREE, 0, vect_body);
    9861              :         }
    9862              : 
    9863              :       /* Now walk relevant parts of the SSA use-def graph.  */
    9864      2305744 :       slp_oprnds child_ops (stmt);
    9865      7185637 :       for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
    9866              :         {
    9867      2574149 :           tree op = child_ops.get_op_for_slp_child (stmt, i);
    9868      2574149 :           stmt_vec_info def = vinfo->lookup_def (op);
    9869      2574149 :           if (def && !visited.add (def))
    9870       779781 :             worklist.safe_push (def);
    9871              :         }
    9872              :     }
    9873       718201 : }
    9874              : 
    9875              : 
    9876              : /* Comparator for the loop-index sorted cost vectors.  */
    9877              : 
    9878              : static int
    9879     20718798 : li_cost_vec_cmp (const void *a_, const void *b_, void *)
    9880              : {
    9881     20718798 :   auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
    9882     20718798 :   auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
    9883     20718798 :   if (a->first < b->first)
    9884              :     return -1;
    9885     19246867 :   else if (a->first == b->first)
    9886     18072385 :     return 0;
    9887              :   return 1;
    9888              : }
    9889              : 
    9890              : /* Check if vectorization of the basic block is profitable for the
    9891              :    subgraph denoted by SLP_INSTANCES.  */
    9892              : 
    9893              : static bool
    9894       690790 : vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
    9895              :                                     vec<slp_instance> slp_instances,
    9896              :                                     loop_p orig_loop)
    9897              : {
    9898       690790 :   slp_instance instance;
    9899       690790 :   int i;
    9900       690790 :   unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
    9901       690790 :   unsigned int vec_prologue_cost = 0, vec_epilogue_cost = 0;
    9902              : 
    9903       690790 :   if (dump_enabled_p ())
    9904              :     {
    9905          125 :       dump_printf_loc (MSG_NOTE, vect_location, "Costing subgraph:\n");
    9906          260 :       FOR_EACH_VEC_ELT (slp_instances, i, instance)
    9907          135 :         dump_printf_loc (MSG_NOTE, vect_location, "   entry instance %p -> "
    9908              :                          "node %p\n", (void *)instance,
    9909          135 :                          (void *)SLP_INSTANCE_TREE (instance));
    9910          125 :       hash_set<slp_tree> visited;
    9911          510 :       FOR_EACH_VEC_ELT (slp_instances, i, instance)
    9912          135 :         vect_print_slp_graph (MSG_NOTE, vect_location,
    9913              :                               SLP_INSTANCE_TREE (instance), visited);
    9914          125 :     }
    9915              : 
    9916              :   /* Then DFS walk scalar stmts, performing costing and handling
    9917              :      still live scalar stmts via the previously computed vector coverage.  */
    9918       690790 :   stmt_vector_for_cost scalar_costs = vNULL;
    9919       690790 :   stmt_vector_for_cost vector_costs = vNULL;
    9920       690790 :   hash_set<slp_tree> visited;
    9921       690790 :   hash_set<stmt_vec_info> svisited;
    9922      2099781 :   FOR_EACH_VEC_ELT (slp_instances, i, instance)
    9923              :     {
    9924       718201 :       auto_vec<stmt_vec_info> worklist;
    9925       718201 :       if (SLP_INSTANCE_ROOT_STMTS (instance).exists ())
    9926       108700 :         record_stmt_cost (&scalar_costs,
    9927        54350 :                           SLP_INSTANCE_ROOT_STMTS (instance).length (),
    9928              :                           scalar_stmt,
    9929        54350 :                           SLP_INSTANCE_ROOT_STMTS (instance)[0], 0, vect_body);
    9930      4031489 :       for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
    9931              :         {
    9932      1876886 :           stmt = vect_orig_stmt (stmt);
    9933      1876886 :           if (!svisited.add (stmt))
    9934      1861368 :             worklist.safe_push (stmt);
    9935              :         }
    9936       718201 :       vect_bb_slp_scalar_cost (bb_vinfo, worklist, &scalar_costs, svisited);
    9937       718201 :       vector_costs.safe_splice (instance->cost_vec);
    9938       718201 :       instance->cost_vec.release ();
    9939       718201 :     }
    9940              : 
    9941       690790 :   if (dump_enabled_p ())
    9942          125 :     dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
    9943              : 
    9944              :   /* When costing non-loop vectorization we need to consider each covered
    9945              :      loop independently and make sure vectorization is profitable.  For
    9946              :      now we assume a loop may be not entered or executed an arbitrary
    9947              :      number of iterations (???  static information can provide more
    9948              :      precise info here) which means we can simply cost each containing
    9949              :      loops stmts separately.  */
    9950              : 
    9951              :   /* First produce cost vectors sorted by loop index.  */
    9952       690790 :   auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
    9953      1381580 :     li_scalar_costs (scalar_costs.length ());
    9954       690790 :   auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
    9955      1381567 :     li_vector_costs (vector_costs.length ());
    9956       690790 :   stmt_info_for_cost *cost;
    9957      3699692 :   FOR_EACH_VEC_ELT (scalar_costs, i, cost)
    9958              :     {
    9959      2318112 :       unsigned l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
    9960      2318112 :       li_scalar_costs.quick_push (std::make_pair (l, cost));
    9961              :     }
    9962              :   /* Use a random used loop as fallback in case the first vector_costs
    9963              :      entry does not have a location associated with it.  */
    9964       690790 :   unsigned l = li_scalar_costs[0].first;
    9965      2675746 :   FOR_EACH_VEC_ELT (vector_costs, i, cost)
    9966              :     {
    9967              :       /* Use SLP node placement according to the computed schedule.  */
    9968      1984956 :       if (cost->node && cost->node->si)
    9969      1237331 :         l = gimple_bb (cost->node->si)->loop_father->num;
    9970              :       /* For schedules at region boundary use the region entry loop.  */
    9971       747625 :       else if (cost->node)
    9972       588234 :         l = bb_vinfo->bbs[0]->loop_father->num;
    9973              :       /* SLP instance root stmts do not have an associated SLP node.  */
    9974       159391 :       else if (cost->stmt_info)
    9975       159391 :         l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
    9976              :       /* And since vect_prologue_cost_for_slp can end up costing with
    9977              :          neither, inherit from the previous node.  */
    9978      1984956 :       li_vector_costs.quick_push (std::make_pair (l, cost));
    9979              :     }
    9980       690790 :   li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
    9981       690790 :   li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
    9982              : 
    9983       690790 :   unsigned total_vec_outside_cost = 0;
    9984       690790 :   unsigned total_vec_inside_cost = 0;
    9985       690790 :   unsigned total_scalar_cost = 0;
    9986              : 
    9987              :   /* Now cost the portions individually.  */
    9988       690790 :   unsigned vi = 0;
    9989       690790 :   unsigned si = 0;
    9990       690790 :   bool profitable = true;
    9991      1382908 :   while (si < li_scalar_costs.length ()
    9992      2134181 :          || vi < li_vector_costs.length ())
    9993              :     {
    9994       751286 :       unsigned sl
    9995      1501244 :         = si < li_scalar_costs.length () ? li_scalar_costs[si].first : -1U;
    9996       751286 :       unsigned vl
    9997      1502488 :         = vi < li_vector_costs.length () ? li_vector_costs[vi].first : -1U;
    9998              : 
    9999       751286 :       class vector_costs *scalar_target_cost_data = nullptr;
   10000       751286 :       scalar_cost = 0;
   10001       751286 :       if (sl <= vl)
   10002              :         {
   10003       706183 :           if (dump_enabled_p ())
   10004          147 :             dump_printf_loc (MSG_NOTE, vect_location,
   10005              :                              "Scalar cost for part in loop %d\n", sl);
   10006       706183 :           scalar_target_cost_data = init_cost (bb_vinfo, true);
   10007      2318112 :           do
   10008              :             {
   10009      2318112 :               add_stmt_cost (scalar_target_cost_data,
   10010      2318112 :                              li_scalar_costs[si].second);
   10011      2318112 :               si++;
   10012              :             }
   10013      2318112 :           while (si < li_scalar_costs.length ()
   10014      4651617 :                  && li_scalar_costs[si].first == sl);
   10015       706183 :           scalar_target_cost_data->finish_cost (nullptr);
   10016       706183 :           scalar_cost = scalar_target_cost_data->body_cost ();
   10017       706183 :           total_scalar_cost += scalar_cost;
   10018       706183 :           if (sl < vl)
   10019              :             {
   10020         1857 :               if (dump_enabled_p ())
   10021            2 :                 dump_printf_loc (MSG_NOTE, vect_location,
   10022              :                                  "Scalar %d loop part does not "
   10023              :                                  "have corresponding vector part\n", sl);
   10024         1857 :               delete scalar_target_cost_data;
   10025       650714 :               continue;
   10026              :             }
   10027              :         }
   10028              : 
   10029              :       /* Complete the target-specific vector cost calculation.  */
   10030       749429 :       if (dump_enabled_p ())
   10031          149 :         dump_printf_loc (MSG_NOTE, vect_location,
   10032              :                          "Vector cost for part in loop %d\n", vl);
   10033       749429 :       class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
   10034       749429 :       auto_vec<stmt_info_for_cost> tem;
   10035      1984956 :       do
   10036              :         {
   10037      1984956 :           tem.safe_push (*li_vector_costs[vi].second);
   10038      1984956 :           vi++;
   10039              :         }
   10040      1984956 :       while (vi < li_vector_costs.length ()
   10041      4028564 :              && li_vector_costs[vi].first == vl);
   10042       749429 :       add_slp_costs (vect_target_cost_data, tem);
   10043       749429 :       vect_target_cost_data->finish_cost (scalar_target_cost_data);
   10044       749429 :       vec_prologue_cost = vect_target_cost_data->prologue_cost ();
   10045       749429 :       vec_inside_cost = vect_target_cost_data->body_cost ();
   10046       749429 :       vec_epilogue_cost = vect_target_cost_data->epilogue_cost ();
   10047       749429 :       if (scalar_target_cost_data)
   10048       704326 :         delete scalar_target_cost_data;
   10049       749429 :       delete vect_target_cost_data;
   10050              : 
   10051       749429 :       vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
   10052              : 
   10053       749429 :       total_vec_outside_cost += vec_outside_cost;
   10054       749429 :       total_vec_inside_cost += vec_inside_cost;
   10055              : 
   10056       749429 :       if (sl > vl && dump_enabled_p ())
   10057            4 :         dump_printf_loc (MSG_NOTE, vect_location,
   10058              :                          "Vector %d loop part does not "
   10059              :                          "have corresponding scalar part\n", vl);
   10060              : 
   10061              :       /* When this is vector costs for the region entry delay costing
   10062              :          and instead only require the total costs to be profitable.  */
   10063       749429 :       if (vl == (unsigned) bb_vinfo->bbs[0]->loop_father->num)
   10064       648857 :         continue;
   10065              : 
   10066       100572 :       if (dump_enabled_p ())
   10067              :         {
   10068           52 :           dump_printf_loc (MSG_NOTE, vect_location,
   10069              :                            "Cost model analysis for part in loop %d:\n",
   10070           52 :                            std::min (sl, vl));
   10071           52 :           dump_printf (MSG_NOTE, "  Vector cost: %d\n",
   10072              :                        vec_inside_cost + vec_outside_cost);
   10073           52 :           dump_printf (MSG_NOTE, "  Scalar cost: %d\n", scalar_cost);
   10074              :         }
   10075              : 
   10076              :       /* Vectorization is profitable if its cost is more than the cost of scalar
   10077              :          version.  Note that we err on the vector side for equal cost because
   10078              :          the cost estimate is otherwise quite pessimistic (constant uses are
   10079              :          free on the scalar side but cost a load on the vector side for
   10080              :          example).  */
   10081       100572 :       if (vec_outside_cost + vec_inside_cost > scalar_cost)
   10082        47225 :         profitable = false;
   10083       749429 :     }
   10084              : 
   10085       690790 :   if (dump_enabled_p ())
   10086              :     {
   10087          125 :       dump_printf_loc (MSG_NOTE, vect_location,
   10088              :                        "Cost model analysis for whole subgraph:\n");
   10089          125 :       dump_printf (MSG_NOTE, "  Vector cost: %d\n",
   10090              :                    total_vec_inside_cost + total_vec_outside_cost);
   10091          125 :       dump_printf (MSG_NOTE, "  Scalar cost: %d\n", total_scalar_cost);
   10092              :     }
   10093              : 
   10094              :   /* For the case where the outermost region had no scalar cost require
   10095              :      overall profitability.  */
   10096       690790 :   if (total_vec_outside_cost + total_vec_inside_cost > total_scalar_cost)
   10097       212975 :     profitable = false;
   10098              : 
   10099              :   /* Unset visited flag.  This is delayed when the subgraph is profitable
   10100              :      and we process the loop for remaining unvectorized if-converted code.  */
   10101       690790 :   if (!orig_loop || !profitable)
   10102      3006850 :     FOR_EACH_VEC_ELT (scalar_costs, i, cost)
   10103      2316185 :       gimple_set_visited  (cost->stmt_info->stmt, false);
   10104              : 
   10105       690790 :   scalar_costs.release ();
   10106       690790 :   vector_costs.release ();
   10107              : 
   10108       690790 :   return profitable;
   10109       690790 : }
   10110              : 
   10111              : /* qsort comparator for lane defs.  */
   10112              : 
   10113              : static int
   10114          128 : vld_cmp (const void *a_, const void *b_)
   10115              : {
   10116          128 :   auto *a = (const std::pair<unsigned, tree> *)a_;
   10117          128 :   auto *b = (const std::pair<unsigned, tree> *)b_;
   10118          128 :   return a->first - b->first;
   10119              : }
   10120              : 
   10121              : /* Return true if USE_STMT is a vector lane insert into VEC and set
   10122              :    *THIS_LANE to the lane number that is set.  */
   10123              : 
   10124              : static bool
   10125          295 : vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
   10126              : {
   10127          295 :   gassign *use_ass = dyn_cast <gassign *> (use_stmt);
   10128          110 :   if (!use_ass
   10129          110 :       || gimple_assign_rhs_code (use_ass) != BIT_INSERT_EXPR
   10130           35 :       || (vec
   10131           35 :           ? gimple_assign_rhs1 (use_ass) != vec
   10132           26 :           : ((vec = gimple_assign_rhs1 (use_ass)), false))
   10133           61 :       || !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (vec)),
   10134           61 :                                      TREE_TYPE (gimple_assign_rhs2 (use_ass)))
   10135           61 :       || !constant_multiple_p
   10136           61 :             (tree_to_poly_uint64 (gimple_assign_rhs3 (use_ass)),
   10137          356 :              tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (TREE_TYPE (vec)))),
   10138              :              this_lane))
   10139              :     return false;
   10140              :   return true;
   10141              : }
   10142              : 
   10143              : /* Find any vectorizable constructors and add them to the grouped_store
   10144              :    array.  */
   10145              : 
   10146              : static void
   10147      2312925 : vect_slp_check_for_roots (bb_vec_info bb_vinfo)
   10148              : {
   10149     18469388 :   for (unsigned i = 0; i < bb_vinfo->nbbs; ++i)
   10150     32312926 :     for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[i]);
   10151    147734531 :          !gsi_end_p (gsi); gsi_next (&gsi))
   10152              :     {
   10153    131578068 :       gassign *assign = dyn_cast<gassign *> (gsi_stmt (gsi));
   10154              :       /* This can be used to start SLP discovery for early breaks for BB early breaks
   10155              :          when we get that far.  */
   10156    131578068 :       if (!assign)
   10157    100348687 :         continue;
   10158              : 
   10159     33360505 :       tree rhs = gimple_assign_rhs1 (assign);
   10160     33360505 :       enum tree_code code = gimple_assign_rhs_code (assign);
   10161     33360505 :       use_operand_p use_p;
   10162     33360505 :       gimple *use_stmt;
   10163     33360505 :       if (code == CONSTRUCTOR)
   10164              :         {
   10165      1743699 :           if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
   10166      1791284 :               || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
   10167       104692 :                            CONSTRUCTOR_NELTS (rhs))
   10168        47585 :               || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
   10169      1791264 :               || uniform_vector_p (rhs))
   10170      1728137 :             continue;
   10171              : 
   10172              :           unsigned j;
   10173              :           tree val;
   10174        77229 :           FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
   10175        61667 :             if (TREE_CODE (val) != SSA_NAME
   10176        61667 :                 || !bb_vinfo->lookup_def (val))
   10177              :               break;
   10178        38248 :           if (j != CONSTRUCTOR_NELTS (rhs))
   10179         3562 :             continue;
   10180              : 
   10181        15562 :           vec<stmt_vec_info> roots = vNULL;
   10182        15562 :           roots.safe_push (bb_vinfo->lookup_stmt (assign));
   10183        15562 :           vec<stmt_vec_info> stmts;
   10184        15562 :           stmts.create (CONSTRUCTOR_NELTS (rhs));
   10185        87190 :           FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
   10186        56066 :             stmts.quick_push
   10187        56066 :               (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
   10188        15562 :           bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
   10189        15562 :                                                stmts, roots));
   10190              :         }
   10191     31616806 :       else if (code == BIT_INSERT_EXPR
   10192         1022 :                && VECTOR_TYPE_P (TREE_TYPE (rhs))
   10193          721 :                && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
   10194          721 :                && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).to_constant () > 1
   10195          718 :                && integer_zerop (gimple_assign_rhs3 (assign))
   10196          392 :                && useless_type_conversion_p
   10197          392 :                     (TREE_TYPE (TREE_TYPE (rhs)),
   10198          392 :                      TREE_TYPE (gimple_assign_rhs2 (assign)))
   10199     31617530 :                && bb_vinfo->lookup_def (gimple_assign_rhs2 (assign)))
   10200              :         {
   10201              :           /* We start to match on insert to lane zero but since the
   10202              :              inserts need not be ordered we'd have to search both
   10203              :              the def and the use chains.  */
   10204          257 :           tree vectype = TREE_TYPE (rhs);
   10205          257 :           unsigned nlanes = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
   10206          257 :           auto_vec<std::pair<unsigned, tree> > lane_defs (nlanes);
   10207          257 :           auto_sbitmap lanes (nlanes);
   10208          257 :           bitmap_clear (lanes);
   10209          257 :           bitmap_set_bit (lanes, 0);
   10210          257 :           tree def = gimple_assign_lhs (assign);
   10211          257 :           lane_defs.quick_push
   10212          257 :                       (std::make_pair (0, gimple_assign_rhs2 (assign)));
   10213          257 :           unsigned lanes_found = 1;
   10214              :           /* Start with the use chains, the last stmt will be the root.  */
   10215          257 :           stmt_vec_info last = bb_vinfo->lookup_stmt (assign);
   10216          257 :           vec<stmt_vec_info> roots = vNULL;
   10217          257 :           roots.safe_push (last);
   10218          268 :           do
   10219              :             {
   10220          268 :               use_operand_p use_p;
   10221          268 :               gimple *use_stmt;
   10222          268 :               if (!single_imm_use (def, &use_p, &use_stmt))
   10223              :                 break;
   10224          256 :               unsigned this_lane;
   10225          256 :               if (!bb_vinfo->lookup_stmt (use_stmt)
   10226          256 :                   || !vect_slp_is_lane_insert (use_stmt, def, &this_lane)
   10227          291 :                   || !bb_vinfo->lookup_def (gimple_assign_rhs2 (use_stmt)))
   10228              :                 break;
   10229           35 :               if (bitmap_bit_p (lanes, this_lane))
   10230              :                 break;
   10231           15 :               lanes_found++;
   10232           15 :               bitmap_set_bit (lanes, this_lane);
   10233           15 :               gassign *use_ass = as_a <gassign *> (use_stmt);
   10234           15 :               lane_defs.quick_push (std::make_pair
   10235           15 :                                      (this_lane, gimple_assign_rhs2 (use_ass)));
   10236           15 :               last = bb_vinfo->lookup_stmt (use_ass);
   10237           15 :               roots.safe_push (last);
   10238           15 :               def = gimple_assign_lhs (use_ass);
   10239              :             }
   10240           15 :           while (lanes_found < nlanes);
   10241          257 :           if (roots.length () > 1)
   10242            7 :             std::swap(roots[0], roots[roots.length () - 1]);
   10243          257 :           if (lanes_found < nlanes)
   10244              :             {
   10245              :               /* Now search the def chain.  */
   10246          253 :               def = gimple_assign_rhs1 (assign);
   10247          255 :               do
   10248              :                 {
   10249          255 :                   if (TREE_CODE (def) != SSA_NAME
   10250          255 :                       || !has_single_use (def))
   10251              :                     break;
   10252           59 :                   gimple *def_stmt = SSA_NAME_DEF_STMT (def);
   10253           59 :                   unsigned this_lane;
   10254           59 :                   if (!bb_vinfo->lookup_stmt (def_stmt)
   10255           39 :                       || !vect_slp_is_lane_insert (def_stmt,
   10256              :                                                    NULL_TREE, &this_lane)
   10257           85 :                       || !bb_vinfo->lookup_def (gimple_assign_rhs2 (def_stmt)))
   10258              :                     break;
   10259           26 :                   if (bitmap_bit_p (lanes, this_lane))
   10260              :                     break;
   10261            6 :                   lanes_found++;
   10262            6 :                   bitmap_set_bit (lanes, this_lane);
   10263           12 :                   lane_defs.quick_push (std::make_pair
   10264            6 :                                           (this_lane,
   10265            6 :                                            gimple_assign_rhs2 (def_stmt)));
   10266            6 :                   roots.safe_push (bb_vinfo->lookup_stmt (def_stmt));
   10267            6 :                   def = gimple_assign_rhs1 (def_stmt);
   10268              :                 }
   10269            6 :               while (lanes_found < nlanes);
   10270              :             }
   10271          257 :           if (lanes_found == nlanes)
   10272              :             {
   10273              :               /* Sort lane_defs after the lane index and register the root.  */
   10274            8 :               lane_defs.qsort (vld_cmp);
   10275            8 :               vec<stmt_vec_info> stmts;
   10276            8 :               stmts.create (nlanes);
   10277           44 :               for (unsigned i = 0; i < nlanes; ++i)
   10278           28 :                 stmts.quick_push (bb_vinfo->lookup_def (lane_defs[i].second));
   10279            8 :               bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
   10280            8 :                                                    stmts, roots));
   10281              :             }
   10282              :           else
   10283          249 :             roots.release ();
   10284          257 :         }
   10285     31616549 :       else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
   10286     30542174 :                && (associative_tree_code (code) || code == MINUS_EXPR)
   10287              :                /* Ops with constants at the tail can be stripped here.  */
   10288      6744313 :                && TREE_CODE (rhs) == SSA_NAME
   10289      6680135 :                && TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
   10290              :                /* Should be the chain end.  */
   10291     34522904 :                && (!single_imm_use (gimple_assign_lhs (assign),
   10292              :                                     &use_p, &use_stmt)
   10293      2340461 :                    || !is_gimple_assign (use_stmt)
   10294      1736887 :                    || (gimple_assign_rhs_code (use_stmt) != code
   10295      1165476 :                        && ((code != PLUS_EXPR && code != MINUS_EXPR)
   10296       672777 :                            || (gimple_assign_rhs_code (use_stmt)
   10297       672777 :                                != (code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR))))))
   10298              :         {
   10299              :           /* We start the match at the end of a possible association
   10300              :              chain.  */
   10301      2215827 :           auto_vec<chain_op_t> chain;
   10302      2215827 :           auto_vec<std::pair<tree_code, gimple *> > worklist;
   10303      2215827 :           auto_vec<gimple *> chain_stmts;
   10304      2215827 :           gimple *code_stmt = NULL, *alt_code_stmt = NULL;
   10305      2215827 :           if (code == MINUS_EXPR)
   10306       362105 :             code = PLUS_EXPR;
   10307      2215827 :           internal_fn reduc_fn;
   10308      2618814 :           if (!reduction_fn_for_scalar_code (code, &reduc_fn)
   10309      2215827 :               || reduc_fn == IFN_LAST)
   10310       402987 :             continue;
   10311      1812840 :           vect_slp_linearize_chain (bb_vinfo, worklist, chain, code, assign,
   10312              :                                     /* ??? */
   10313              :                                     code_stmt, alt_code_stmt, &chain_stmts,
   10314              :                                     false);
   10315      3625680 :           if (chain.length () > 1)
   10316              :             {
   10317              :               /* Sort the chain according to def_type and operation.  */
   10318      1812840 :               chain.sort (dt_sort_cmp, bb_vinfo);
   10319              :               /* ???  Now we'd want to strip externals and constants
   10320              :                  but record those to be handled in the epilogue.  */
   10321              :               /* ???  For now do not allow mixing ops or externs/constants.  */
   10322      1812840 :               bool invalid = false;
   10323      1812840 :               unsigned remain_cnt = 0;
   10324      7483200 :               for (unsigned i = 0; i < chain.length (); ++i)
   10325              :                 {
   10326      4219625 :                   if (chain[i].code != code)
   10327              :                     {
   10328              :                       invalid = true;
   10329              :                       break;
   10330              :                     }
   10331      3857520 :                   if (chain[i].dt != vect_internal_def
   10332              :                       /* Avoid stmts where the def is not the LHS, like
   10333              :                          ASMs.  */
   10334      7342785 :                       || (gimple_get_lhs (bb_vinfo->lookup_def
   10335      3485265 :                                                       (chain[i].op)->stmt)
   10336      3485265 :                           != chain[i].op))
   10337       375357 :                     remain_cnt++;
   10338              :                 }
   10339      1812840 :               if (!invalid && chain.length () - remain_cnt > 1)
   10340              :                 {
   10341      1366013 :                   vec<stmt_vec_info> stmts;
   10342      1366013 :                   vec<tree> remain = vNULL;
   10343      1366013 :                   stmts.create (chain.length ());
   10344      1366013 :                   if (remain_cnt > 0)
   10345        52503 :                     remain.create (remain_cnt);
   10346      4553027 :                   for (unsigned i = 0; i < chain.length (); ++i)
   10347              :                     {
   10348      3187014 :                       stmt_vec_info stmt_info;
   10349      3187014 :                       if (chain[i].dt == vect_internal_def
   10350      3187014 :                           && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)),
   10351      3080150 :                               gimple_get_lhs (stmt_info->stmt) == chain[i].op))
   10352      3079928 :                         stmts.quick_push (stmt_info);
   10353              :                       else
   10354       107086 :                         remain.quick_push (chain[i].op);
   10355              :                     }
   10356      1366013 :                   vec<stmt_vec_info> roots;
   10357      1366013 :                   roots.create (chain_stmts.length ());
   10358      4553027 :                   for (unsigned i = 0; i < chain_stmts.length (); ++i)
   10359      1821001 :                     roots.quick_push (bb_vinfo->lookup_stmt (chain_stmts[i]));
   10360      1366013 :                   bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_bb_reduc,
   10361      1366013 :                                                        stmts, roots, remain));
   10362              :                 }
   10363              :             }
   10364      2215827 :         }
   10365              :     }
   10366      2312925 : }
   10367              : 
   10368              : /* Walk the grouped store chains and replace entries with their
   10369              :    pattern variant if any.  */
   10370              : 
   10371              : static void
   10372       677367 : vect_fixup_store_groups_with_patterns (vec_info *vinfo)
   10373              : {
   10374       677367 :   stmt_vec_info first_element;
   10375       677367 :   unsigned i;
   10376              : 
   10377      1593019 :   FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
   10378              :     {
   10379              :       /* We also have CTORs in this array.  */
   10380       915652 :       if (!STMT_VINFO_GROUPED_ACCESS (first_element))
   10381            0 :         continue;
   10382       915652 :       if (STMT_VINFO_IN_PATTERN_P (first_element))
   10383              :         {
   10384          255 :           stmt_vec_info orig = first_element;
   10385          255 :           first_element = STMT_VINFO_RELATED_STMT (first_element);
   10386          255 :           DR_GROUP_FIRST_ELEMENT (first_element) = first_element;
   10387          255 :           DR_GROUP_SIZE (first_element) = DR_GROUP_SIZE (orig);
   10388          255 :           DR_GROUP_GAP (first_element) = DR_GROUP_GAP (orig);
   10389          255 :           DR_GROUP_NEXT_ELEMENT (first_element) = DR_GROUP_NEXT_ELEMENT (orig);
   10390          255 :           vinfo->grouped_stores[i] = first_element;
   10391              :         }
   10392       915652 :       stmt_vec_info prev = first_element;
   10393      2573829 :       while (DR_GROUP_NEXT_ELEMENT (prev))
   10394              :         {
   10395      1658177 :           stmt_vec_info elt = DR_GROUP_NEXT_ELEMENT (prev);
   10396      1658177 :           if (STMT_VINFO_IN_PATTERN_P (elt))
   10397              :             {
   10398          870 :               stmt_vec_info orig = elt;
   10399          870 :               elt = STMT_VINFO_RELATED_STMT (elt);
   10400          870 :               DR_GROUP_NEXT_ELEMENT (prev) = elt;
   10401          870 :               DR_GROUP_GAP (elt) = DR_GROUP_GAP (orig);
   10402          870 :               DR_GROUP_NEXT_ELEMENT (elt) = DR_GROUP_NEXT_ELEMENT (orig);
   10403              :             }
   10404      1658177 :           DR_GROUP_FIRST_ELEMENT (elt) = first_element;
   10405      1658177 :           prev = elt;
   10406              :         }
   10407              :     }
   10408       677367 : }
   10409              : 
   10410              : /* Check if the region described by BB_VINFO can be vectorized, returning
   10411              :    true if so.  When returning false, set FATAL to true if the same failure
   10412              :    would prevent vectorization at other vector sizes, false if it is still
   10413              :    worth trying other sizes.  N_STMTS is the number of statements in the
   10414              :    region.  */
   10415              : 
   10416              : static bool
   10417      2312925 : vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
   10418              :                        vec<int> *dataref_groups)
   10419              : {
   10420      2312925 :   DUMP_VECT_SCOPE ("vect_slp_analyze_bb");
   10421              : 
   10422      2312925 :   slp_instance instance;
   10423      2312925 :   int i;
   10424              : 
   10425              :   /* The first group of checks is independent of the vector size.  */
   10426      2312925 :   fatal = true;
   10427              : 
   10428              :   /* Analyze the data references.  */
   10429              : 
   10430      2312925 :   if (!vect_analyze_data_refs (bb_vinfo, NULL))
   10431              :     {
   10432            0 :       if (dump_enabled_p ())
   10433            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10434              :                          "not vectorized: unhandled data-ref in basic "
   10435              :                          "block.\n");
   10436              :       return false;
   10437              :     }
   10438              : 
   10439      2312925 :   if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups))
   10440              :     {
   10441            0 :      if (dump_enabled_p ())
   10442            0 :        dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10443              :                         "not vectorized: unhandled data access in "
   10444              :                         "basic block.\n");
   10445              :       return false;
   10446              :     }
   10447              : 
   10448      2312925 :   vect_slp_check_for_roots (bb_vinfo);
   10449              : 
   10450              :   /* If there are no grouped stores and no constructors in the region
   10451              :      there is no need to continue with pattern recog as vect_analyze_slp
   10452              :      will fail anyway.  */
   10453      2312925 :   if (bb_vinfo->grouped_stores.is_empty ()
   10454      1960437 :       && bb_vinfo->roots.is_empty ())
   10455              :     {
   10456      1635558 :       if (dump_enabled_p ())
   10457         1114 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10458              :                          "not vectorized: no grouped stores in "
   10459              :                          "basic block.\n");
   10460              :       return false;
   10461              :     }
   10462              : 
   10463              :   /* While the rest of the analysis below depends on it in some way.  */
   10464       677367 :   fatal = false;
   10465              : 
   10466       677367 :   vect_pattern_recog (bb_vinfo);
   10467              : 
   10468              :   /* Update store groups from pattern processing.  */
   10469       677367 :   vect_fixup_store_groups_with_patterns (bb_vinfo);
   10470              : 
   10471              :   /* Check the SLP opportunities in the basic block, analyze and build SLP
   10472              :      trees.  */
   10473       677367 :   if (!vect_analyze_slp (bb_vinfo, n_stmts, false))
   10474              :     {
   10475            0 :       if (dump_enabled_p ())
   10476              :         {
   10477            0 :           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10478              :                            "Failed to SLP the basic block.\n");
   10479            0 :           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10480              :                            "not vectorized: failed to find SLP opportunities "
   10481              :                            "in basic block.\n");
   10482              :         }
   10483              :       return false;
   10484              :     }
   10485              : 
   10486              :   /* Optimize permutations.  */
   10487       677367 :   vect_optimize_slp (bb_vinfo);
   10488              : 
   10489              :   /* Gather the loads reachable from the SLP graph entries.  */
   10490       677367 :   vect_gather_slp_loads (bb_vinfo);
   10491              : 
   10492       677367 :   vect_record_base_alignments (bb_vinfo);
   10493              : 
   10494              :   /* Analyze and verify the alignment of data references and the
   10495              :      dependence in the SLP instances.  */
   10496      2189215 :   for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
   10497              :     {
   10498       834481 :       vect_location = instance->location ();
   10499       834481 :       if (! vect_slp_analyze_instance_alignment (bb_vinfo, instance)
   10500       834481 :           || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
   10501              :         {
   10502         8799 :           slp_tree node = SLP_INSTANCE_TREE (instance);
   10503         8799 :           stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
   10504         8799 :           if (dump_enabled_p ())
   10505            4 :             dump_printf_loc (MSG_NOTE, vect_location,
   10506              :                              "removing SLP instance operations starting from: %G",
   10507              :                              stmt_info->stmt);
   10508         8799 :           vect_free_slp_instance (instance);
   10509         8799 :           BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
   10510         8799 :           continue;
   10511         8799 :         }
   10512              : 
   10513              :       /* Mark all the statements that we want to vectorize as relevant.  */
   10514       825682 :       vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
   10515              : 
   10516       825682 :       i++;
   10517              :     }
   10518      2593474 :   if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
   10519              :     return false;
   10520              : 
   10521       280549 :   if (!vect_slp_analyze_operations (bb_vinfo))
   10522              :     {
   10523        32003 :       if (dump_enabled_p ())
   10524           67 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10525              :                          "not vectorized: bad operation in basic block.\n");
   10526              :       return false;
   10527              :     }
   10528              : 
   10529              :   /* Compute vector stmt placement.  */
   10530       248546 :   if (!vect_schedule_slp (bb_vinfo, BB_VINFO_SLP_INSTANCES (bb_vinfo), true))
   10531              :     {
   10532            0 :       if (dump_enabled_p ())
   10533            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10534              :                          "not vectorized: cannot schedule SLP graph\n");
   10535              :       return false;
   10536              :     }
   10537              : 
   10538              :   /* Mark all the statements that we vectorize.  */
   10539       248546 :   vect_bb_slp_mark_stmts_vectorized (bb_vinfo);
   10540              : 
   10541              :   /* Compute vectorizable live stmts.  */
   10542       248546 :   vect_bb_slp_mark_live_stmts (bb_vinfo);
   10543              : 
   10544       248546 :   vect_bb_partition_graph (bb_vinfo);
   10545              : 
   10546       248546 :   return true;
   10547              : }
   10548              : 
   10549              : /* Subroutine of vect_slp_bb.  Try to vectorize the statements for all
   10550              :    basic blocks in BBS, returning true on success.
   10551              :    The region has N_STMTS statements and has the datarefs given by DATAREFS.  */
   10552              : 
   10553              : static bool
   10554      1946392 : vect_slp_region (vec<basic_block> bbs, vec<data_reference_p> datarefs,
   10555              :                  vec<int> *dataref_groups, unsigned int n_stmts,
   10556              :                  loop_p orig_loop)
   10557              : {
   10558      1946392 :   bb_vec_info bb_vinfo;
   10559      1946392 :   auto_vector_modes vector_modes;
   10560              : 
   10561              :   /* Autodetect first vector size we try.  */
   10562      1946392 :   machine_mode next_vector_mode = VOIDmode;
   10563      1946392 :   targetm.vectorize.autovectorize_vector_modes (&vector_modes, false);
   10564      1946392 :   unsigned int mode_i = 0;
   10565              : 
   10566      1946392 :   vec_info_shared shared;
   10567              : 
   10568      1946392 :   machine_mode autodetected_vector_mode = VOIDmode;
   10569      2679458 :   while (1)
   10570              :     {
   10571      2312925 :       bool vectorized = false;
   10572      2312925 :       bool fatal = false;
   10573      2312925 :       bb_vinfo = new _bb_vec_info (bbs, &shared);
   10574              : 
   10575      2312925 :       bool first_time_p = shared.datarefs.is_empty ();
   10576      2312925 :       BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
   10577      2312925 :       if (first_time_p)
   10578      1971414 :         bb_vinfo->shared->save_datarefs ();
   10579              :       else
   10580       341511 :         bb_vinfo->shared->check_datarefs ();
   10581      2312925 :       bb_vinfo->vector_mode = next_vector_mode;
   10582              : 
   10583      2312925 :       if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups))
   10584              :         {
   10585       248546 :           if (dump_enabled_p ())
   10586              :             {
   10587         1734 :               dump_printf_loc (MSG_NOTE, vect_location,
   10588              :                                "***** Analysis succeeded with vector mode"
   10589          867 :                                " %s\n", GET_MODE_NAME (bb_vinfo->vector_mode));
   10590          867 :               dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
   10591              :             }
   10592              : 
   10593       248546 :           bb_vinfo->shared->check_datarefs ();
   10594              : 
   10595       248546 :           bool force_clear = false;
   10596       248546 :           auto_vec<slp_instance> profitable_subgraphs;
   10597      1468133 :           for (slp_instance instance : BB_VINFO_SLP_INSTANCES (bb_vinfo))
   10598              :             {
   10599       722495 :               if (instance->subgraph_entries.is_empty ())
   10600       240922 :                 continue;
   10601              : 
   10602       694876 :               dump_user_location_t saved_vect_location = vect_location;
   10603       694876 :               vect_location = instance->location ();
   10604       694876 :               if (!unlimited_cost_model (NULL)
   10605       690799 :                   && !param_vect_allow_possibly_not_worthwhile_vectorizations
   10606      1385666 :                   && !vect_bb_vectorization_profitable_p
   10607       690790 :                         (bb_vinfo, instance->subgraph_entries, orig_loop))
   10608              :                 {
   10609       213303 :                   if (dump_enabled_p ())
   10610           51 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10611              :                                      "not vectorized: vectorization is not "
   10612              :                                      "profitable.\n");
   10613       213303 :                   vect_location = saved_vect_location;
   10614       213303 :                   continue;
   10615              :                 }
   10616              : 
   10617       481573 :               vect_location = saved_vect_location;
   10618       481573 :               if (!dbg_cnt (vect_slp))
   10619              :                 {
   10620            0 :                   force_clear = true;
   10621            0 :                   continue;
   10622              :                 }
   10623              : 
   10624       481573 :               profitable_subgraphs.safe_push (instance);
   10625              :             }
   10626              : 
   10627              :           /* When we're vectorizing an if-converted loop body make sure
   10628              :              we vectorized all if-converted code.  */
   10629       406023 :           if ((!profitable_subgraphs.is_empty () || force_clear) && orig_loop)
   10630              :             {
   10631          130 :               gcc_assert (bb_vinfo->nbbs == 1);
   10632          260 :               for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[0]);
   10633         5837 :                    !gsi_end_p (gsi); gsi_next (&gsi))
   10634              :                 {
   10635              :                   /* The costing above left us with DCEable vectorized scalar
   10636              :                      stmts having the visited flag set on profitable
   10637              :                      subgraphs.  Do the delayed clearing of the flag here.  */
   10638         5707 :                   if (gimple_visited_p (gsi_stmt (gsi)))
   10639              :                     {
   10640         1816 :                       gimple_set_visited (gsi_stmt (gsi), false);
   10641         1816 :                       continue;
   10642              :                     }
   10643         3891 :                   if (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED)
   10644          893 :                     continue;
   10645              : 
   10646         8450 :                   if (gassign *ass = dyn_cast <gassign *> (gsi_stmt (gsi)))
   10647         3456 :                     if (gimple_assign_rhs_code (ass) == COND_EXPR)
   10648              :                       {
   10649          151 :                         if (!profitable_subgraphs.is_empty ()
   10650           66 :                             && dump_enabled_p ())
   10651            0 :                           dump_printf_loc (MSG_NOTE, vect_location,
   10652              :                                            "not profitable because of "
   10653              :                                            "unprofitable if-converted scalar "
   10654              :                                            "code\n");
   10655           85 :                         profitable_subgraphs.truncate (0);
   10656              :                       }
   10657              :                 }
   10658              :             }
   10659              : 
   10660              :           /* Finally schedule the profitable subgraphs.  */
   10661      1044990 :           for (slp_instance instance : profitable_subgraphs)
   10662              :             {
   10663       481490 :               if (!vectorized && dump_enabled_p ())
   10664          828 :                 dump_printf_loc (MSG_NOTE, vect_location,
   10665              :                                  "Basic block will be vectorized "
   10666              :                                  "using SLP\n");
   10667       481490 :               vectorized = true;
   10668              : 
   10669              :               /* Dump before scheduling as store vectorization will remove
   10670              :                  the original stores and mess with the instance tree
   10671              :                  so querying its location will eventually ICE.  */
   10672       481490 :               if (flag_checking)
   10673      1937166 :                 for (slp_instance sub : instance->subgraph_entries)
   10674       492696 :                   gcc_assert (SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub)));
   10675       481490 :               unsigned HOST_WIDE_INT bytes;
   10676       481490 :               if (dump_enabled_p ())
   10677         3999 :                 for (slp_instance sub : instance->subgraph_entries)
   10678              :                   {
   10679         1077 :                     tree vtype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub));
   10680         2154 :                     if (GET_MODE_SIZE (TYPE_MODE (vtype)).is_constant (&bytes))
   10681         1077 :                       dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
   10682         1077 :                                        sub->location (),
   10683              :                                        "basic block part vectorized using %wu "
   10684              :                                        "byte vectors\n", bytes);
   10685              :                     else
   10686              :                       dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
   10687              :                                        sub->location (),
   10688              :                                        "basic block part vectorized using "
   10689              :                                        "variable length vectors\n");
   10690              :                   }
   10691              : 
   10692       481490 :               dump_user_location_t saved_vect_location = vect_location;
   10693       481490 :               vect_location = instance->location ();
   10694              : 
   10695       481490 :               vect_schedule_slp (bb_vinfo, instance->subgraph_entries, false);
   10696              : 
   10697       481490 :               vect_location = saved_vect_location;
   10698              :             }
   10699              : 
   10700              :           /* Generate the invariant statements.  */
   10701       248546 :           if (!gimple_seq_empty_p (bb_vinfo->inv_pattern_def_seq))
   10702              :             {
   10703           23 :               if (dump_enabled_p ())
   10704            0 :                 dump_printf_loc (MSG_NOTE, vect_location,
   10705              :                          "------>generating invariant statements\n");
   10706              : 
   10707           23 :               bb_vinfo->insert_seq_on_entry (NULL,
   10708              :                                              bb_vinfo->inv_pattern_def_seq);
   10709              :             }
   10710       248546 :         }
   10711              :       else
   10712              :         {
   10713      2064379 :           if (dump_enabled_p ())
   10714         1448 :             dump_printf_loc (MSG_NOTE, vect_location,
   10715              :                              "***** Analysis failed with vector mode %s\n",
   10716         1448 :                              GET_MODE_NAME (bb_vinfo->vector_mode));
   10717              :         }
   10718              : 
   10719      2312925 :       if (mode_i == 0)
   10720      1946392 :         autodetected_vector_mode = bb_vinfo->vector_mode;
   10721              : 
   10722      2312925 :       if (!fatal)
   10723      3347316 :         while (mode_i < vector_modes.length ()
   10724      1915315 :                && vect_chooses_same_modes_p (bb_vinfo, vector_modes[mode_i]))
   10725              :           {
   10726       357024 :             if (dump_enabled_p ())
   10727         1892 :               dump_printf_loc (MSG_NOTE, vect_location,
   10728              :                                "***** The result for vector mode %s would"
   10729              :                                " be the same\n",
   10730          946 :                                GET_MODE_NAME (vector_modes[mode_i]));
   10731       357024 :             mode_i += 1;
   10732              :           }
   10733              : 
   10734      2312925 :       delete bb_vinfo;
   10735              : 
   10736      2312925 :       if (mode_i < vector_modes.length ()
   10737      2111858 :           && VECTOR_MODE_P (autodetected_vector_mode)
   10738      2089722 :           && (related_vector_mode (vector_modes[mode_i],
   10739              :                                    GET_MODE_INNER (autodetected_vector_mode))
   10740      1044861 :               == autodetected_vector_mode)
   10741      4424783 :           && (related_vector_mode (autodetected_vector_mode,
   10742       520817 :                                    GET_MODE_INNER (vector_modes[mode_i]))
   10743      1041634 :               == vector_modes[mode_i]))
   10744              :         {
   10745       520817 :           if (dump_enabled_p ())
   10746          230 :             dump_printf_loc (MSG_NOTE, vect_location,
   10747              :                              "***** Skipping vector mode %s, which would"
   10748              :                              " repeat the analysis for %s\n",
   10749          230 :                              GET_MODE_NAME (vector_modes[mode_i]),
   10750          230 :                              GET_MODE_NAME (autodetected_vector_mode));
   10751       520817 :           mode_i += 1;
   10752              :         }
   10753              : 
   10754      2312925 :       if (vectorized
   10755      2155514 :           || mode_i == vector_modes.length ()
   10756      1954491 :           || autodetected_vector_mode == VOIDmode
   10757              :           /* If vect_slp_analyze_bb_1 signaled that analysis for all
   10758              :              vector sizes will fail do not bother iterating.  */
   10759      3200419 :           || fatal)
   10760      3892784 :         return vectorized;
   10761              : 
   10762              :       /* Try the next biggest vector size.  */
   10763       366533 :       next_vector_mode = vector_modes[mode_i++];
   10764       366533 :       if (dump_enabled_p ())
   10765          257 :         dump_printf_loc (MSG_NOTE, vect_location,
   10766              :                          "***** Re-trying analysis with vector mode %s\n",
   10767          257 :                          GET_MODE_NAME (next_vector_mode));
   10768       366533 :     }
   10769      1946392 : }
   10770              : 
   10771              : 
   10772              : /* Main entry for the BB vectorizer.  Analyze and transform BBS, returns
   10773              :    true if anything in the basic-block was vectorized.  */
   10774              : 
   10775              : static bool
   10776      1946392 : vect_slp_bbs (const vec<basic_block> &bbs, loop_p orig_loop)
   10777              : {
   10778      1946392 :   vec<data_reference_p> datarefs = vNULL;
   10779      1946392 :   auto_vec<int> dataref_groups;
   10780      1946392 :   int insns = 0;
   10781      1946392 :   int current_group = 0;
   10782              : 
   10783     12871403 :   for (unsigned i = 0; i < bbs.length (); i++)
   10784              :     {
   10785     10925011 :       basic_block bb = bbs[i];
   10786     94873300 :       for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
   10787     83948289 :            gsi_next (&gsi))
   10788              :         {
   10789     83948289 :           gimple *stmt = gsi_stmt (gsi);
   10790     83948289 :           if (is_gimple_debug (stmt))
   10791     53067155 :             continue;
   10792              : 
   10793     30881134 :           insns++;
   10794              : 
   10795     30881134 :           if (gimple_location (stmt) != UNKNOWN_LOCATION)
   10796     27692030 :             vect_location = stmt;
   10797              : 
   10798     30881134 :           if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs,
   10799              :                                               &dataref_groups, current_group))
   10800      5315747 :             ++current_group;
   10801              :         }
   10802              :       /* New BBs always start a new DR group.  */
   10803     10925011 :       ++current_group;
   10804              :     }
   10805              : 
   10806      1946392 :   return vect_slp_region (bbs, datarefs, &dataref_groups, insns, orig_loop);
   10807      1946392 : }
   10808              : 
   10809              : /* Special entry for the BB vectorizer.  Analyze and transform a single
   10810              :    if-converted BB with ORIG_LOOPs body being the not if-converted
   10811              :    representation.  Returns true if anything in the basic-block was
   10812              :    vectorized.  */
   10813              : 
   10814              : bool
   10815        16943 : vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
   10816              : {
   10817        16943 :   auto_vec<basic_block> bbs;
   10818        16943 :   bbs.safe_push (bb);
   10819        16943 :   return vect_slp_bbs (bbs, orig_loop);
   10820        16943 : }
   10821              : 
   10822              : /* Main entry for the BB vectorizer.  Analyze and transform BB, returns
   10823              :    true if anything in the basic-block was vectorized.  */
   10824              : 
   10825              : bool
   10826       925882 : vect_slp_function (function *fun)
   10827              : {
   10828       925882 :   bool r = false;
   10829       925882 :   int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fun));
   10830       925882 :   auto_bitmap exit_bbs;
   10831       925882 :   bitmap_set_bit (exit_bbs, EXIT_BLOCK);
   10832       925882 :   edge entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
   10833       925882 :   unsigned n = rev_post_order_and_mark_dfs_back_seme (fun, entry, exit_bbs,
   10834       925882 :                                                       true, rpo, NULL);
   10835              : 
   10836              :   /* For the moment split the function into pieces to avoid making
   10837              :      the iteration on the vector mode moot.  Split at points we know
   10838              :      to not handle well which is CFG merges (SLP discovery doesn't
   10839              :      handle non-loop-header PHIs) and loop exits.  Since pattern
   10840              :      recog requires reverse iteration to visit uses before defs
   10841              :      simply chop RPO into pieces.  */
   10842       925882 :   auto_vec<basic_block> bbs;
   10843     11860734 :   for (unsigned i = 0; i < n; i++)
   10844              :     {
   10845     10934852 :       basic_block bb = BASIC_BLOCK_FOR_FN (fun, rpo[i]);
   10846     10934852 :       bool split = false;
   10847              : 
   10848              :       /* Split when a BB is not dominated by the first block.  */
   10849     20627440 :       if (!bbs.is_empty ()
   10850      9692588 :           && !dominated_by_p (CDI_DOMINATORS, bb, bbs[0]))
   10851              :         {
   10852       702626 :           if (dump_enabled_p ())
   10853          185 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10854              :                              "splitting region at dominance boundary bb%d\n",
   10855              :                              bb->index);
   10856              :           split = true;
   10857              :         }
   10858              :       /* Split when the loop determined by the first block
   10859              :          is exited.  This is because we eventually insert
   10860              :          invariants at region begin.  */
   10861     19222188 :       else if (!bbs.is_empty ()
   10862      8989962 :                && bbs[0]->loop_father != bb->loop_father
   10863      2356054 :                && !flow_loop_nested_p (bbs[0]->loop_father, bb->loop_father))
   10864              :         {
   10865         3881 :           if (dump_enabled_p ())
   10866            3 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10867              :                              "splitting region at loop %d exit at bb%d\n",
   10868            3 :                              bbs[0]->loop_father->num, bb->index);
   10869              :           split = true;
   10870              :         }
   10871     10228345 :       else if (!bbs.is_empty ()
   10872      8986081 :                && bb->loop_father->header == bb
   10873       483423 :                && bb->loop_father->dont_vectorize)
   10874              :         {
   10875         6225 :           if (dump_enabled_p ())
   10876           72 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10877              :                              "splitting region at dont-vectorize loop %d "
   10878              :                              "entry at bb%d\n",
   10879              :                              bb->loop_father->num, bb->index);
   10880              :           split = true;
   10881              :         }
   10882     10222120 :       else if (!bbs.is_empty ()
   10883      8979856 :                && bb_has_abnormal_pred (bb))
   10884              :         {
   10885         1237 :           if (dump_enabled_p ())
   10886           15 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10887              :                              "splitting region at incoming abnormal edge "
   10888              :                              "at bb%d\n", bb->index);
   10889              :           split = true;
   10890              :         }
   10891              : 
   10892          275 :       if (split && !bbs.is_empty ())
   10893              :         {
   10894       713969 :           r |= vect_slp_bbs (bbs, NULL);
   10895       713969 :           bbs.truncate (0);
   10896              :         }
   10897              : 
   10898     10934852 :       if (bbs.is_empty ())
   10899              :         {
   10900              :           /* We need to be able to insert at the head of the region which
   10901              :              we cannot for region starting with a returns-twice call.  */
   10902      1956233 :           if (gcall *first = safe_dyn_cast <gcall *> (first_stmt (bb)))
   10903       415089 :             if (gimple_call_flags (first) & ECF_RETURNS_TWICE)
   10904              :               {
   10905          612 :                 if (dump_enabled_p ())
   10906           12 :                   dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10907              :                                    "skipping bb%d as start of region as it "
   10908              :                                    "starts with returns-twice call\n",
   10909              :                                    bb->index);
   10910        26784 :                 continue;
   10911              :               }
   10912              :           /* If the loop this BB belongs to is marked as not to be vectorized
   10913              :              honor that also for BB vectorization.  */
   10914      1955621 :           if (bb->loop_father->dont_vectorize)
   10915        26172 :             continue;
   10916              :         }
   10917              : 
   10918     10908068 :       bbs.safe_push (bb);
   10919              : 
   10920              :       /* When we have a stmt ending this block and defining a
   10921              :          value we have to insert on edges when inserting after it for
   10922              :          a vector containing its definition.  Avoid this for now.  */
   10923     21816136 :       if (gimple *last = *gsi_last_bb (bb))
   10924      8876804 :         if (gimple_get_lhs (last)
   10925      8876804 :             && is_ctrl_altering_stmt (last))
   10926              :           {
   10927       289608 :             if (dump_enabled_p ())
   10928            7 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10929              :                                "splitting region at control altering "
   10930              :                                "definition %G", last);
   10931       289608 :             r |= vect_slp_bbs (bbs, NULL);
   10932       289608 :             bbs.truncate (0);
   10933              :           }
   10934              :     }
   10935              : 
   10936       925882 :   if (!bbs.is_empty ())
   10937       925872 :     r |= vect_slp_bbs (bbs, NULL);
   10938              : 
   10939       925882 :   free (rpo);
   10940              : 
   10941       925882 :   return r;
   10942       925882 : }
   10943              : 
   10944              : /* Build a variable-length vector in which the elements in ELTS are repeated
   10945              :    to a fill NRESULTS vectors of type VECTOR_TYPE.  Store the vectors in
   10946              :    RESULTS and add any new instructions to SEQ.
   10947              : 
   10948              :    The approach we use is:
   10949              : 
   10950              :    (1) Find a vector mode VM with integer elements of mode IM.
   10951              : 
   10952              :    (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
   10953              :        ELTS' has mode IM.  This involves creating NELTS' VIEW_CONVERT_EXPRs
   10954              :        from small vectors to IM.
   10955              : 
   10956              :    (3) Duplicate each ELTS'[I] into a vector of mode VM.
   10957              : 
   10958              :    (4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
   10959              :        correct byte contents.
   10960              : 
   10961              :    (5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
   10962              : 
   10963              :    We try to find the largest IM for which this sequence works, in order
   10964              :    to cut down on the number of interleaves.  */
   10965              : 
   10966              : void
   10967            0 : duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
   10968              :                           const vec<tree> &elts, unsigned int nresults,
   10969              :                           vec<tree> &results)
   10970              : {
   10971            0 :   unsigned int nelts = elts.length ();
   10972            0 :   tree element_type = TREE_TYPE (vector_type);
   10973              : 
   10974              :   /* (1) Find a vector mode VM with integer elements of mode IM.  */
   10975            0 :   unsigned int nvectors = 1;
   10976            0 :   tree new_vector_type;
   10977            0 :   tree permutes[2];
   10978            0 :   if (!can_duplicate_and_interleave_p (vinfo, nelts, element_type,
   10979              :                                        &nvectors, &new_vector_type,
   10980              :                                        permutes))
   10981            0 :     gcc_unreachable ();
   10982              : 
   10983              :   /* Get a vector type that holds ELTS[0:NELTS/NELTS'].  */
   10984            0 :   unsigned int partial_nelts = nelts / nvectors;
   10985            0 :   tree partial_vector_type = build_vector_type (element_type, partial_nelts);
   10986              : 
   10987            0 :   tree_vector_builder partial_elts;
   10988            0 :   auto_vec<tree, 32> pieces (nvectors * 2);
   10989            0 :   pieces.quick_grow_cleared (nvectors * 2);
   10990            0 :   for (unsigned int i = 0; i < nvectors; ++i)
   10991              :     {
   10992              :       /* (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
   10993              :              ELTS' has mode IM.  */
   10994            0 :       partial_elts.new_vector (partial_vector_type, partial_nelts, 1);
   10995            0 :       for (unsigned int j = 0; j < partial_nelts; ++j)
   10996            0 :         partial_elts.quick_push (elts[i * partial_nelts + j]);
   10997            0 :       tree t = gimple_build_vector (seq, &partial_elts);
   10998            0 :       t = gimple_build (seq, VIEW_CONVERT_EXPR,
   10999            0 :                         TREE_TYPE (new_vector_type), t);
   11000              : 
   11001              :       /* (3) Duplicate each ELTS'[I] into a vector of mode VM.  */
   11002            0 :       pieces[i] = gimple_build_vector_from_val (seq, new_vector_type, t);
   11003              :     }
   11004              : 
   11005              :   /* (4) Use a tree of VEC_PERM_EXPRs to create a single VM with the
   11006              :          correct byte contents.
   11007              : 
   11008              :      Conceptually, we need to repeat the following operation log2(nvectors)
   11009              :      times, where hi_start = nvectors / 2:
   11010              : 
   11011              :         out[i * 2] = VEC_PERM_EXPR (in[i], in[i + hi_start], lo_permute);
   11012              :         out[i * 2 + 1] = VEC_PERM_EXPR (in[i], in[i + hi_start], hi_permute);
   11013              : 
   11014              :      However, if each input repeats every N elements and the VF is
   11015              :      a multiple of N * 2, the HI result is the same as the LO result.
   11016              :      This will be true for the first N1 iterations of the outer loop,
   11017              :      followed by N2 iterations for which both the LO and HI results
   11018              :      are needed.  I.e.:
   11019              : 
   11020              :         N1 + N2 = log2(nvectors)
   11021              : 
   11022              :      Each "N1 iteration" doubles the number of redundant vectors and the
   11023              :      effect of the process as a whole is to have a sequence of nvectors/2**N1
   11024              :      vectors that repeats 2**N1 times.  Rather than generate these redundant
   11025              :      vectors, we halve the number of vectors for each N1 iteration.  */
   11026              :   unsigned int in_start = 0;
   11027              :   unsigned int out_start = nvectors;
   11028              :   unsigned int new_nvectors = nvectors;
   11029            0 :   for (unsigned int in_repeat = 1; in_repeat < nvectors; in_repeat *= 2)
   11030              :     {
   11031            0 :       unsigned int hi_start = new_nvectors / 2;
   11032            0 :       unsigned int out_i = 0;
   11033            0 :       for (unsigned int in_i = 0; in_i < new_nvectors; ++in_i)
   11034              :         {
   11035            0 :           if ((in_i & 1) != 0
   11036            0 :               && multiple_p (TYPE_VECTOR_SUBPARTS (new_vector_type),
   11037              :                              2 * in_repeat))
   11038            0 :             continue;
   11039              : 
   11040            0 :           tree output = make_ssa_name (new_vector_type);
   11041            0 :           tree input1 = pieces[in_start + (in_i / 2)];
   11042            0 :           tree input2 = pieces[in_start + (in_i / 2) + hi_start];
   11043            0 :           gassign *stmt = gimple_build_assign (output, VEC_PERM_EXPR,
   11044              :                                                input1, input2,
   11045              :                                                permutes[in_i & 1]);
   11046            0 :           gimple_seq_add_stmt (seq, stmt);
   11047            0 :           pieces[out_start + out_i] = output;
   11048            0 :           out_i += 1;
   11049              :         }
   11050            0 :       std::swap (in_start, out_start);
   11051            0 :       new_nvectors = out_i;
   11052              :     }
   11053              : 
   11054              :   /* (5) Use VIEW_CONVERT_EXPR to cast the final VM to the required type.  */
   11055            0 :   results.reserve (nresults);
   11056            0 :   for (unsigned int i = 0; i < nresults; ++i)
   11057            0 :     if (i < new_nvectors)
   11058            0 :       results.quick_push (gimple_build (seq, VIEW_CONVERT_EXPR, vector_type,
   11059            0 :                                         pieces[in_start + i]));
   11060              :     else
   11061            0 :       results.quick_push (results[i - new_nvectors]);
   11062            0 : }
   11063              : 
   11064              : 
   11065              : /* For constant and loop invariant defs in OP_NODE this function creates
   11066              :    vector defs that will be used in the vectorized stmts and stores them
   11067              :    to SLP_TREE_VEC_DEFS of OP_NODE.  */
   11068              : 
   11069              : static void
   11070       503059 : vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
   11071              : {
   11072       503059 :   unsigned HOST_WIDE_INT nunits;
   11073       503059 :   tree vec_cst;
   11074       503059 :   unsigned j, number_of_places_left_in_vector;
   11075       503059 :   tree vector_type;
   11076       503059 :   tree vop;
   11077       503059 :   int group_size = op_node->ops.length ();
   11078       503059 :   unsigned int vec_num, i;
   11079       503059 :   unsigned number_of_copies = 1;
   11080       503059 :   bool constant_p;
   11081       503059 :   gimple_seq ctor_seq = NULL;
   11082       503059 :   auto_vec<tree, 16> permute_results;
   11083              : 
   11084              :   /* We always want SLP_TREE_VECTYPE (op_node) here correctly set.  */
   11085       503059 :   vector_type = SLP_TREE_VECTYPE (op_node);
   11086              : 
   11087       503059 :   unsigned int number_of_vectors, excess_elts;
   11088       503059 :   bool res = vect_get_num_copies_for_invariant (vinfo, op_node,
   11089              :                                                 &number_of_vectors,
   11090              :                                                 &excess_elts);
   11091       503059 :   gcc_assert (res);
   11092       503059 :   SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
   11093       503059 :   auto_vec<tree> voprnds (number_of_vectors);
   11094              : 
   11095              :   /* NUMBER_OF_COPIES is the number of times we need to use the same values in
   11096              :      created vectors. It is greater than 1 if unrolling is performed.
   11097              : 
   11098              :      For example, we have two scalar operands, s1 and s2 (e.g., group of
   11099              :      strided accesses of size two), while NUNITS is four (i.e., four scalars
   11100              :      of this type can be packed in a vector).  The output vector will contain
   11101              :      two copies of each scalar operand: {s1, s2, s1, s2}.  (NUMBER_OF_COPIES
   11102              :      will be 2).
   11103              : 
   11104              :      If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
   11105              :      containing the operands.
   11106              : 
   11107              :      For example, NUNITS is four as before, and the group size is 8
   11108              :      (s1, s2, ..., s8).  We will create two vectors {s1, s2, s3, s4} and
   11109              :      {s5, s6, s7, s8}.  */
   11110              : 
   11111              :   /* When using duplicate_and_interleave, we just need one element for
   11112              :      each scalar statement.  */
   11113       503059 :   if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
   11114              :     nunits = group_size;
   11115              : 
   11116       503059 :   number_of_copies = (nunits * number_of_vectors - excess_elts) / group_size;
   11117              : 
   11118       503059 :   constant_p = true;
   11119       503059 :   tree uniform_elt = NULL_TREE;
   11120       503059 :   tree_vector_builder elts (vector_type, nunits, 1);
   11121       503059 :   elts.quick_grow (nunits);
   11122              :   /* Zero-pad the last vector if necessary.  */
   11123       503059 :   number_of_places_left_in_vector = nunits - excess_elts;
   11124       503317 :   for (i = nunits; i > number_of_places_left_in_vector; --i)
   11125          258 :     elts[i-1] = build_zero_cst (TREE_TYPE (vector_type));
   11126              :   stmt_vec_info insert_after = NULL;
   11127      1501902 :   for (j = 0; j < number_of_copies; j++)
   11128              :     {
   11129       998843 :       tree op;
   11130      3825581 :       for (i = group_size - 1; op_node->ops.iterate (i, &op); i--)
   11131              :         {
   11132              :           /* Create 'vect_ = {op0,op1,...,opn}'.  */
   11133      1827895 :           tree orig_op = op;
   11134      1827895 :           if (number_of_places_left_in_vector == nunits)
   11135              :             uniform_elt = op;
   11136      1193072 :           else if (uniform_elt && operand_equal_p (uniform_elt, op))
   11137       757266 :             op = elts[number_of_places_left_in_vector];
   11138              :           else
   11139              :             uniform_elt = NULL_TREE;
   11140      1827895 :           number_of_places_left_in_vector--;
   11141      1827895 :           if (!types_compatible_p (TREE_TYPE (vector_type), TREE_TYPE (op)))
   11142              :             {
   11143       279627 :               if (CONSTANT_CLASS_P (op))
   11144              :                 {
   11145       101213 :                   if (VECTOR_BOOLEAN_TYPE_P (vector_type))
   11146              :                     {
   11147              :                       /* Can't use VIEW_CONVERT_EXPR for booleans because
   11148              :                          of possibly different sizes of scalar value and
   11149              :                          vector element.  */
   11150           66 :                       if (integer_zerop (op))
   11151           66 :                         op = build_int_cst (TREE_TYPE (vector_type), 0);
   11152            0 :                       else if (integer_onep (op))
   11153            0 :                         op = build_all_ones_cst (TREE_TYPE (vector_type));
   11154              :                       else
   11155            0 :                         gcc_unreachable ();
   11156              :                     }
   11157              :                   else
   11158       101147 :                     op = fold_unary (VIEW_CONVERT_EXPR,
   11159              :                                      TREE_TYPE (vector_type), op);
   11160       101213 :                   gcc_assert (op && CONSTANT_CLASS_P (op));
   11161              :                 }
   11162              :               else
   11163              :                 {
   11164       178414 :                   tree new_temp = make_ssa_name (TREE_TYPE (vector_type));
   11165       178414 :                   gimple *init_stmt;
   11166       178414 :                   if (VECTOR_BOOLEAN_TYPE_P (vector_type))
   11167              :                     {
   11168          427 :                       tree true_val
   11169          427 :                         = build_all_ones_cst (TREE_TYPE (vector_type));
   11170          427 :                       tree false_val
   11171          427 :                         = build_zero_cst (TREE_TYPE (vector_type));
   11172          427 :                       gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
   11173          427 :                       init_stmt = gimple_build_assign (new_temp, COND_EXPR,
   11174              :                                                        op, true_val,
   11175              :                                                        false_val);
   11176              :                     }
   11177              :                   else
   11178              :                     {
   11179       177987 :                       op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vector_type),
   11180              :                                    op);
   11181       177987 :                       init_stmt
   11182       177987 :                         = gimple_build_assign (new_temp, VIEW_CONVERT_EXPR,
   11183              :                                                op);
   11184              :                     }
   11185       178414 :                   gimple_seq_add_stmt (&ctor_seq, init_stmt);
   11186       178414 :                   op = new_temp;
   11187              :                 }
   11188              :             }
   11189      1827895 :           elts[number_of_places_left_in_vector] = op;
   11190      1827895 :           if (!CONSTANT_CLASS_P (op))
   11191       322876 :             constant_p = false;
   11192              :           /* For BB vectorization we have to compute an insert location
   11193              :              when a def is inside the analyzed region since we cannot
   11194              :              simply insert at the BB start in this case.  */
   11195      1827895 :           stmt_vec_info opdef;
   11196      1827895 :           if (TREE_CODE (orig_op) == SSA_NAME
   11197       185406 :               && !SSA_NAME_IS_DEFAULT_DEF (orig_op)
   11198       164570 :               && is_a <bb_vec_info> (vinfo)
   11199      1934398 :               && (opdef = vinfo->lookup_def (orig_op)))
   11200              :             {
   11201        86346 :               if (!insert_after)
   11202              :                 insert_after = opdef;
   11203              :               else
   11204        47617 :                 insert_after = get_later_stmt (insert_after, opdef);
   11205              :             }
   11206              : 
   11207      1827895 :           if (number_of_places_left_in_vector == 0)
   11208              :             {
   11209       634852 :               auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
   11210       634852 :               if (uniform_elt)
   11211       657080 :                 vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
   11212       328540 :                                                         elts[0]);
   11213       612624 :               else if (constant_p
   11214       612624 :                        ? multiple_p (type_nunits, nunits)
   11215       111864 :                        : known_eq (type_nunits, nunits))
   11216       306312 :                 vec_cst = gimple_build_vector (&ctor_seq, &elts);
   11217              :               else
   11218              :                 {
   11219            0 :                   if (permute_results.is_empty ())
   11220            0 :                     duplicate_and_interleave (vinfo, &ctor_seq, vector_type,
   11221              :                                               elts, number_of_vectors,
   11222              :                                               permute_results);
   11223            0 :                   vec_cst = permute_results[number_of_vectors - j - 1];
   11224              :                 }
   11225       634852 :               if (!gimple_seq_empty_p (ctor_seq))
   11226              :                 {
   11227       139704 :                   if (insert_after)
   11228              :                     {
   11229        38729 :                       gimple_stmt_iterator gsi;
   11230        38729 :                       if (gimple_code (insert_after->stmt) == GIMPLE_PHI)
   11231              :                         {
   11232          612 :                           gsi = gsi_after_labels (gimple_bb (insert_after->stmt));
   11233          612 :                           gsi_insert_seq_before (&gsi, ctor_seq,
   11234              :                                                  GSI_CONTINUE_LINKING);
   11235              :                         }
   11236        38117 :                       else if (!stmt_ends_bb_p (insert_after->stmt))
   11237              :                         {
   11238        38117 :                           gsi = gsi_for_stmt (insert_after->stmt);
   11239        38117 :                           gsi_insert_seq_after (&gsi, ctor_seq,
   11240              :                                                 GSI_CONTINUE_LINKING);
   11241              :                         }
   11242              :                       else
   11243              :                         {
   11244              :                           /* When we want to insert after a def where the
   11245              :                              defining stmt throws then insert on the fallthru
   11246              :                              edge.  */
   11247            0 :                           edge e = find_fallthru_edge
   11248            0 :                                      (gimple_bb (insert_after->stmt)->succs);
   11249            0 :                           basic_block new_bb
   11250            0 :                             = gsi_insert_seq_on_edge_immediate (e, ctor_seq);
   11251            0 :                           gcc_assert (!new_bb);
   11252              :                         }
   11253              :                     }
   11254              :                   else
   11255       100975 :                     vinfo->insert_seq_on_entry (NULL, ctor_seq);
   11256       139704 :                   ctor_seq = NULL;
   11257              :                 }
   11258       634852 :               voprnds.quick_push (vec_cst);
   11259       634852 :               insert_after = NULL;
   11260       634852 :               number_of_places_left_in_vector = nunits;
   11261       634852 :               constant_p = true;
   11262       634852 :               elts.new_vector (vector_type, nunits, 1);
   11263       634852 :               elts.quick_grow (nunits);
   11264              :             }
   11265              :         }
   11266              :     }
   11267              : 
   11268       503059 :   gcc_assert (number_of_places_left_in_vector == nunits);
   11269              : 
   11270              :   /* Since the vectors are created in the reverse order, we should invert
   11271              :      them.  */
   11272       503059 :   vec_num = voprnds.length ();
   11273       503059 :   gcc_checking_assert (vec_num > 0);
   11274              : 
   11275      1137911 :   for (j = vec_num; j != 0; j--)
   11276              :     {
   11277       634852 :       vop = voprnds[j - 1];
   11278       634852 :       SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
   11279              :     }
   11280              : 
   11281              :   /* In case that VF is greater than the unrolling factor needed for the SLP
   11282              :      group of stmts, NUMBER_OF_VECTORS to be created is greater than
   11283              :      NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
   11284              :      to replicate the vectors.  */
   11285      1006118 :   while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ())
   11286            0 :     for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num;
   11287              :          i++)
   11288            0 :       SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
   11289       503059 : }
   11290              : 
   11291              : /* Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE
   11292              :    if there is no definition for it in the scalar IL or it is not known.  */
   11293              : 
   11294              : tree
   11295        78953 : vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
   11296              : {
   11297        78953 :   if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
   11298              :     {
   11299         2716 :       if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
   11300              :         return NULL_TREE;
   11301         2716 :       stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
   11302         2716 :       if (!def)
   11303              :         return NULL_TREE;
   11304         2716 :       return gimple_get_lhs (STMT_VINFO_STMT (def));
   11305              :     }
   11306              :   else
   11307        76237 :     return SLP_TREE_SCALAR_OPS (slp_node)[n];
   11308              : }
   11309              : 
   11310              : /* Get the Ith vectorized definition from SLP_NODE.  */
   11311              : 
   11312              : tree
   11313       141103 : vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
   11314              : {
   11315       141103 :   return SLP_TREE_VEC_DEFS (slp_node)[i];
   11316              : }
   11317              : 
   11318              : /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS.  */
   11319              : 
   11320              : void
   11321       942003 : vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
   11322              : {
   11323      1884006 :   vec_defs->create (SLP_TREE_VEC_DEFS (slp_node).length ());
   11324       942003 :   vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
   11325       942003 : }
   11326              : 
   11327              : /* Get N vectorized definitions for SLP_NODE.  */
   11328              : 
   11329              : void
   11330         3378 : vect_get_slp_defs (vec_info *,
   11331              :                    slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
   11332              : {
   11333         3378 :   if (n == -1U)
   11334         3378 :     n = SLP_TREE_CHILDREN (slp_node).length ();
   11335              : 
   11336        11803 :   for (unsigned i = 0; i < n; ++i)
   11337              :     {
   11338         8425 :       slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
   11339         8425 :       vec<tree> vec_defs = vNULL;
   11340         8425 :       vect_get_slp_defs (child, &vec_defs);
   11341         8425 :       vec_oprnds->quick_push (vec_defs);
   11342              :     }
   11343         3378 : }
   11344              : 
   11345              : /* A subroutine of vect_transform_slp_perm_load with two extra arguments:
   11346              :    - PERM gives the permutation that the caller wants to use for NODE,
   11347              :      which might be different from SLP_LOAD_PERMUTATION.
   11348              :    - DUMP_P controls whether the function dumps information.  */
   11349              : 
   11350              : static bool
   11351       134954 : vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
   11352              :                                 load_permutation_t &perm,
   11353              :                                 const vec<tree> &dr_chain,
   11354              :                                 gimple_stmt_iterator *gsi, poly_uint64 vf,
   11355              :                                 bool analyze_only, bool dump_p,
   11356              :                                 unsigned *n_perms, unsigned int *n_loads,
   11357              :                                 bool dce_chain)
   11358              : {
   11359       134954 :   stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
   11360       134954 :   int vec_index = 0;
   11361       134954 :   tree vectype = SLP_TREE_VECTYPE (node);
   11362       134954 :   unsigned int group_size = SLP_TREE_LANES (node);
   11363       134954 :   unsigned int mask_element;
   11364       134954 :   unsigned dr_group_size;
   11365       134954 :   machine_mode mode;
   11366              : 
   11367       134954 :   if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
   11368              :     {
   11369              :       /* We have both splats of the same non-grouped load and groups
   11370              :          of distinct invariant loads entering here.  */
   11371         1538 :       unsigned max_idx = 0;
   11372         8498 :       for (auto idx : perm)
   11373         3884 :         max_idx = idx > max_idx ? idx : max_idx;
   11374         1538 :       dr_group_size = max_idx + 1;
   11375              :     }
   11376              :   else
   11377              :     {
   11378       133416 :       stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
   11379       133416 :       dr_group_size = DR_GROUP_SIZE (stmt_info);
   11380              :     }
   11381              : 
   11382       134954 :   mode = TYPE_MODE (vectype);
   11383       134954 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
   11384       134954 :   unsigned int nstmts = vect_get_num_copies (vinfo, node);
   11385              : 
   11386              :   /* Initialize the vect stmts of NODE to properly insert the generated
   11387              :      stmts later.  */
   11388       134954 :   if (! analyze_only)
   11389        57621 :     for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++)
   11390        22235 :       SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE);
   11391              : 
   11392              :   /* Generate permutation masks for every NODE. Number of masks for each NODE
   11393              :      is equal to GROUP_SIZE.
   11394              :      E.g., we have a group of three nodes with three loads from the same
   11395              :      location in each node, and the vector size is 4. I.e., we have a
   11396              :      a0b0c0a1b1c1... sequence and we need to create the following vectors:
   11397              :      for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
   11398              :      for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
   11399              :      ...
   11400              : 
   11401              :      The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
   11402              :      The last mask is illegal since we assume two operands for permute
   11403              :      operation, and the mask element values can't be outside that range.
   11404              :      Hence, the last mask must be converted into {2,5,5,5}.
   11405              :      For the first two permutations we need the first and the second input
   11406              :      vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
   11407              :      we need the second and the third vectors: {b1,c1,a2,b2} and
   11408              :      {c2,a3,b3,c3}.  */
   11409              : 
   11410       134954 :   int vect_stmts_counter = 0;
   11411       134954 :   unsigned int index = 0;
   11412       134954 :   int first_vec_index = -1;
   11413       134954 :   int second_vec_index = -1;
   11414       134954 :   bool noop_p = true;
   11415       134954 :   *n_perms = 0;
   11416              : 
   11417       134954 :   vec_perm_builder mask;
   11418       134954 :   unsigned int nelts_to_build;
   11419       134954 :   unsigned int nvectors_per_build;
   11420       134954 :   unsigned int in_nlanes;
   11421       134954 :   bool repeating_p = (group_size == dr_group_size
   11422       170809 :                       && multiple_p (nunits, group_size));
   11423       134954 :   if (repeating_p)
   11424              :     {
   11425              :       /* A single vector contains a whole number of copies of the node, so:
   11426              :          (a) all permutes can use the same mask; and
   11427              :          (b) the permutes only need a single vector input.  */
   11428        33345 :       mask.new_vector (nunits, group_size, 3);
   11429        33345 :       nelts_to_build = mask.encoded_nelts ();
   11430              :       /* It's possible to obtain zero nstmts during analyze_only, so make
   11431              :          it at least one to ensure the later computation for n_perms
   11432              :          proceed.  */
   11433        33345 :       nvectors_per_build = nstmts > 0 ? nstmts : 1;
   11434        33345 :       in_nlanes = dr_group_size * 3;
   11435              :     }
   11436              :   else
   11437              :     {
   11438              :       /* We need to construct a separate mask for each vector statement.  */
   11439       101609 :       unsigned HOST_WIDE_INT const_nunits, const_vf;
   11440       101609 :       if (!nunits.is_constant (&const_nunits)
   11441       101609 :           || !vf.is_constant (&const_vf))
   11442              :         return false;
   11443       101609 :       mask.new_vector (const_nunits, const_nunits, 1);
   11444       101609 :       nelts_to_build = const_vf * group_size;
   11445       101609 :       nvectors_per_build = 1;
   11446       101609 :       in_nlanes = const_vf * dr_group_size;
   11447              :     }
   11448       134954 :   auto_sbitmap used_in_lanes (in_nlanes);
   11449       134954 :   bitmap_clear (used_in_lanes);
   11450       134954 :   auto_bitmap used_defs;
   11451              : 
   11452       134954 :   unsigned int count = mask.encoded_nelts ();
   11453       134954 :   mask.quick_grow (count);
   11454       134954 :   vec_perm_indices indices;
   11455              : 
   11456       709405 :   for (unsigned int j = 0; j < nelts_to_build; j++)
   11457              :     {
   11458       585515 :       unsigned int iter_num = j / group_size;
   11459       585515 :       unsigned int stmt_num = j % group_size;
   11460       585515 :       unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
   11461       585515 :       bitmap_set_bit (used_in_lanes, i);
   11462       585515 :       if (repeating_p)
   11463              :         {
   11464              :           first_vec_index = 0;
   11465              :           mask_element = i;
   11466              :         }
   11467              :       else
   11468              :         {
   11469              :           /* Enforced before the loop when !repeating_p.  */
   11470       368135 :           unsigned int const_nunits = nunits.to_constant ();
   11471       368135 :           vec_index = i / const_nunits;
   11472       368135 :           mask_element = i % const_nunits;
   11473       368135 :           if (vec_index == first_vec_index
   11474       368135 :               || first_vec_index == -1)
   11475              :             {
   11476              :               first_vec_index = vec_index;
   11477              :             }
   11478       146081 :           else if (vec_index == second_vec_index
   11479       146081 :                    || second_vec_index == -1)
   11480              :             {
   11481       139545 :               second_vec_index = vec_index;
   11482       139545 :               mask_element += const_nunits;
   11483              :             }
   11484              :           else
   11485              :             {
   11486         6536 :               if (dump_p)
   11487          280 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   11488              :                                  "permutation requires at "
   11489              :                                  "least three vectors %G",
   11490              :                                  stmt_info->stmt);
   11491         6536 :               gcc_assert (analyze_only);
   11492              :               return false;
   11493              :             }
   11494              : 
   11495       361599 :           gcc_assert (mask_element < 2 * const_nunits);
   11496              :         }
   11497              : 
   11498       578979 :       if (mask_element != index)
   11499       365456 :         noop_p = false;
   11500       578979 :       mask[index++] = mask_element;
   11501              : 
   11502       578979 :       if (index == count)
   11503              :         {
   11504       159586 :           if (!noop_p)
   11505              :             {
   11506       129297 :               indices.new_vector (mask, second_vec_index == -1 ? 1 : 2, nunits);
   11507       129297 :               if (!can_vec_perm_const_p (mode, mode, indices))
   11508              :                 {
   11509         4528 :                   if (dump_p)
   11510              :                     {
   11511           90 :                       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   11512              :                                        "unsupported vect permute { ");
   11513          876 :                       for (i = 0; i < count; ++i)
   11514              :                         {
   11515          786 :                           dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
   11516          786 :                           dump_printf (MSG_MISSED_OPTIMIZATION, " ");
   11517              :                         }
   11518           90 :                       dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
   11519              :                     }
   11520         4528 :                   gcc_assert (analyze_only);
   11521              :                   return false;
   11522              :                 }
   11523              : 
   11524       124769 :               tree mask_vec = NULL_TREE;
   11525       124769 :               if (!analyze_only)
   11526        20513 :                 mask_vec = vect_gen_perm_mask_checked (vectype, indices);
   11527              : 
   11528       124769 :               if (second_vec_index == -1)
   11529        36809 :                 second_vec_index = first_vec_index;
   11530              : 
   11531       250363 :               for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
   11532              :                 {
   11533       125594 :                   ++*n_perms;
   11534       125594 :                   if (analyze_only)
   11535       104798 :                     continue;
   11536              :                   /* Generate the permute statement if necessary.  */
   11537        20796 :                   tree first_vec = dr_chain[first_vec_index + ri];
   11538        20796 :                   tree second_vec = dr_chain[second_vec_index + ri];
   11539        20796 :                   gassign *stmt = as_a<gassign *> (stmt_info->stmt);
   11540        20796 :                   tree perm_dest
   11541        20796 :                     = vect_create_destination_var (gimple_assign_lhs (stmt),
   11542              :                                                    vectype);
   11543        20796 :                   perm_dest = make_ssa_name (perm_dest);
   11544        20796 :                   gimple *perm_stmt
   11545        20796 :                     = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec,
   11546              :                                            second_vec, mask_vec);
   11547        20796 :                   vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
   11548              :                                                gsi);
   11549        20796 :                   if (dce_chain)
   11550              :                     {
   11551        19873 :                       bitmap_set_bit (used_defs, first_vec_index + ri);
   11552        19873 :                       bitmap_set_bit (used_defs, second_vec_index + ri);
   11553              :                     }
   11554              : 
   11555              :                   /* Store the vector statement in NODE.  */
   11556        20796 :                   SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest;
   11557              :                 }
   11558              :             }
   11559        30289 :           else if (!analyze_only)
   11560              :             {
   11561         2878 :               for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
   11562              :                 {
   11563         1439 :                   tree first_vec = dr_chain[first_vec_index + ri];
   11564              :                   /* If mask was NULL_TREE generate the requested
   11565              :                      identity transform.  */
   11566         1439 :                   if (dce_chain)
   11567         1432 :                     bitmap_set_bit (used_defs, first_vec_index + ri);
   11568              : 
   11569              :                   /* Store the vector statement in NODE.  */
   11570         1439 :                   SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec;
   11571              :                 }
   11572              :             }
   11573              : 
   11574              :           index = 0;
   11575              :           first_vec_index = -1;
   11576              :           second_vec_index = -1;
   11577              :           noop_p = true;
   11578              :         }
   11579              :     }
   11580              : 
   11581       123890 :   if (n_loads)
   11582              :     {
   11583        86741 :       if (repeating_p)
   11584         9334 :         *n_loads = nstmts;
   11585              :       else
   11586              :         {
   11587              :           /* Enforced above when !repeating_p.  */
   11588        77407 :           unsigned int const_nunits = nunits.to_constant ();
   11589        77407 :           *n_loads = 0;
   11590        77407 :           bool load_seen = false;
   11591      1051979 :           for (unsigned i = 0; i < in_nlanes; ++i)
   11592              :             {
   11593       974572 :               if (i % const_nunits == 0)
   11594              :                 {
   11595       420474 :                   if (load_seen)
   11596       123491 :                     *n_loads += 1;
   11597              :                   load_seen = false;
   11598              :                 }
   11599       974572 :               if (bitmap_bit_p (used_in_lanes, i))
   11600       265428 :                 load_seen = true;
   11601              :             }
   11602        77407 :           if (load_seen)
   11603        50902 :             *n_loads += 1;
   11604              :         }
   11605              :     }
   11606              : 
   11607       123890 :   if (dce_chain)
   11608       208689 :     for (unsigned i = 0; i < dr_chain.length (); ++i)
   11609        73735 :       if (!bitmap_bit_p (used_defs, i))
   11610              :         {
   11611        40470 :           tree def = dr_chain[i];
   11612        40844 :           do
   11613              :             {
   11614        40844 :               gimple *stmt = SSA_NAME_DEF_STMT (def);
   11615        40844 :               if (is_gimple_assign (stmt)
   11616        40844 :                   && (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
   11617        40844 :                       || gimple_assign_rhs_code (stmt) == CONSTRUCTOR))
   11618         4987 :                 def = single_ssa_tree_operand (stmt, SSA_OP_USE);
   11619              :               else
   11620              :                 def = NULL;
   11621        40844 :               gimple_stmt_iterator rgsi = gsi_for_stmt (stmt);
   11622        40844 :               gsi_remove (&rgsi, true);
   11623        40844 :               release_defs (stmt);
   11624              :             }
   11625        40844 :           while (def);
   11626              :         }
   11627              : 
   11628              :   return true;
   11629       134954 : }
   11630              : 
   11631              : /* Generate vector permute statements from a list of loads in DR_CHAIN.
   11632              :    If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
   11633              :    permute statements for the SLP node NODE.  Store the number of vector
   11634              :    permute instructions in *N_PERMS and the number of vector load
   11635              :    instructions in *N_LOADS.  If DCE_CHAIN is true, remove all definitions
   11636              :    that were not needed.  */
   11637              : 
   11638              : bool
   11639        96600 : vect_transform_slp_perm_load (vec_info *vinfo,
   11640              :                               slp_tree node, const vec<tree> &dr_chain,
   11641              :                               gimple_stmt_iterator *gsi, poly_uint64 vf,
   11642              :                               bool analyze_only, unsigned *n_perms,
   11643              :                               unsigned int *n_loads, bool dce_chain)
   11644              : {
   11645        96600 :   return vect_transform_slp_perm_load_1 (vinfo, node,
   11646        96600 :                                          SLP_TREE_LOAD_PERMUTATION (node),
   11647              :                                          dr_chain, gsi, vf, analyze_only,
   11648              :                                          dump_enabled_p (), n_perms, n_loads,
   11649        96600 :                                          dce_chain);
   11650              : }
   11651              : 
   11652              : /* Produce the next vector result for SLP permutation NODE by adding a vector
   11653              :    statement at GSI.  If MASK_VEC is nonnull, add:
   11654              : 
   11655              :       <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
   11656              : 
   11657              :    otherwise add:
   11658              : 
   11659              :       <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
   11660              :                                       { N, N+1, N+2, ... }>
   11661              : 
   11662              :    where N == IDENTITY_OFFSET which is either zero or equal to the
   11663              :    number of elements of the result.  */
   11664              : 
   11665              : static void
   11666        32257 : vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
   11667              :                           slp_tree node, tree first_def, tree second_def,
   11668              :                           tree mask_vec, poly_uint64 identity_offset)
   11669              : {
   11670        32257 :   tree vectype = SLP_TREE_VECTYPE (node);
   11671              : 
   11672              :   /* ???  We SLP match existing vector element extracts but
   11673              :      allow punning which we need to re-instantiate at uses
   11674              :      but have no good way of explicitly representing.  */
   11675        32257 :   if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)), TYPE_SIZE (vectype))
   11676        32257 :       && !types_compatible_p (TREE_TYPE (first_def), vectype))
   11677              :     {
   11678           15 :       gassign *conv_stmt
   11679           15 :         = gimple_build_assign (make_ssa_name (vectype),
   11680              :                                build1 (VIEW_CONVERT_EXPR, vectype, first_def));
   11681           15 :       vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
   11682           15 :       first_def = gimple_assign_lhs (conv_stmt);
   11683              :     }
   11684        32257 :   gassign *perm_stmt;
   11685        32257 :   if (mask_vec)
   11686              :     {
   11687        28339 :       if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)),
   11688        28339 :                            TYPE_SIZE (vectype))
   11689        28339 :           && !types_compatible_p (TREE_TYPE (second_def), vectype))
   11690              :         {
   11691            8 :           gassign *conv_stmt
   11692            8 :             = gimple_build_assign (make_ssa_name (vectype),
   11693              :                                    build1 (VIEW_CONVERT_EXPR,
   11694              :                                            vectype, second_def));
   11695            8 :           vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
   11696            8 :           second_def = gimple_assign_lhs (conv_stmt);
   11697              :         }
   11698        28339 :       tree perm_dest = make_ssa_name (vectype);
   11699        28339 :       perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
   11700              :                                        first_def, second_def,
   11701              :                                        mask_vec);
   11702              :     }
   11703              :   else
   11704              :     {
   11705         3918 :       auto def_nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (first_def));
   11706         3918 :       unsigned HOST_WIDE_INT vecno;
   11707         3918 :       poly_uint64 eltno;
   11708         3918 :       if (!can_div_trunc_p (poly_uint64 (identity_offset), def_nunits,
   11709              :                             &vecno, &eltno))
   11710              :         gcc_unreachable ();
   11711         3918 :       tree def = vecno & 1 ? second_def : first_def;
   11712         3918 :       if (!types_compatible_p (TREE_TYPE (def), vectype))
   11713              :         {
   11714              :           /* For identity permutes we still need to handle the case
   11715              :              of offsetted extracts or concats.  */
   11716          370 :           tree perm_dest = make_ssa_name (vectype);
   11717          370 :           unsigned HOST_WIDE_INT c;
   11718          370 :           if (known_le (TYPE_VECTOR_SUBPARTS (vectype), def_nunits))
   11719              :             {
   11720          366 :               unsigned HOST_WIDE_INT elsz
   11721          366 :                 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
   11722          732 :               tree lowpart = build3 (BIT_FIELD_REF, vectype, def,
   11723          366 :                                      TYPE_SIZE (vectype),
   11724          366 :                                      bitsize_int (eltno * elsz));
   11725          366 :               perm_stmt = gimple_build_assign (perm_dest, lowpart);
   11726              :             }
   11727            4 :           else if (constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
   11728            4 :                                         def_nunits, &c) && c == 2)
   11729              :             {
   11730            4 :               gcc_assert (known_eq (identity_offset, 0U));
   11731            4 :               tree ctor = build_constructor_va (vectype, 2,
   11732              :                                                 NULL_TREE, first_def,
   11733              :                                                 NULL_TREE, second_def);
   11734            4 :               perm_stmt = gimple_build_assign (perm_dest, ctor);
   11735              :             }
   11736              :           else
   11737            0 :             gcc_unreachable ();
   11738              :         }
   11739              :       else
   11740              :         {
   11741         3548 :           gcc_assert (known_eq (eltno, 0U));
   11742         3548 :           node->push_vec_def (def);
   11743         3548 :           return;
   11744              :         }
   11745              :     }
   11746        28709 :   vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi);
   11747              :   /* Store the vector statement in NODE.  */
   11748        28709 :   node->push_vec_def (perm_stmt);
   11749              : }
   11750              : 
   11751              : /* Subroutine of vectorizable_slp_permutation.  Check whether the target
   11752              :    can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
   11753              :    If GSI is nonnull, emit the permutation there.
   11754              : 
   11755              :    When GSI is null, the only purpose of NODE is to give properties
   11756              :    of the result, such as the vector type and number of SLP lanes.
   11757              :    The node does not need to be a VEC_PERM_EXPR.
   11758              : 
   11759              :    If the target supports the operation, return the number of individual
   11760              :    VEC_PERM_EXPRs needed, otherwise return -1.  Print information to the
   11761              :    dump file if DUMP_P is true.  */
   11762              : 
   11763              : static int
   11764       473095 : vectorizable_slp_permutation_1 (vec_info *vinfo, gimple_stmt_iterator *gsi,
   11765              :                                 slp_tree node, lane_permutation_t &perm,
   11766              :                                 vec<slp_tree> &children, bool dump_p)
   11767              : {
   11768       473095 :   tree vectype = SLP_TREE_VECTYPE (node);
   11769              : 
   11770              :   /* ???  We currently only support all same vector input types
   11771              :      while the SLP IL should really do a concat + select and thus accept
   11772              :      arbitrary mismatches.  */
   11773       473095 :   slp_tree child;
   11774       473095 :   unsigned i;
   11775       473095 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
   11776       473095 :   bool repeating_p = multiple_p (nunits, SLP_TREE_LANES (node));
   11777              :   /* True if we're permuting a single input of 2N vectors down
   11778              :      to N vectors.  This case doesn't generalize beyond 2 since
   11779              :      VEC_PERM_EXPR only takes 2 inputs.  */
   11780       473095 :   bool pack_p = false;
   11781              :   /* If we're permuting inputs of N vectors each into X*N outputs,
   11782              :      this is the value of X, otherwise it is 1.  */
   11783       473095 :   unsigned int unpack_factor = 1;
   11784       473095 :   tree op_vectype = NULL_TREE;
   11785       474699 :   FOR_EACH_VEC_ELT (children, i, child)
   11786       474599 :     if (SLP_TREE_VECTYPE (child))
   11787              :       {
   11788              :         op_vectype = SLP_TREE_VECTYPE (child);
   11789              :         break;
   11790              :       }
   11791       473095 :   if (!op_vectype)
   11792          100 :     op_vectype = vectype;
   11793       996757 :   FOR_EACH_VEC_ELT (children, i, child)
   11794              :     {
   11795       523662 :       if ((SLP_TREE_DEF_TYPE (child) != vect_internal_def
   11796        18783 :            && !vect_maybe_update_slp_op_vectype (child, op_vectype))
   11797       523662 :           || !types_compatible_p (SLP_TREE_VECTYPE (child), op_vectype)
   11798      1047324 :           || !types_compatible_p (TREE_TYPE (vectype), TREE_TYPE (op_vectype)))
   11799              :         {
   11800            0 :           if (dump_p)
   11801            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   11802              :                              "Unsupported vector types in lane permutation\n");
   11803       473095 :           return -1;
   11804              :         }
   11805       523662 :       auto op_nunits = TYPE_VECTOR_SUBPARTS (op_vectype);
   11806       523662 :       unsigned int this_unpack_factor;
   11807              :       /* Detect permutations of external, pre-existing vectors.  The external
   11808              :          node's SLP_TREE_LANES stores the total number of units in the vector,
   11809              :          or zero if the vector has variable length.
   11810              : 
   11811              :          We are expected to keep the original VEC_PERM_EXPR for such cases.
   11812              :          There is no repetition to model.  */
   11813       523662 :       if (SLP_TREE_DEF_TYPE (child) == vect_external_def
   11814       523662 :           && SLP_TREE_SCALAR_OPS (child).is_empty ())
   11815              :         repeating_p = false;
   11816              :       /* Check whether the input has twice as many lanes per vector.  */
   11817       507623 :       else if (children.length () == 1
   11818       507623 :                && known_eq (SLP_TREE_LANES (child) * nunits,
   11819              :                             SLP_TREE_LANES (node) * op_nunits * 2))
   11820              :         pack_p = true;
   11821              :       /* Check whether the output has N times as many lanes per vector.  */
   11822       523662 :       else if (constant_multiple_p (SLP_TREE_LANES (node) * op_nunits,
   11823       463351 :                                     SLP_TREE_LANES (child) * nunits,
   11824              :                                     &this_unpack_factor)
   11825       428428 :                && (i == 0 || unpack_factor == this_unpack_factor))
   11826              :         unpack_factor = this_unpack_factor;
   11827              :       else
   11828              :         repeating_p = false;
   11829              :     }
   11830              : 
   11831       946190 :   gcc_assert (perm.length () == SLP_TREE_LANES (node));
   11832              : 
   11833              :   /* Load-lanes permute.  This permute only acts as a forwarder to
   11834              :      select the correct vector def of the load-lanes load which
   11835              :      has the permuted vectors in its vector defs like
   11836              :      { v0, w0, r0, v1, w1, r1 ... } for a ld3.  All costs are
   11837              :      accounted for in the costing for the actual load so we
   11838              :      return zero here.  */
   11839       473095 :   if (node->ldst_lanes)
   11840              :     {
   11841            0 :       gcc_assert (children.length () == 1);
   11842            0 :       if (!gsi)
   11843              :         /* This is a trivial op always supported.  */
   11844              :         return 0;
   11845            0 :       slp_tree child = children[0];
   11846            0 :       unsigned vec_idx = (SLP_TREE_LANE_PERMUTATION (node)[0].second
   11847            0 :                           / SLP_TREE_LANES (node));
   11848            0 :       unsigned vec_num = SLP_TREE_LANES (child) / SLP_TREE_LANES (node);
   11849            0 :       unsigned nvectors = vect_get_num_copies (vinfo, node);
   11850            0 :       for (unsigned i = 0; i < nvectors; ++i)
   11851              :         {
   11852            0 :           tree def = SLP_TREE_VEC_DEFS (child)[i * vec_num  + vec_idx];
   11853            0 :           node->push_vec_def (def);
   11854              :         }
   11855              :       return 0;
   11856              :     }
   11857              : 
   11858              :   /* Set REPEATING_P to true if the permutations are cyclical wrt UNPACK_FACTOR
   11859              :      and if we can generate the vectors in a vector-length agnostic way.
   11860              :      This requires UNPACK_STEP == NUNITS / UNPACK_FACTOR to be known at
   11861              :      compile time.
   11862              : 
   11863              :      The significance of UNPACK_STEP is that, when PACK_P is false,
   11864              :      output vector I operates on a window of UNPACK_STEP elements from each
   11865              :      input, starting at lane UNPACK_STEP * (I % UNPACK_FACTOR).  For example,
   11866              :      when UNPACK_FACTOR is 2, the first output vector operates on lanes
   11867              :      [0, NUNITS / 2 - 1] of each input vector and the second output vector
   11868              :      operates on lanes [NUNITS / 2, NUNITS - 1] of each input vector.
   11869              : 
   11870              :      When REPEATING_P is true, NOUTPUTS holds the total number of outputs
   11871              :      that we actually need to generate.  */
   11872       473095 :   uint64_t noutputs = 0;
   11873       473095 :   poly_uint64 unpack_step = 0;
   11874       473095 :   loop_vec_info linfo = dyn_cast <loop_vec_info> (vinfo);
   11875       210147 :   if (!linfo
   11876       512206 :       || !multiple_p (nunits, unpack_factor, &unpack_step)
   11877       209207 :       || !constant_multiple_p (LOOP_VINFO_VECT_FACTOR (linfo)
   11878       209207 :                                * SLP_TREE_LANES (node), nunits, &noutputs))
   11879              :     repeating_p = false;
   11880              : 
   11881              :   /* We can handle the conditions described for REPEATING_P above for
   11882              :      both variable- and constant-length vectors.  The fallback requires
   11883              :      us to generate every element of every permute vector explicitly,
   11884              :      which is only possible for constant-length permute vectors.
   11885              : 
   11886              :      Set:
   11887              : 
   11888              :      - NPATTERNS and NELTS_PER_PATTERN to the encoding of the permute
   11889              :        mask vectors that we want to build.
   11890              : 
   11891              :      - NCOPIES to the number of copies of PERM that we need in order
   11892              :        to build the necessary permute mask vectors.  */
   11893       209207 :   uint64_t npatterns;
   11894       209207 :   unsigned nelts_per_pattern;
   11895       209207 :   uint64_t ncopies;
   11896       209207 :   if (repeating_p)
   11897              :     {
   11898              :       /* We need permute mask vectors that have the form:
   11899              : 
   11900              :            { X1, ..., Xn, X1 + n, ..., Xn + n, X1 + 2n, ..., Xn + 2n, ... }
   11901              : 
   11902              :          In other words, the original n-element permute in PERM is
   11903              :          "unrolled" to fill a full vector.  The stepped vector encoding
   11904              :          that we use for permutes requires 3n elements.  */
   11905       170096 :       npatterns = SLP_TREE_LANES (node);
   11906       170096 :       nelts_per_pattern = ncopies = 3;
   11907              :     }
   11908              :   else
   11909              :     {
   11910              :       /* Calculate every element of every permute mask vector explicitly,
   11911              :          instead of relying on the pattern described above.  */
   11912       302999 :       if (!nunits.is_constant (&npatterns)
   11913       302999 :           || !TYPE_VECTOR_SUBPARTS (op_vectype).is_constant ())
   11914              :         {
   11915              :           if (dump_p)
   11916              :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   11917              :                              "unsupported permutation %p on variable-length"
   11918              :                              " vectors\n", (void *) node);
   11919              :           return -1;
   11920              :         }
   11921       302999 :       nelts_per_pattern = ncopies = 1;
   11922       302999 :       if (linfo && !LOOP_VINFO_VECT_FACTOR (linfo).is_constant (&ncopies))
   11923              :         {
   11924              :           if (dump_p)
   11925              :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   11926              :                              "unsupported permutation %p for variable VF\n",
   11927              :                              (void *) node);
   11928              :           return -1;
   11929              :         }
   11930              :       pack_p = false;
   11931              :       unpack_factor = 1;
   11932              :     }
   11933       473095 :   unsigned olanes = unpack_factor * ncopies * SLP_TREE_LANES (node);
   11934       473095 :   gcc_assert (repeating_p || multiple_p (olanes, nunits));
   11935              : 
   11936              :   /* Compute the { { SLP operand, vector index}, lane } permutation sequence
   11937              :      from the { SLP operand, scalar lane } permutation as recorded in the
   11938              :      SLP node as intermediate step.  This part should already work
   11939              :      with SLP children with arbitrary number of lanes.  */
   11940       473095 :   auto_vec<std::pair<std::pair<unsigned, unsigned>, poly_uint64>> vperm;
   11941       473095 :   auto_vec<poly_uint64> active_lane;
   11942       473095 :   vperm.create (olanes);
   11943       473095 :   active_lane.safe_grow_cleared (children.length (), true);
   11944      1427829 :   for (unsigned int ui = 0; ui < unpack_factor; ++ui)
   11945              :     {
   11946      2044730 :       for (unsigned j = 0; j < children.length (); ++j)
   11947       540726 :         active_lane[j] = ui * unpack_step;
   11948      1418310 :       for (unsigned i = 0; i < ncopies; ++i)
   11949              :         {
   11950      5887130 :           for (unsigned pi = 0; pi < perm.length (); ++pi)
   11951              :             {
   11952      2006894 :               std::pair<unsigned, unsigned> p = perm[pi];
   11953      2006894 :               tree vtype = SLP_TREE_VECTYPE (children[p.first]);
   11954      2006894 :               if (repeating_p)
   11955       999150 :                 vperm.quick_push ({{p.first, 0},
   11956       999150 :                                    p.second + active_lane[p.first]});
   11957              :               else
   11958              :                 {
   11959              :                   /* We checked above that the vectors are constant-length.  */
   11960      1007744 :                   unsigned vnunits = TYPE_VECTOR_SUBPARTS (vtype)
   11961      1007744 :                     .to_constant ();
   11962      1007744 :                   unsigned lane = active_lane[p.first].to_constant ();
   11963      1007744 :                   unsigned vi = (lane + p.second) / vnunits;
   11964      1007744 :                   unsigned vl = (lane + p.second) % vnunits;
   11965      1007744 :                   vperm.quick_push ({{p.first, vi}, vl});
   11966              :                 }
   11967              :             }
   11968              :           /* Advance to the next group.  */
   11969      2008896 :           for (unsigned j = 0; j < children.length (); ++j)
   11970      1072225 :             active_lane[j] += SLP_TREE_LANES (children[j]);
   11971              :         }
   11972              :     }
   11973              : 
   11974       473095 :   if (dump_p)
   11975              :     {
   11976         9035 :       dump_printf_loc (MSG_NOTE, vect_location,
   11977              :                        "vectorizing permutation %p", (void *)node);
   11978        32694 :       for (unsigned i = 0; i < perm.length (); ++i)
   11979        23659 :         dump_printf (MSG_NOTE, " op%u[%u]", perm[i].first, perm[i].second);
   11980         9035 :       if (repeating_p)
   11981         7574 :         dump_printf (MSG_NOTE, " (repeat %d)", SLP_TREE_LANES (node));
   11982         9035 :       dump_printf (MSG_NOTE, "\n");
   11983         9035 :       dump_printf_loc (MSG_NOTE, vect_location, "as");
   11984        90774 :       for (unsigned i = 0; i < vperm.length (); ++i)
   11985              :         {
   11986        81739 :           if (i != 0
   11987        81739 :               && (repeating_p
   11988        55351 :                   ? multiple_p (i, npatterns)
   11989        81739 :                   : multiple_p (i, TYPE_VECTOR_SUBPARTS (vectype))))
   11990        24385 :             dump_printf (MSG_NOTE, ",");
   11991        81739 :           dump_printf (MSG_NOTE, " vops%u[%u][",
   11992        81739 :                        vperm[i].first.first, vperm[i].first.second);
   11993        81739 :           dump_dec (MSG_NOTE, vperm[i].second);
   11994        81739 :           dump_printf (MSG_NOTE, "]");
   11995              :         }
   11996         9035 :       dump_printf (MSG_NOTE, "\n");
   11997              :     }
   11998              : 
   11999              :   /* We can only handle two-vector permutes, everything else should
   12000              :      be lowered on the SLP level.  The following is closely inspired
   12001              :      by vect_transform_slp_perm_load and is supposed to eventually
   12002              :      replace it.
   12003              :      ???   As intermediate step do code-gen in the SLP tree representation
   12004              :      somehow?  */
   12005       473095 :   std::pair<unsigned, unsigned> first_vec = std::make_pair (-1U, -1U);
   12006       473095 :   std::pair<unsigned, unsigned> second_vec = std::make_pair (-1U, -1U);
   12007       473095 :   unsigned int index = 0;
   12008       473095 :   poly_uint64 mask_element;
   12009       473095 :   vec_perm_builder mask;
   12010       473095 :   mask.new_vector (nunits, npatterns, nelts_per_pattern);
   12011       473095 :   unsigned int count = mask.encoded_nelts ();
   12012       473095 :   mask.quick_grow (count);
   12013       473095 :   vec_perm_indices indices;
   12014       473095 :   unsigned nperms = 0;
   12015              :   /* When REPEATING_P is true, we only have UNPACK_FACTOR unique permute
   12016              :      vectors to check during analysis, but we need to generate NOUTPUTS
   12017              :      vectors during transformation.  */
   12018       473095 :   unsigned total_nelts = olanes;
   12019       473095 :   unsigned process_nelts = olanes;
   12020       473095 :   if (repeating_p)
   12021              :     {
   12022       170096 :       total_nelts = (total_nelts / unpack_factor) * noutputs;
   12023       170096 :       if (gsi)
   12024         9986 :         process_nelts = total_nelts;
   12025              :     }
   12026       473095 :   unsigned last_ei = (total_nelts - 1) % process_nelts;
   12027      2484307 :   for (unsigned i = 0; i < process_nelts; ++i)
   12028              :     {
   12029              :       /* VI is the input vector index when generating code for REPEATING_P.  */
   12030      2021407 :       unsigned vi = i / olanes * (pack_p ? 2 : 1);
   12031      2021407 :       unsigned ei = i % olanes;
   12032      2021407 :       mask_element = vperm[ei].second;
   12033      2021407 :       if (pack_p)
   12034              :         {
   12035              :           /* In this case, we have N outputs and the single child provides 2N
   12036              :              inputs.  Output X permutes inputs 2X and 2X+1.
   12037              : 
   12038              :              The mask indices are taken directly from the SLP permutation node.
   12039              :              Index X selects from the first vector if (X / NUNITS) % 2 == 0;
   12040              :              X selects from the second vector otherwise.  These conditions
   12041              :              are only known at compile time for constant-length vectors.  */
   12042              :           first_vec = std::make_pair (0, 0);
   12043              :           second_vec = std::make_pair (0, 1);
   12044              :         }
   12045      1850560 :       else if (first_vec.first == -1U
   12046      1850560 :                || first_vec == vperm[ei].first)
   12047      1636454 :         first_vec = vperm[ei].first;
   12048       214106 :       else if (second_vec.first == -1U
   12049       214106 :                || second_vec == vperm[ei].first)
   12050              :         {
   12051       213698 :           second_vec = vperm[ei].first;
   12052       213698 :           mask_element += nunits;
   12053              :         }
   12054              :       else
   12055              :         {
   12056          408 :           if (dump_p)
   12057            7 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12058              :                              "permutation requires at "
   12059              :                              "least three vectors\n");
   12060          408 :           gcc_assert (!gsi);
   12061              :           return -1;
   12062              :         }
   12063              : 
   12064      2020999 :       mask[index++] = mask_element;
   12065              : 
   12066      2020999 :       if (index == count)
   12067              :         {
   12068       617779 :           indices.new_vector (mask, second_vec.first == -1U ? 1 : 2,
   12069              :                               TYPE_VECTOR_SUBPARTS (op_vectype));
   12070       617779 :           bool identity_p = (indices.series_p (0, 1, mask[0], 1)
   12071       956411 :                              && constant_multiple_p (mask[0], nunits));
   12072       617779 :           machine_mode vmode = TYPE_MODE (vectype);
   12073       617779 :           machine_mode op_vmode = TYPE_MODE (op_vectype);
   12074       617779 :           unsigned HOST_WIDE_INT c;
   12075       617779 :           if ((!identity_p
   12076       568386 :                && !can_vec_perm_const_p (vmode, op_vmode, indices))
   12077       617779 :               || (identity_p
   12078        49393 :                   && !known_le (nunits,
   12079              :                                 TYPE_VECTOR_SUBPARTS (op_vectype))
   12080            8 :                   && (!constant_multiple_p (nunits,
   12081         9795 :                                             TYPE_VECTOR_SUBPARTS (op_vectype),
   12082            8 :                                             &c) || c != 2)))
   12083              :             {
   12084         9787 :               if (dump_p)
   12085              :                 {
   12086          154 :                   dump_printf_loc (MSG_MISSED_OPTIMIZATION,
   12087              :                                    vect_location,
   12088              :                                    "unsupported vect permute { ");
   12089         1596 :                   for (i = 0; i < count; ++i)
   12090              :                     {
   12091         1442 :                       dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
   12092         1442 :                       dump_printf (MSG_MISSED_OPTIMIZATION, " ");
   12093              :                     }
   12094          154 :                   dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
   12095              :                 }
   12096         9787 :               gcc_assert (!gsi);
   12097       473095 :               return -1;
   12098              :             }
   12099              : 
   12100       607992 :           if (!identity_p)
   12101       558599 :             nperms += CEIL (total_nelts, process_nelts) - (ei > last_ei);
   12102       607992 :           if (gsi)
   12103              :             {
   12104        32257 :               if (second_vec.first == -1U)
   12105         7760 :                 second_vec = first_vec;
   12106              : 
   12107        32257 :               slp_tree
   12108        32257 :                 first_node = children[first_vec.first],
   12109        32257 :                 second_node = children[second_vec.first];
   12110              : 
   12111        32257 :               tree mask_vec = NULL_TREE;
   12112        32257 :               if (!identity_p)
   12113        28339 :                 mask_vec = vect_gen_perm_mask_checked (vectype, indices);
   12114              : 
   12115        32257 :               tree first_def
   12116        32257 :                 = vect_get_slp_vect_def (first_node, first_vec.second + vi);
   12117        32257 :               tree second_def
   12118        32257 :                 = vect_get_slp_vect_def (second_node, second_vec.second + vi);
   12119        32257 :               vect_add_slp_permutation (vinfo, gsi, node, first_def,
   12120        32257 :                                         second_def, mask_vec, mask[0]);
   12121              :             }
   12122              : 
   12123              :           index = 0;
   12124              :           first_vec = std::make_pair (-1U, -1U);
   12125              :           second_vec = std::make_pair (-1U, -1U);
   12126              :         }
   12127              :     }
   12128              : 
   12129       462900 :   return nperms;
   12130       473095 : }
   12131              : 
   12132              : /* Vectorize the SLP permutations in NODE as specified
   12133              :    in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
   12134              :    child number and lane number.
   12135              :    Interleaving of two two-lane two-child SLP subtrees (not supported):
   12136              :      [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
   12137              :    A blend of two four-lane two-child SLP subtrees:
   12138              :      [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
   12139              :    Highpart of a four-lane one-child SLP subtree (not supported):
   12140              :      [ { 0, 2 }, { 0, 3 } ]
   12141              :    Where currently only a subset is supported by code generating below.  */
   12142              : 
   12143              : bool
   12144       100075 : vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
   12145              :                               slp_tree node, stmt_vector_for_cost *cost_vec)
   12146              : {
   12147       100075 :   tree vectype = SLP_TREE_VECTYPE (node);
   12148       100075 :   lane_permutation_t &perm = SLP_TREE_LANE_PERMUTATION (node);
   12149       100075 :   int nperms = vectorizable_slp_permutation_1 (vinfo, gsi, node, perm,
   12150       100075 :                                                SLP_TREE_CHILDREN (node),
   12151              :                                                dump_enabled_p ());
   12152       100075 :   if (nperms < 0)
   12153              :     return false;
   12154              : 
   12155        98454 :   if (!gsi && nperms != 0)
   12156        74610 :     record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
   12157              : 
   12158              :   return true;
   12159              : }
   12160              : 
   12161              : /* Vectorize SLP NODE.  Only compute the vector insertion places when
   12162              :    PLACE_ONLY is true.  When placing, return false if there is no possible
   12163              :    schedule.  */
   12164              : 
   12165              : static bool
   12166      3144981 : vect_schedule_slp_node (vec_info *vinfo,
   12167              :                         slp_tree node, slp_instance instance, bool place_only)
   12168              : {
   12169      3144981 :   int i;
   12170      3144981 :   slp_tree child;
   12171              : 
   12172              :   /* Vectorize externals and constants.  */
   12173      3144981 :   if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
   12174      3144981 :       || SLP_TREE_DEF_TYPE (node) == vect_external_def)
   12175              :     {
   12176      1215091 :       if (place_only)
   12177              :         {
   12178       703498 :           if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
   12179              :             return true;
   12180       320421 :           gimple *last_stmt = NULL;
   12181       320421 :           vec<tree> &defs = (!SLP_TREE_SCALAR_OPS (node).is_empty ()
   12182              :                              ? SLP_TREE_SCALAR_OPS (node)
   12183       320421 :                              : SLP_TREE_VEC_DEFS (node));
   12184      1715528 :           for (tree def : defs)
   12185              :             /* If the stmt is not inside the region do not
   12186              :                use it as possible insertion point.  */
   12187       754265 :             if (auto stmt_info = vinfo->lookup_def (def))
   12188              :               {
   12189       393034 :                 gimple *stmt = stmt_info->stmt;
   12190       393034 :                 if (!last_stmt)
   12191              :                   last_stmt = stmt;
   12192       221441 :                 else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
   12193              :                   last_stmt = stmt;
   12194        67639 :                 else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
   12195              :                   ;
   12196              :                 else
   12197            0 :                   gcc_unreachable ();
   12198              :               }
   12199       320421 :           node->si = last_stmt;
   12200       320421 :           return true;
   12201              :         }
   12202              : 
   12203              :       /* ???  vectorizable_shift can end up using a scalar operand which is
   12204              :          currently denoted as !SLP_TREE_VECTYPE.  No need to vectorize the
   12205              :          node in this case.  */
   12206       511593 :       if (!SLP_TREE_VECTYPE (node))
   12207              :         return true;
   12208              : 
   12209              :       /* There are two reasons vector defs might already exist.  The first
   12210              :          is that we are vectorizing an existing vector def.  The second is
   12211              :          when performing BB vectorization shared constant/external nodes
   12212              :          are not split apart during partitioning so during the code-gen
   12213              :          DFS walk we can end up visiting them twice.  */
   12214       503916 :       if (! SLP_TREE_VEC_DEFS (node).exists ())
   12215       503059 :         vect_create_constant_vectors (vinfo, node);
   12216              :       return true;
   12217              :     }
   12218              : 
   12219      1929890 :   stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
   12220              : 
   12221      1929890 :   gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ());
   12222      1929890 :   if (!place_only && SLP_TREE_VECTYPE (node))
   12223       976988 :     SLP_TREE_VEC_DEFS (node).create (vect_get_num_copies (vinfo, node));
   12224              : 
   12225      1929890 :   gimple *last_stmt;
   12226      1929890 :   gimple_stmt_iterator si;
   12227              :   /* ???  When !place_only we'd like to re-use place_only computed info,
   12228              :      but this is a bit awkward due to using gsi_insert_before and the
   12229              :      requirement to insert after vector defs.  So we compute last_stmt
   12230              :      during pre-scheduling and si during scheduling.  */
   12231      1929890 :   if (!SLP_TREE_PERMUTE_P (node) && STMT_VINFO_DATA_REF (stmt_info))
   12232              :     {
   12233              :       /* Vectorized loads go before the first scalar load to make it
   12234              :          ready early, vectorized stores go before the last scalar
   12235              :          stmt which is where all uses are ready.
   12236              :          In theory, if we delay dependence checking until after
   12237              :          placing, we can schedule at other points, but then
   12238              :          dependence checking would need to honor that.  On the
   12239              :          other hand dependence checking could request a different
   12240              :          scheduling point as well, if dependences require that.  */
   12241      1494206 :       stmt_vec_info last_stmt_info = NULL;
   12242      1494206 :       if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
   12243       276382 :         last_stmt_info = vect_find_first_scalar_stmt_in_slp (node);
   12244              :       else /* DR_IS_WRITE */
   12245              :         {
   12246      1217824 :           last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
   12247      1217824 :           if (place_only)
   12248      1334620 :             FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
   12249              :               {
   12250       667312 :                 if (child->si
   12251       667312 :                     && !vect_stmt_dominates_stmt_p (child->si,
   12252              :                                                     last_stmt_info->stmt))
   12253              :                   return false;
   12254              :               }
   12255              :         }
   12256      1494202 :       last_stmt = last_stmt_info->stmt;
   12257      1494202 :       si = gsi_for_stmt (last_stmt);
   12258      1494202 :     }
   12259       435684 :   else if (!SLP_TREE_PERMUTE_P (node)
   12260       413950 :            && (SLP_TREE_TYPE (node) == cycle_phi_info_type
   12261              :                || SLP_TREE_TYPE (node) == induc_vec_info_type
   12262              :                || SLP_TREE_TYPE (node) == phi_info_type))
   12263              :     {
   12264              :       /* For PHI node vectorization we do not use the insertion iterator.  */
   12265       107322 :       last_stmt = SLP_TREE_SCALAR_STMTS (node)[0]->stmt;
   12266       107322 :       if (place_only)
   12267       184225 :         FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
   12268              :           {
   12269       130306 :             if (child->si
   12270       227279 :                 && !dominated_by_p (CDI_DOMINATORS,
   12271              :                                     gimple_phi_arg_edge
   12272        96973 :                                       (as_a <gphi *> (last_stmt), i)->src,
   12273        96973 :                                     gimple_bb (child->si)))
   12274              :               return false;
   12275              :           }
   12276       107320 :       si = gsi_none ();
   12277              :     }
   12278              :   else
   12279              :     {
   12280              :       /* Emit other stmts after the children vectorized defs which is
   12281              :          earliest possible.  */
   12282              :       last_stmt = NULL;
   12283       910051 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
   12284       581691 :         if (place_only)
   12285              :           {
   12286       197724 :             gimple *vstmt = child->si;
   12287       197724 :             if (!vstmt)
   12288              :               {
   12289              :                 /* vect_constant_def and defs at region boundary do not
   12290              :                    constrain placement.  */
   12291              :                 gcc_assert (SLP_TREE_DEF_TYPE (child) == vect_constant_def
   12292              :                             /* ???  Region boundary is not representated
   12293              :                                by a NULL stmt.  */
   12294              :                             || true);
   12295              :               }
   12296       154577 :             else if (!last_stmt)
   12297              :               last_stmt = vstmt;
   12298        44667 :             else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
   12299              :               last_stmt = vstmt;
   12300        11194 :             else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
   12301              :               ;
   12302              :             else
   12303              :               /* Non-trapping stmts from different BBs might be combined,
   12304              :                  and if we later CSE a low/high part we can run into this.  */
   12305              :               return false;
   12306              :           }
   12307       383967 :         else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
   12308              :           {
   12309              :             /* For fold-left reductions we are retaining the scalar
   12310              :                reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS
   12311              :                set so the representation isn't perfect.  Resort to the
   12312              :                last scalar def here.  */
   12313       306147 :             if (SLP_TREE_VEC_DEFS (child).is_empty ())
   12314              :               {
   12315         1006 :                 gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
   12316         1006 :                 gphi *phi = as_a <gphi *>
   12317         1006 :                               (vect_find_last_scalar_stmt_in_slp (child)->stmt);
   12318         1006 :                 if (!last_stmt)
   12319              :                   last_stmt = phi;
   12320          756 :                 else if (vect_stmt_dominates_stmt_p (last_stmt, phi))
   12321              :                   last_stmt = phi;
   12322          745 :                 else if (vect_stmt_dominates_stmt_p (phi, last_stmt))
   12323              :                   ;
   12324              :                 else
   12325            0 :                   gcc_unreachable ();
   12326              :               }
   12327              :             /* We are emitting all vectorized stmts in the same place and
   12328              :                the last one is the last.
   12329              :                ???  Unless we have a load permutation applied and that
   12330              :                figures to re-use an earlier generated load.  */
   12331       306147 :             unsigned j;
   12332       306147 :             tree vdef;
   12333       720754 :             FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
   12334       414607 :               if (TREE_CODE (vdef) == SSA_NAME
   12335       414607 :                   && !SSA_NAME_IS_DEFAULT_DEF (vdef))
   12336              :                 {
   12337       414555 :                   gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
   12338              :                   /* Ignore existing external vector defs so we insert
   12339              :                      at region boundary.  Those can leak to an
   12340              :                      internal identity permute node.  */
   12341       414555 :                   if (is_a <bb_vec_info> (vinfo)
   12342        30568 :                       && vinfo->bbs[0] != gimple_bb (vstmt)
   12343       435145 :                       && dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
   12344        20590 :                                          gimple_bb (vstmt)))
   12345              :                     ;
   12346       414541 :                   else if (!last_stmt)
   12347              :                     last_stmt = vstmt;
   12348       210624 :                   else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
   12349              :                     last_stmt = vstmt;
   12350        47964 :                   else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
   12351              :                     ;
   12352              :                   else
   12353            0 :                     gcc_unreachable ();
   12354              :                 }
   12355              :           }
   12356        77820 :         else if (!SLP_TREE_VECTYPE (child))
   12357              :           {
   12358              :             /* For externals we use unvectorized at all scalar defs.  */
   12359              :             unsigned j;
   12360              :             tree def;
   12361        16524 :             FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (child), j, def)
   12362              :               /* If the stmt is not inside the region do not
   12363              :                  use it as possible insertion point.  */
   12364         9477 :               if (auto stmt_info = vinfo->lookup_def (def))
   12365              :                 {
   12366          138 :                   gimple *stmt = stmt_info->stmt;
   12367          138 :                   if (!last_stmt)
   12368              :                     last_stmt = stmt;
   12369          118 :                   else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
   12370              :                     last_stmt = stmt;
   12371           26 :                   else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
   12372              :                     ;
   12373              :                   else
   12374            0 :                     gcc_unreachable ();
   12375              :                 }
   12376              :           }
   12377              :         else
   12378              :           {
   12379              :             /* For externals we have to look at all defs since their
   12380              :                insertion place is decided per vector.  But beware
   12381              :                of pre-existing vectors where we need to make sure
   12382              :                we do not insert before the region boundary.  */
   12383       141403 :             if (SLP_TREE_SCALAR_OPS (child).is_empty ()
   12384          620 :                 && !vinfo->lookup_def (SLP_TREE_VEC_DEFS (child)[0]))
   12385              :               ;
   12386              :             else
   12387              :               {
   12388              :                 unsigned j;
   12389              :                 tree vdef;
   12390       750811 :                 FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
   12391        98492 :                   if (TREE_CODE (vdef) == SSA_NAME
   12392        98492 :                       && !SSA_NAME_IS_DEFAULT_DEF (vdef))
   12393              :                     {
   12394        20811 :                       gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
   12395        20811 :                       if (!last_stmt)
   12396              :                         last_stmt = vstmt;
   12397        11356 :                       else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
   12398              :                         last_stmt = vstmt;
   12399         9014 :                       else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
   12400              :                         ;
   12401              :                       else
   12402            0 :                         gcc_unreachable ();
   12403              :                     }
   12404              :               }
   12405              :           }
   12406              : 
   12407              :       /* We split regions to vectorize at control altering stmts
   12408              :          with a definition so this can only be an external.  */
   12409       328360 :       gcc_checking_assert (!last_stmt
   12410              :                            || !is_ctrl_altering_stmt (last_stmt));
   12411              : 
   12412       328360 :       if (is_a <bb_vec_info> (vinfo)
   12413       133212 :           && !SLP_TREE_PERMUTE_P (node)
   12414       126711 :           && (!last_stmt
   12415       123811 :               || !dominated_by_p (CDI_DOMINATORS, gimple_bb (last_stmt),
   12416       123811 :                                   gimple_bb (stmt_info->stmt)))
   12417       353944 :           && gimple_could_trap_p (stmt_info->stmt))
   12418              :         {
   12419              :           /* We've constrained possibly trapping operations to all come
   12420              :              from the same basic-block, if vectorized defs would allow earlier
   12421              :              scheduling still force vectorized stmts to the original block.
   12422              :              This is only necessary for BB vectorization since for loop vect
   12423              :              all operations are in a single BB and scalar stmt based
   12424              :              placement doesn't play well with epilogue vectorization.  */
   12425          686 :           if (last_stmt
   12426         1272 :               && !dominated_by_p (CDI_DOMINATORS,
   12427          586 :                                   gimple_bb (stmt_info->stmt),
   12428          586 :                                   gimple_bb (last_stmt)))
   12429              :             {
   12430            4 :               gcc_assert (place_only);
   12431              :               return false;
   12432              :             }
   12433          682 :           si = gsi_after_labels (gimple_bb (stmt_info->stmt));
   12434          682 :           last_stmt = gsi_stmt (si);
   12435              :         }
   12436              :       /* When there is no in-region child def to guide placement, insert
   12437              :          at region boundary.  */
   12438       327674 :       else if (!last_stmt)
   12439              :         {
   12440         4710 :           si = gsi_after_labels (vinfo->bbs[0]);
   12441              :           /* last_stmt NULL marks the region start.  */
   12442              :         }
   12443       322964 :       else if (is_a <gphi *> (last_stmt))
   12444        31939 :         si = gsi_after_labels (gimple_bb (last_stmt));
   12445              :       else
   12446              :         {
   12447       291025 :           si = gsi_for_stmt (last_stmt);
   12448              :           /* We use gsi_insert_before, so when last_stmt is a vector
   12449              :              def we have to advance (or use gsi_insert_after).  */
   12450       291025 :           gsi_next (&si);
   12451              : 
   12452       291025 :           if (auto loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
   12453              :             {
   12454              :               /* We'll have to fix this up for loop vect.  */
   12455       178761 :               gcc_assert (!place_only);
   12456              :               /* Avoid scheduling stmts to random places in the CFG, any
   12457              :                  stmt dominance check we performed is possibly wrong as UIDs
   12458              :                  are not initialized for all of the function for loop
   12459              :                  vectorization.  Instead append to the loop preheader.  */
   12460       178761 :               if ((LOOP_VINFO_LOOP (loop_vinfo)->header
   12461       178761 :                    != gimple_bb (last_stmt))
   12462       181664 :                   && dominated_by_p (CDI_DOMINATORS,
   12463              :                                      LOOP_VINFO_LOOP (loop_vinfo)->header,
   12464         2903 :                                      gimple_bb (last_stmt)))
   12465         1120 :                 si = gsi_end_bb (loop_preheader_edge
   12466          560 :                                    (LOOP_VINFO_LOOP (loop_vinfo))->src);
   12467              :               /* Avoid scheduling internal defs outside of the loop when
   12468              :                  we might have only implicitly tracked loop mask/len defs.  */
   12469           84 :               if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
   12470       178761 :                   || LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
   12471              :                 {
   12472           84 :                   gimple_stmt_iterator si2
   12473           84 :                     = gsi_after_labels (LOOP_VINFO_LOOP (loop_vinfo)->header);
   12474           84 :                   if ((gsi_end_p (si2)
   12475            0 :                        && (LOOP_VINFO_LOOP (loop_vinfo)->header
   12476            0 :                            != gimple_bb (last_stmt))
   12477            0 :                        && dominated_by_p (CDI_DOMINATORS,
   12478              :                                           LOOP_VINFO_LOOP (loop_vinfo)->header,
   12479            0 :                                           gimple_bb (last_stmt)))
   12480           84 :                       || (!gsi_end_p (si2)
   12481           84 :                           && last_stmt != *si2
   12482           79 :                           && vect_stmt_dominates_stmt_p (last_stmt, *si2)))
   12483            3 :                     si = si2;
   12484              :                 }
   12485              :             }
   12486              :         }
   12487              :     }
   12488              : 
   12489      1929878 :   if (place_only)
   12490              :     {
   12491       952884 :       if (dump_enabled_p () && last_stmt)
   12492         4016 :         dump_printf_loc (MSG_NOTE, vect_location,
   12493              :                          "placing node %p at %G:", (void *)node, last_stmt);
   12494              :       /* Verify we either get a stmt anchor or region start.  */
   12495       952884 :       gcc_assert ((last_stmt && gimple_bb (last_stmt))
   12496              :                   || (!last_stmt && gsi_bb (si)));
   12497       952884 :       node->si = last_stmt;
   12498       952884 :       return true;
   12499              :     }
   12500              : 
   12501              :   /* ???  Asserting vect_stmt_dominates_stmt_p (gsi_stmt (si), node->si)
   12502              :      does not work because in some cases we advance si from last_stmt (as
   12503              :      we want to insert after vector stmts) and because vector stmts of
   12504              :      children have been inserted possibly at the same location constraint,
   12505              :      moving si even further.  */
   12506       976994 :   if (flag_checking && node->si && gimple_bb (node->si) && !gsi_end_p (si))
   12507              :     {
   12508       602534 :       auto gsi2 = si;
   12509       602534 :       while (1)
   12510              :         {
   12511       602534 :           if (vect_stmt_dominates_stmt_p (gsi_stmt (gsi2), node->si))
   12512              :             break;
   12513              :           /* As we have possibly advanced si it might now point to the
   12514              :              scalar stmt immediately following node->si.  That's OK.  */
   12515        23830 :           if (gsi_stmt (gsi2) != gsi_stmt (si)
   12516        23830 :               && gimple_uid (gsi_stmt (gsi2)) != 0)
   12517            0 :             gcc_unreachable ();
   12518        23830 :           gsi_prev (&gsi2);
   12519        23830 :           if (gsi_end_p (gsi2))
   12520              :             {
   12521          545 :               if (is_a <gphi *> (node->si)
   12522          545 :                   && gimple_bb (node->si) == gsi_bb (si))
   12523              :                 break;
   12524            0 :               gcc_unreachable ();
   12525              :             }
   12526              :         }
   12527              :     }
   12528              : 
   12529       976994 :   if (dump_enabled_p ())
   12530              :     {
   12531        75104 :       if (stmt_info)
   12532        71682 :         dump_printf_loc (MSG_NOTE, vect_location,
   12533              :                          "------>vectorizing SLP node starting from: %G",
   12534              :                          stmt_info->stmt);
   12535              :       else
   12536              :         {
   12537         3422 :           dump_printf_loc (MSG_NOTE, vect_location,
   12538              :                            "------>vectorizing SLP node:\n");
   12539         3422 :           vect_print_slp_tree (MSG_NOTE, vect_location, node);
   12540              :         }
   12541              :     }
   12542       976994 :   vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
   12543       976994 :   return true;
   12544              : }
   12545              : 
   12546              : /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
   12547              :    For loop vectorization this is done in vectorizable_call, but for SLP
   12548              :    it needs to be deferred until end of vect_schedule_slp, because multiple
   12549              :    SLP instances may refer to the same scalar stmt.  */
   12550              : 
   12551              : static void
   12552       619861 : vect_remove_slp_scalar_calls (vec_info *vinfo,
   12553              :                               slp_tree node, hash_set<slp_tree> &visited)
   12554              : {
   12555       619861 :   gimple *new_stmt;
   12556       619861 :   gimple_stmt_iterator gsi;
   12557       619861 :   tree lhs;
   12558              : 
   12559       619861 :   if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
   12560       194780 :     return;
   12561              : 
   12562       469439 :   if (visited.add (node))
   12563              :     return;
   12564              : 
   12565      1586645 :   for (auto child : SLP_TREE_CHILDREN (node))
   12566       526990 :     vect_remove_slp_scalar_calls (vinfo, child, visited);
   12567              : 
   12568      1765540 :   for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
   12569              :     {
   12570       498941 :       if (!stmt_info)
   12571         3763 :         continue;
   12572       495178 :       stmt_info = vect_orig_stmt (stmt_info);
   12573       495178 :       gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
   12574         5442 :       if (!stmt || gimple_bb (stmt) == NULL)
   12575       489778 :         continue;
   12576         5400 :       lhs = gimple_call_lhs (stmt);
   12577         5400 :       if (lhs)
   12578         4816 :         new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
   12579              :       else
   12580          584 :         new_stmt = gimple_build_nop ();
   12581         5400 :       unlink_stmt_vdef (stmt_info->stmt);
   12582         5400 :       gsi = gsi_for_stmt (stmt);
   12583         5400 :       vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
   12584         5400 :       if (lhs)
   12585         4816 :         SSA_NAME_DEF_STMT (lhs) = new_stmt;
   12586              :     }
   12587              : }
   12588              : 
   12589              : static void
   12590        92871 : vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
   12591              : {
   12592        92871 :   hash_set<slp_tree> visited;
   12593        92871 :   vect_remove_slp_scalar_calls (vinfo, node, visited);
   12594        92871 : }
   12595              : 
   12596              : /* Vectorize the instance root.  */
   12597              : 
   12598              : void
   12599         9614 : vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance instance)
   12600              : {
   12601         9614 :   if (instance->kind == slp_inst_kind_ctor)
   12602              :     {
   12603         5724 :       tree new_def;
   12604         5724 :       if (SLP_TREE_VEC_DEFS (node).length () == 1)
   12605              :         {
   12606         5678 :           new_def = SLP_TREE_VEC_DEFS (node)[0];
   12607         5678 :           tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
   12608         5678 :           if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
   12609         5678 :                                           TREE_TYPE (new_def)))
   12610            0 :             new_def = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (root_lhs),
   12611              :                                new_def);
   12612              :         }
   12613              :       else
   12614              :         {
   12615           46 :           gcc_assert (SLP_TREE_VEC_DEFS (node).length () > 1);
   12616           46 :           tree child_def;
   12617           46 :           int j;
   12618           46 :           vec<constructor_elt, va_gc> *v;
   12619           46 :           vec_alloc (v, SLP_TREE_VEC_DEFS (node).length ());
   12620              : 
   12621              :           /* A CTOR can handle V16HI composition from VNx8HI so we
   12622              :              do not need to convert vector elements if the types
   12623              :              do not match.  */
   12624          186 :           FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def)
   12625           94 :             CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def);
   12626           46 :           tree rtype
   12627           46 :             = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt));
   12628           46 :           new_def = build_constructor (rtype, v);
   12629              :         }
   12630              : 
   12631         5724 :       gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
   12632         5724 :       gimple_assign_set_rhs_from_tree (&rgsi, new_def);
   12633         5724 :       update_stmt (gsi_stmt (rgsi));
   12634         5724 :       return;
   12635              :     }
   12636         3890 :   else if (instance->kind == slp_inst_kind_bb_reduc)
   12637              :     {
   12638              :       /* Largely inspired by reduction chain epilogue handling in
   12639              :          vect_create_epilog_for_reduction.  */
   12640         2233 :       vec<tree> vec_defs = vNULL;
   12641         2233 :       vect_get_slp_defs (node, &vec_defs);
   12642         2233 :       enum tree_code reduc_code
   12643         2233 :         = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
   12644              :       /* ???  We actually have to reflect signs somewhere.  */
   12645         2233 :       if (reduc_code == MINUS_EXPR)
   12646            0 :         reduc_code = PLUS_EXPR;
   12647         2233 :       gimple_seq epilogue = NULL;
   12648              :       /* We may end up with more than one vector result, reduce them
   12649              :          to one vector.  */
   12650         2233 :       tree vec_def = vec_defs[0];
   12651         2233 :       tree vectype = TREE_TYPE (vec_def);
   12652         2233 :       tree compute_vectype = vectype;
   12653         2233 :       bool pun_for_overflow_p = (ANY_INTEGRAL_TYPE_P (vectype)
   12654         1071 :                                  && TYPE_OVERFLOW_UNDEFINED (vectype)
   12655         2917 :                                  && operation_can_overflow (reduc_code));
   12656          495 :       if (pun_for_overflow_p)
   12657              :         {
   12658          495 :           compute_vectype = unsigned_type_for (vectype);
   12659          495 :           vec_def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
   12660              :                                   compute_vectype, vec_def);
   12661              :         }
   12662         2500 :       for (unsigned i = 1; i < vec_defs.length (); ++i)
   12663              :         {
   12664          267 :           tree def = vec_defs[i];
   12665          267 :           if (pun_for_overflow_p)
   12666           41 :             def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
   12667              :                                 compute_vectype, def);
   12668          267 :           vec_def = gimple_build (&epilogue, reduc_code, compute_vectype,
   12669              :                                   vec_def, def);
   12670              :         }
   12671         2233 :       vec_defs.release ();
   12672              :       /* ???  Support other schemes than direct internal fn or two
   12673              :          element vectors.  */
   12674         2233 :       tree scalar_def;
   12675         2233 :       internal_fn reduc_fn;
   12676         3225 :       if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
   12677         2233 :           || reduc_fn == IFN_LAST
   12678         4466 :           || !direct_internal_fn_supported_p (reduc_fn, compute_vectype,
   12679              :                                               OPTIMIZE_FOR_BOTH))
   12680              :         {
   12681          992 :           gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (compute_vectype), 2u));
   12682          992 :           tree tem0 = gimple_build (&epilogue, BIT_FIELD_REF,
   12683          992 :                                     TREE_TYPE (compute_vectype), vec_def,
   12684          992 :                                     TYPE_SIZE (TREE_TYPE (compute_vectype)),
   12685          992 :                                     bitsize_zero_node);
   12686          992 :           tree tem1 = gimple_build (&epilogue, BIT_FIELD_REF,
   12687          992 :                                     TREE_TYPE (compute_vectype), vec_def,
   12688          992 :                                     TYPE_SIZE (TREE_TYPE (compute_vectype)),
   12689          992 :                                     TYPE_SIZE (TREE_TYPE (compute_vectype)));
   12690          992 :           scalar_def = gimple_build (&epilogue, reduc_code,
   12691          992 :                                      TREE_TYPE (compute_vectype), tem0, tem1);
   12692              :         }
   12693              :       else
   12694         1241 :         scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
   12695         1241 :                                    TREE_TYPE (compute_vectype), vec_def);
   12696         2233 :       if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
   12697              :         {
   12698          917 :           tree rem_def = NULL_TREE;
   12699         2121 :           for (auto def : SLP_INSTANCE_REMAIN_DEFS (instance))
   12700              :             {
   12701         1204 :               def = gimple_convert (&epilogue, TREE_TYPE (scalar_def), def);
   12702         1204 :               if (!rem_def)
   12703              :                 rem_def = def;
   12704              :               else
   12705          287 :                 rem_def = gimple_build (&epilogue, reduc_code,
   12706          287 :                                         TREE_TYPE (scalar_def),
   12707              :                                         rem_def, def);
   12708              :             }
   12709          917 :           scalar_def = gimple_build (&epilogue, reduc_code,
   12710          917 :                                      TREE_TYPE (scalar_def),
   12711              :                                      scalar_def, rem_def);
   12712              :         }
   12713         2233 :       scalar_def = gimple_convert (&epilogue,
   12714         2233 :                                    TREE_TYPE (vectype), scalar_def);
   12715         2233 :       gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
   12716         2233 :       gsi_insert_seq_before (&rgsi, epilogue, GSI_SAME_STMT);
   12717         2233 :       gimple_assign_set_rhs_from_tree (&rgsi, scalar_def);
   12718         2233 :       update_stmt (gsi_stmt (rgsi));
   12719         2233 :       return;
   12720              :     }
   12721         1657 :   else if (instance->kind == slp_inst_kind_gcond)
   12722              :     {
   12723              :       /* Only support a single root for now as we can't codegen CFG yet and so we
   12724              :          can't support lane > 1 at this time.  */
   12725         1657 :       gcc_assert (instance->root_stmts.length () == 1);
   12726         1657 :       auto root_stmt_info = instance->root_stmts[0];
   12727         1657 :       auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
   12728         1657 :       gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
   12729         1657 :       gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
   12730         1657 :       bool res = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
   12731              :                                           root_stmt_info, &rgsi, node, NULL);
   12732         1657 :       gcc_assert (res);
   12733         1657 :       return;
   12734              :     }
   12735              :   else
   12736            0 :     gcc_unreachable ();
   12737              : }
   12738              : 
   12739              : struct slp_scc_info
   12740              : {
   12741              :   bool on_stack;
   12742              :   bool res;
   12743              :   int dfs;
   12744              :   int lowlink;
   12745              : };
   12746              : 
   12747              : /* Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs.
   12748              :    When PLACE_ONLY, return false if there is no possible schedule.  */
   12749              : 
   12750              : static bool
   12751      3144981 : vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance,
   12752              :                    hash_map<slp_tree, slp_scc_info> &scc_info,
   12753              :                    int &maxdfs, vec<slp_tree> &stack, bool place_only)
   12754              : {
   12755      3144981 :   bool existed_p;
   12756      3144981 :   slp_scc_info *info = &scc_info.get_or_insert (node, &existed_p);
   12757      3144981 :   gcc_assert (!existed_p);
   12758      3144981 :   info->dfs = maxdfs;
   12759      3144981 :   info->lowlink = maxdfs;
   12760      3144981 :   maxdfs++;
   12761              : 
   12762              :   /* Leaf.  */
   12763      3144981 :   if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
   12764              :     {
   12765      1215091 :       info->on_stack = false;
   12766      1215091 :       bool res = vect_schedule_slp_node (vinfo, node, instance, place_only);
   12767      1215091 :       gcc_assert (res);
   12768      1215091 :       info->res = res;
   12769      1215091 :       return res;
   12770              :     }
   12771              : 
   12772      1929890 :   info->on_stack = true;
   12773      1929890 :   info->res = true;
   12774      1929890 :   stack.safe_push (node);
   12775              : 
   12776      1929890 :   bool res = true;
   12777      1929890 :   unsigned i;
   12778      1929890 :   slp_tree child;
   12779              :   /* DFS recurse.  */
   12780      7559390 :   FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
   12781              :     {
   12782      2046412 :       if (!child)
   12783        56824 :         continue;
   12784      1989588 :       slp_scc_info *child_info = scc_info.get (child);
   12785      1989588 :       if (!child_info)
   12786              :         {
   12787      1840786 :           res &= vect_schedule_scc (vinfo, child, instance, scc_info,
   12788              :                                     maxdfs, stack, place_only);
   12789              :           /* Recursion might have re-allocated the node.  */
   12790      1840786 :           info = scc_info.get (node);
   12791      1840786 :           child_info = scc_info.get (child);
   12792      1840786 :           info->lowlink = MIN (info->lowlink, child_info->lowlink);
   12793              :         }
   12794       148802 :       else if (child_info->on_stack)
   12795        35573 :         info->lowlink = MIN (info->lowlink, child_info->dfs);
   12796              :       else
   12797       113229 :         res &= child_info->res;
   12798              :     }
   12799      1929890 :   if (info->lowlink != info->dfs)
   12800              :     return res;
   12801              : 
   12802      1890455 :   auto_vec<slp_tree, 4> phis_to_fixup;
   12803              : 
   12804              :   /* Singleton.  */
   12805      1890455 :   if (stack.last () == node)
   12806              :     {
   12807      1861494 :       stack.pop ();
   12808      1861494 :       info->on_stack = false;
   12809      1861494 :       res &= vect_schedule_slp_node (vinfo, node, instance, place_only);
   12810      1861494 :       info->res = res;
   12811      1861494 :       if (!SLP_TREE_PERMUTE_P (node)
   12812      1861494 :           && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (node)->stmt))
   12813        72429 :         phis_to_fixup.quick_push (node);
   12814              :     }
   12815              :   else
   12816              :     {
   12817              :       /* SCC.  */
   12818        28961 :       int last_idx = stack.length () - 1;
   12819        68396 :       while (stack[last_idx] != node)
   12820        39435 :         last_idx--;
   12821              :       /* We can break the cycle at PHIs who have at least one child
   12822              :          code generated.  Then we could re-start the DFS walk until
   12823              :          all nodes in the SCC are covered (we might have new entries
   12824              :          for only back-reachable nodes).  But it's simpler to just
   12825              :          iterate and schedule those that are ready.  */
   12826        28961 :       unsigned todo = stack.length () - last_idx;
   12827        28961 :       auto_vec<slp_tree, 4> saved_scc (todo);
   12828        28961 :       saved_scc.quick_grow (todo);
   12829        86883 :       memcpy (saved_scc.address (), &stack.address ()[last_idx],
   12830              :               sizeof (slp_tree) * todo);
   12831        29757 :       do
   12832              :         {
   12833       132073 :           for (int idx = stack.length () - 1; idx >= last_idx; --idx)
   12834              :             {
   12835        72559 :               slp_tree entry = stack[idx];
   12836        72559 :               if (!entry)
   12837         2531 :                 continue;
   12838        70028 :               bool phi = (!SLP_TREE_PERMUTE_P (entry)
   12839        70028 :                           && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (entry)->stmt));
   12840        70028 :               bool ready = !phi;
   12841       213019 :               FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (entry), i, child)
   12842       132853 :                 if (!child)
   12843              :                   {
   12844        23009 :                     gcc_assert (phi);
   12845              :                     ready = true;
   12846              :                     break;
   12847              :                   }
   12848       109844 :                 else if (scc_info.get (child)->on_stack)
   12849              :                   {
   12850        30801 :                     if (!phi)
   12851              :                       {
   12852              :                         ready = false;
   12853              :                         break;
   12854              :                       }
   12855              :                   }
   12856              :                 else
   12857              :                   {
   12858        79043 :                     if (phi)
   12859              :                       {
   12860              :                         ready = true;
   12861              :                         break;
   12862              :                       }
   12863              :                   }
   12864        47019 :               if (ready)
   12865              :                 {
   12866        68396 :                   res &= vect_schedule_slp_node (vinfo, entry, instance,
   12867              :                                                  place_only);
   12868        68396 :                   scc_info.get (entry)->on_stack = false;
   12869        68396 :                   stack[idx] = NULL;
   12870        68396 :                   todo--;
   12871        68396 :                   if (phi)
   12872        35425 :                     phis_to_fixup.safe_push (entry);
   12873              :                 }
   12874              :             }
   12875              :         }
   12876        29757 :       while (todo != 0);
   12877              : 
   12878              :       /* Push the scheduling result to all of the SCC.  */
   12879       155279 :       for (slp_tree entry : saved_scc)
   12880        68396 :         scc_info.get (entry)->res = res;
   12881              : 
   12882              :       /* Pop the SCC.  */
   12883        28961 :       stack.truncate (last_idx);
   12884        28961 :     }
   12885              : 
   12886      1890455 :   if (place_only)
   12887              :     return res;
   12888              : 
   12889              :   /* Now fixup the backedge def of the vectorized PHIs in this SCC.  */
   12890              :   slp_tree phi_node;
   12891       999149 :   FOR_EACH_VEC_ELT (phis_to_fixup, i, phi_node)
   12892              :     {
   12893        53933 :       gphi *phi = as_a <gphi *> (SLP_TREE_REPRESENTATIVE (phi_node)->stmt);
   12894        53933 :       edge_iterator ei;
   12895        53933 :       edge e;
   12896       168045 :       FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
   12897              :         {
   12898       114112 :           unsigned dest_idx = e->dest_idx;
   12899       114112 :           child = SLP_TREE_CHILDREN (phi_node)[dest_idx];
   12900       114112 :           if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def)
   12901        66800 :             continue;
   12902        47312 :           unsigned n = SLP_TREE_VEC_DEFS (phi_node).length ();
   12903              :           /* Simply fill all args.  */
   12904        47312 :           if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node))
   12905              :               != vect_first_order_recurrence)
   12906        99769 :             for (unsigned i = 0; i < n; ++i)
   12907              :               {
   12908        52500 :                 tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i];
   12909        52500 :                 gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef));
   12910        52500 :                 add_phi_arg (phi, vect_get_slp_vect_def (child, i),
   12911              :                              e, gimple_phi_arg_location (phi, dest_idx));
   12912              :               }
   12913              :           else
   12914              :             {
   12915              :               /* Unless it is a first order recurrence which needs
   12916              :                  args filled in for both the PHI node and the permutes.  */
   12917           43 :               gimple *perm
   12918           43 :                 = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]);
   12919           43 :               gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm));
   12920           43 :               add_phi_arg (as_a <gphi *> (rphi),
   12921              :                            vect_get_slp_vect_def (child, n - 1),
   12922              :                            e, gimple_phi_arg_location (phi, dest_idx));
   12923          166 :               for (unsigned i = 0; i < n; ++i)
   12924              :                 {
   12925           80 :                   gimple *perm
   12926           80 :                     = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]);
   12927           80 :                   if (i > 0)
   12928           37 :                     gimple_assign_set_rhs1 (perm,
   12929              :                                             vect_get_slp_vect_def (child, i - 1));
   12930           80 :                   gimple_assign_set_rhs2 (perm,
   12931              :                                           vect_get_slp_vect_def (child, i));
   12932           80 :                   update_stmt (perm);
   12933              :                 }
   12934              :             }
   12935              :         }
   12936              :     }
   12937              : 
   12938       945216 :   gcc_assert (res);
   12939              :   return true;
   12940      1890455 : }
   12941              : 
   12942              : /* Generate vector code for SLP_INSTANCES in the loop/basic block.  Perform
   12943              :    vector stmt placement only when PLACE_ONLY is true, removing SLP graph
   12944              :    entries that cannot be scheduled.  If placing, return false if a schedule
   12945              :    cannot be computed for any entry.  */
   12946              : 
   12947              : bool
   12948       794428 : vect_schedule_slp (vec_info *vinfo, vec<slp_instance> &slp_instances,
   12949              :                    bool place_only)
   12950              : {
   12951       794428 :   slp_instance instance;
   12952       794428 :   unsigned int i;
   12953              : 
   12954       794428 :   hash_map<slp_tree, slp_scc_info> scc_info;
   12955       794428 :   int maxdfs = 0;
   12956      2102528 :   for (i = 0; slp_instances.iterate (i, &instance); )
   12957              :     {
   12958      1308100 :       slp_tree node = SLP_INSTANCE_TREE (instance);
   12959      1308100 :       if (!place_only && dump_enabled_p ())
   12960              :         {
   12961        16816 :           dump_printf_loc (MSG_NOTE, vect_location,
   12962              :                            "Vectorizing SLP tree:\n");
   12963              :           /* ???  Dump all?  */
   12964        16816 :           if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
   12965          592 :             dump_printf_loc (MSG_NOTE, vect_location, "Root stmt: %G",
   12966          592 :                          SLP_INSTANCE_ROOT_STMTS (instance)[0]->stmt);
   12967        16816 :           vect_print_slp_graph (MSG_NOTE, vect_location,
   12968              :                                 SLP_INSTANCE_TREE (instance));
   12969              :         }
   12970              :       /* Schedule the tree of INSTANCE, scheduling SCCs in a way to
   12971              :          have a PHI be the node breaking the cycle.  */
   12972      1308100 :       bool res;
   12973      1308100 :       if (slp_scc_info *info = scc_info.get (node))
   12974         3905 :         res = info->res;
   12975              :       else
   12976              :         {
   12977      1304195 :           auto_vec<slp_tree> stack;
   12978      1304195 :           res = vect_schedule_scc (vinfo, node, instance, scc_info,
   12979              :                                    maxdfs, stack, place_only);
   12980      1304195 :         }
   12981              : 
   12982      1308100 :       if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
   12983              :         {
   12984        64835 :           if (place_only)
   12985              :             {
   12986        55221 :               gimple *root_stmt = instance->root_stmts[0]->stmt;
   12987        55221 :               res &= (!node->si
   12988              :                       /* As we instert after node->si it may not be the
   12989              :                          root_stmt itself.  */
   12990        55221 :                       || (node->si != root_stmt
   12991        54300 :                           && vect_stmt_dominates_stmt_p (node->si, root_stmt)));
   12992              :             }
   12993              :           else
   12994         9614 :             vectorize_slp_instance_root_stmt (vinfo, node, instance);
   12995              :         }
   12996              : 
   12997      1308100 :       if (!place_only && dump_enabled_p ())
   12998        16816 :         dump_printf_loc (MSG_NOTE, vect_location,
   12999              :                          "vectorizing stmts using SLP.\n");
   13000              : 
   13001      1308100 :       if (!res)
   13002              :         {
   13003           38 :           gcc_assert (place_only);
   13004           38 :           if (dump_enabled_p ())
   13005           24 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   13006              :                              "not vectorized: cannot schedule SLP graph "
   13007              :                              "entry %p\n", (void *)instance);
   13008           38 :           vect_free_slp_instance (instance);
   13009           38 :           slp_instances.ordered_remove (i);
   13010           38 :           continue;
   13011              :         }
   13012      1308062 :       ++i;
   13013              :     }
   13014              : 
   13015       794428 :   if (place_only)
   13016       497092 :     return !slp_instances.is_empty ();
   13017              : 
   13018      1925877 :   FOR_EACH_VEC_ELT (slp_instances, i, instance)
   13019              :     {
   13020       585567 :       slp_tree root = SLP_INSTANCE_TREE (instance);
   13021       585567 :       stmt_vec_info store_info;
   13022       585567 :       unsigned int j;
   13023              : 
   13024              :       /* Remove scalar call stmts.  Do not do this for basic-block
   13025              :          vectorization as not all uses may be vectorized.
   13026              :          ???  Why should this be necessary?  DCE should be able to
   13027              :          remove the stmts itself.
   13028              :          ???  For BB vectorization we can as well remove scalar
   13029              :          stmts starting from the SLP tree root if they have no
   13030              :          uses.  */
   13031       585567 :       if (is_a <loop_vec_info> (vinfo))
   13032        92871 :         vect_remove_slp_scalar_calls (vinfo, root);
   13033              : 
   13034              :       /* Remove vectorized stores original scalar stmts.  */
   13035      2617854 :       for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
   13036              :         {
   13037      1481775 :           if (!store_info
   13038      1481758 :               || !STMT_VINFO_DATA_REF (store_info)
   13039      1452698 :               || !DR_IS_WRITE (STMT_VINFO_DATA_REF (store_info)))
   13040              :             break;
   13041              : 
   13042      1446720 :           store_info = vect_orig_stmt (store_info);
   13043              :           /* Free the attached stmt_vec_info and remove the stmt.  */
   13044      1446720 :           vinfo->remove_stmt (store_info);
   13045              : 
   13046              :           /* Invalidate SLP_TREE_REPRESENTATIVE in case we released it
   13047              :              to not crash in vect_free_slp_tree later.  */
   13048      1446720 :           if (SLP_TREE_REPRESENTATIVE (root) == store_info)
   13049       550179 :             SLP_TREE_REPRESENTATIVE (root) = NULL;
   13050              :         }
   13051              :     }
   13052              : 
   13053              :   return true;
   13054       794428 : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.