LCOV - code coverage report
Current view: top level - gcc - tree-vect-stmts.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 82.0 % 7074 5804
Test Date: 2026-11-14 16:39:50 Functions: 89.7 % 107 96
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Statement Analysis and Transformation for Vectorization
       2              :    Copyright (C) 2003-2026 Free Software Foundation, Inc.
       3              :    Contributed by Dorit Naishlos <dorit@il.ibm.com>
       4              :    and Ira Rosen <irar@il.ibm.com>
       5              : 
       6              : This file is part of GCC.
       7              : 
       8              : GCC is free software; you can redistribute it and/or modify it under
       9              : the terms of the GNU General Public License as published by the Free
      10              : Software Foundation; either version 3, or (at your option) any later
      11              : version.
      12              : 
      13              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      14              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      15              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      16              : for more details.
      17              : 
      18              : You should have received a copy of the GNU General Public License
      19              : along with GCC; see the file COPYING3.  If not see
      20              : <http://www.gnu.org/licenses/>.  */
      21              : 
      22              : #include "config.h"
      23              : #include "system.h"
      24              : #include "coretypes.h"
      25              : #include "backend.h"
      26              : #include "target.h"
      27              : #include "rtl.h"
      28              : #include "tree.h"
      29              : #include "gimple.h"
      30              : #include "ssa.h"
      31              : #include "optabs-tree.h"
      32              : #include "insn-config.h"
      33              : #include "recog.h"            /* FIXME: for insn_data */
      34              : #include "cgraph.h"
      35              : #include "dumpfile.h"
      36              : #include "alias.h"
      37              : #include "fold-const.h"
      38              : #include "stor-layout.h"
      39              : #include "tree-eh.h"
      40              : #include "gimplify.h"
      41              : #include "gimple-iterator.h"
      42              : #include "gimplify-me.h"
      43              : #include "tree-cfg.h"
      44              : #include "tree-ssa-loop-manip.h"
      45              : #include "cfgloop.h"
      46              : #include "explow.h"
      47              : #include "tree-ssa-loop.h"
      48              : #include "tree-scalar-evolution.h"
      49              : #include "tree-vectorizer.h"
      50              : #include "builtins.h"
      51              : #include "internal-fn.h"
      52              : #include "tree-vector-builder.h"
      53              : #include "vec-perm-indices.h"
      54              : #include "gimple-range.h"
      55              : #include "tree-ssa-loop-niter.h"
      56              : #include "gimple-fold.h"
      57              : #include "regs.h"
      58              : #include "attribs.h"
      59              : #include "optabs-libfuncs.h"
      60              : #include "tree-dfa.h"
      61              : #include "hierarchical_discriminator.h"
      62              : 
      63              : /* For lang_hooks.types.type_for_mode.  */
      64              : #include "langhooks.h"
      65              : 
      66              : static tree vector_vector_composition_type (tree, poly_uint64, tree *,
      67              :                                             bool = false);
      68              : 
      69              : /* Return TRUE iff the given statement is in an inner loop relative to
      70              :    the loop being vectorized.  */
      71              : bool
      72      6669288 : stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
      73              : {
      74      6669288 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
      75      6669288 :   basic_block bb = gimple_bb (stmt);
      76      6669288 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
      77      3157574 :   class loop* loop;
      78              : 
      79      3157574 :   if (!loop_vinfo)
      80              :     return false;
      81              : 
      82      3157574 :   loop = LOOP_VINFO_LOOP (loop_vinfo);
      83              : 
      84      3157574 :   return (bb->loop_father == loop->inner);
      85              : }
      86              : 
      87              : /* Record the cost of a statement, either by directly informing the
      88              :    target model or by saving it in a vector for later processing.
      89              :    Return a preliminary estimate of the statement's cost.  */
      90              : 
      91              : unsigned
      92      9314254 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
      93              :                   enum vect_cost_for_stmt kind,
      94              :                   stmt_vec_info stmt_info, slp_tree node,
      95              :                   tree vectype, int misalign,
      96              :                   enum vect_cost_model_location where)
      97              : {
      98      9314254 :   if ((kind == vector_load || kind == unaligned_load)
      99      1679824 :       && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
     100              :     kind = vector_gather_load;
     101      9314254 :   if ((kind == vector_store || kind == unaligned_store)
     102      1058961 :       && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
     103      9314254 :     kind = vector_scatter_store;
     104              : 
     105      9314254 :   stmt_info_for_cost si
     106      9314254 :     = { count, kind, where, stmt_info, node, vectype, misalign };
     107      9314254 :   body_cost_vec->safe_push (si);
     108              : 
     109      9314254 :   return (unsigned)
     110      9314254 :       (builtin_vectorization_cost (kind, vectype, misalign) * count);
     111              : }
     112              : 
     113              : unsigned
     114      4331093 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
     115              :                   enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
     116              :                   tree vectype, int misalign,
     117              :                   enum vect_cost_model_location where)
     118              : {
     119      4331093 :   return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
     120      4331093 :                            vectype, misalign, where);
     121              : }
     122              : 
     123              : unsigned
     124      1803192 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
     125              :                   enum vect_cost_for_stmt kind, slp_tree node,
     126              :                   tree vectype, int misalign,
     127              :                   enum vect_cost_model_location where)
     128              : {
     129      1803192 :   return record_stmt_cost (body_cost_vec, count, kind,
     130              :                            SLP_TREE_REPRESENTATIVE (node), node,
     131      1803192 :                            vectype, misalign, where);
     132              : }
     133              : 
     134              : unsigned
     135            0 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
     136              :                   enum vect_cost_for_stmt kind,
     137              :                   enum vect_cost_model_location where)
     138              : {
     139            0 :   gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
     140              :               || kind == scalar_stmt);
     141            0 :   return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
     142            0 :                            NULL_TREE, 0, where);
     143              : }
     144              : 
     145              : /* Return a variable of type ELEM_TYPE[NELEMS].  */
     146              : 
     147              : static tree
     148            0 : create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
     149              : {
     150            0 :   return create_tmp_var (build_array_type_nelts (elem_type, nelems),
     151            0 :                          "vect_array");
     152              : }
     153              : 
     154              : /* ARRAY is an array of vectors created by create_vector_array.
     155              :    Return an SSA_NAME for the vector in index N.  The reference
     156              :    is part of the vectorization of STMT_INFO and the vector is associated
     157              :    with scalar destination SCALAR_DEST.
     158              :    If we need to ensure that inactive elements are set to zero,
     159              :    NEED_ZEROING is true, MASK contains the loop mask to be used.  */
     160              : 
     161              : static tree
     162            0 : read_vector_array (vec_info *vinfo,
     163              :                    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     164              :                    tree scalar_dest, tree array, unsigned HOST_WIDE_INT n,
     165              :                    bool need_zeroing, tree mask)
     166              : {
     167            0 :   tree vect_type, vect, vect_name, tmp, tmp_name, array_ref;
     168            0 :   gimple *new_stmt;
     169              : 
     170            0 :   gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
     171            0 :   vect_type = TREE_TYPE (TREE_TYPE (array));
     172            0 :   tmp = vect_create_destination_var (scalar_dest, vect_type);
     173            0 :   vect = vect_create_destination_var (scalar_dest, vect_type);
     174            0 :   array_ref = build4 (ARRAY_REF, vect_type, array,
     175            0 :                       build_int_cst (size_type_node, n),
     176              :                       NULL_TREE, NULL_TREE);
     177              : 
     178            0 :   new_stmt = gimple_build_assign (tmp, array_ref);
     179            0 :   tmp_name = make_ssa_name (vect, new_stmt);
     180            0 :   gimple_assign_set_lhs (new_stmt, tmp_name);
     181            0 :   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
     182              : 
     183            0 :   if (need_zeroing)
     184              :     {
     185            0 :       tree vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
     186              :                                               vect_type);
     187            0 :       vect_name = make_ssa_name (vect, new_stmt);
     188            0 :       new_stmt
     189            0 :         = gimple_build_assign (vect_name, VEC_COND_EXPR,
     190              :                                mask, tmp_name, vec_els);
     191            0 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
     192              :     }
     193              :   else
     194              :     vect_name = tmp_name;
     195              : 
     196            0 :   return vect_name;
     197              : }
     198              : 
     199              : /* ARRAY is an array of vectors created by create_vector_array.
     200              :    Emit code to store SSA_NAME VECT in index N of the array.
     201              :    The store is part of the vectorization of STMT_INFO.  */
     202              : 
     203              : static void
     204            0 : write_vector_array (vec_info *vinfo,
     205              :                     stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     206              :                     tree vect, tree array, unsigned HOST_WIDE_INT n)
     207              : {
     208            0 :   tree array_ref;
     209            0 :   gimple *new_stmt;
     210              : 
     211            0 :   array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
     212            0 :                       build_int_cst (size_type_node, n),
     213              :                       NULL_TREE, NULL_TREE);
     214              : 
     215            0 :   new_stmt = gimple_build_assign (array_ref, vect);
     216            0 :   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
     217            0 : }
     218              : 
     219              : /* PTR is a pointer to an array of type TYPE.  Return a representation
     220              :    of *PTR.  The memory reference replaces those in FIRST_DR
     221              :    (and its group).  */
     222              : 
     223              : static tree
     224            0 : create_array_ref (tree type, tree ptr, tree alias_ptr_type)
     225              : {
     226            0 :   tree mem_ref;
     227              : 
     228            0 :   mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
     229              :   /* Arrays have the same alignment as their type.  */
     230            0 :   set_ssa_name_alignment (ptr, TYPE_ALIGN_UNIT (type), 0);
     231            0 :   return mem_ref;
     232              : }
     233              : 
     234              : /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
     235              :    Emit the clobber before *GSI.  */
     236              : 
     237              : static void
     238           20 : vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
     239              :                        gimple_stmt_iterator *gsi, tree var)
     240              : {
     241           20 :   tree clobber = build_clobber (TREE_TYPE (var));
     242           20 :   gimple *new_stmt = gimple_build_assign (var, clobber);
     243           20 :   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
     244           20 : }
     245              : 
     246              : /* Utility functions used by vect_mark_stmts_to_be_vectorized.  */
     247              : 
     248              : /* Function vect_mark_relevant.
     249              : 
     250              :    Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST.  */
     251              : 
     252              : static void
     253      3331317 : vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
     254              :                     enum vect_relevant relevant, bool live_p)
     255              : {
     256      3331317 :   enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
     257      3331317 :   bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
     258              : 
     259      3331317 :   if (dump_enabled_p ())
     260       172732 :     dump_printf_loc (MSG_NOTE, vect_location,
     261              :                      "mark relevant %d, live %d: %G", relevant, live_p,
     262              :                      stmt_info->stmt);
     263              : 
     264              :   /* If this stmt is an original stmt in a pattern, we might need to mark its
     265              :      related pattern stmt instead of the original stmt.  However, such stmts
     266              :      may have their own uses that are not in any pattern, in such cases the
     267              :      stmt itself should be marked.  */
     268      3331317 :   if (STMT_VINFO_IN_PATTERN_P (stmt_info))
     269              :     {
     270              :       /* This is the last stmt in a sequence that was detected as a
     271              :          pattern that can potentially be vectorized.  Don't mark the stmt
     272              :          as relevant/live because it's not going to be vectorized.
     273              :          Instead mark the pattern-stmt that replaces it.  */
     274              : 
     275       258629 :       if (dump_enabled_p ())
     276         3134 :         dump_printf_loc (MSG_NOTE, vect_location,
     277              :                          "last stmt in pattern. don't mark"
     278              :                          " relevant/live.\n");
     279              : 
     280       258629 :       stmt_vec_info old_stmt_info = stmt_info;
     281       258629 :       stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
     282       258629 :       gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
     283       258629 :       save_relevant = STMT_VINFO_RELEVANT (stmt_info);
     284       258629 :       save_live_p = STMT_VINFO_LIVE_P (stmt_info);
     285              : 
     286       258629 :       if (live_p && relevant == vect_unused_in_scope)
     287              :         {
     288          119 :           if (dump_enabled_p ())
     289           10 :             dump_printf_loc (MSG_NOTE, vect_location,
     290              :                              "vec_stmt_relevant_p: forcing live pattern stmt "
     291              :                              "relevant.\n");
     292              :           relevant = vect_used_only_live;
     293              :         }
     294              : 
     295       258629 :       if (dump_enabled_p ())
     296         3134 :         dump_printf_loc (MSG_NOTE, vect_location,
     297              :                          "mark relevant %d, live %d: %G", relevant, live_p,
     298              :                          stmt_info->stmt);
     299              :     }
     300              : 
     301      3331317 :   STMT_VINFO_LIVE_P (stmt_info) |= live_p;
     302      3331317 :   if (relevant > STMT_VINFO_RELEVANT (stmt_info))
     303      2995778 :     STMT_VINFO_RELEVANT (stmt_info) = relevant;
     304              : 
     305      3331317 :   if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
     306       335539 :       && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
     307              :     {
     308       334855 :       if (dump_enabled_p ())
     309        20912 :         dump_printf_loc (MSG_NOTE, vect_location,
     310              :                          "already marked relevant/live.\n");
     311              :       return;
     312              :     }
     313              : 
     314      2996462 :   worklist->safe_push (stmt_info);
     315              : }
     316              : 
     317              : 
     318              : /* Function is_simple_and_all_uses_invariant
     319              : 
     320              :    Return true if STMT_INFO is simple and all uses of it are invariant.  */
     321              : 
     322              : bool
     323       260911 : is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
     324              :                                   loop_vec_info loop_vinfo)
     325              : {
     326       260911 :   tree op;
     327       260911 :   ssa_op_iter iter;
     328              : 
     329       260911 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
     330       204046 :   if (!stmt)
     331              :     return false;
     332              : 
     333       211512 :   FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
     334              :     {
     335       210709 :       enum vect_def_type dt = vect_uninitialized_def;
     336              : 
     337       210709 :       if (!vect_is_simple_use (op, loop_vinfo, &dt))
     338              :         {
     339         5515 :           if (dump_enabled_p ())
     340           24 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     341              :                              "use not simple.\n");
     342       203243 :           return false;
     343              :         }
     344              : 
     345       205194 :       if (dt != vect_external_def && dt != vect_constant_def)
     346              :         return false;
     347              :     }
     348              :   return true;
     349              : }
     350              : 
     351              : /* Function vect_stmt_relevant_p.
     352              : 
     353              :    Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
     354              :    is "relevant for vectorization".
     355              : 
     356              :    A stmt is considered "relevant for vectorization" if:
     357              :    - it has uses outside the loop.
     358              :    - it has vdefs (it alters memory).
     359              :    - control stmts in the loop (except for the exit condition).
     360              : 
     361              :    CHECKME: what other side effects would the vectorizer allow?  */
     362              : 
     363              : static bool
     364      5393706 : vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
     365              :                       enum vect_relevant *relevant, bool *live_p)
     366              : {
     367      5393706 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     368      5393706 :   ssa_op_iter op_iter;
     369      5393706 :   imm_use_iterator imm_iter;
     370      5393706 :   use_operand_p use_p;
     371      5393706 :   def_operand_p def_p;
     372              : 
     373      5393706 :   *relevant = vect_unused_in_scope;
     374      5393706 :   *live_p = false;
     375              : 
     376              :   /* cond stmt other than loop exit cond.  */
     377      5393706 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
     378      5393706 :   if (is_ctrl_stmt (stmt)
     379       643012 :       && LOOP_VINFO_LOOP_IV_COND (loop_vinfo) != stmt
     380      5638511 :       && (!loop->inner || gimple_bb (stmt)->loop_father == loop))
     381       242829 :     *relevant = vect_used_in_scope;
     382              : 
     383              :   /* changing memory.  */
     384      5393706 :   if (gimple_code (stmt_info->stmt) != GIMPLE_PHI)
     385      4466756 :     if (gimple_vdef (stmt_info->stmt)
     386      3823744 :         && !gimple_clobber_p (stmt_info->stmt))
     387              :       {
     388       382933 :         if (dump_enabled_p ())
     389        29165 :           dump_printf_loc (MSG_NOTE, vect_location,
     390              :                            "vec_stmt_relevant_p: stmt has vdefs.\n");
     391       382933 :         *relevant = vect_used_in_scope;
     392       382933 :         if (! STMT_VINFO_DATA_REF (stmt_info)
     393       382933 :             && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
     394           26 :           LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
     395              :       }
     396              : 
     397              :   /* uses outside the loop.  */
     398     15148600 :   FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
     399              :     {
     400     11703740 :       FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
     401              :         {
     402      7342552 :           basic_block bb = gimple_bb (USE_STMT (use_p));
     403      7342552 :           if (!flow_bb_inside_loop_p (loop, bb))
     404              :             {
     405       277464 :               if (is_gimple_debug (USE_STMT (use_p)))
     406         1187 :                 continue;
     407              : 
     408       276277 :               if (dump_enabled_p ())
     409         6367 :                 dump_printf_loc (MSG_NOTE, vect_location,
     410              :                                  "vec_stmt_relevant_p: used out of loop.\n");
     411              : 
     412              :               /* We expect all such uses to be in the loop exit phis
     413              :                  (because of loop closed form)   */
     414       276277 :               gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
     415              : 
     416       276277 :               *live_p = true;
     417       276277 :               LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
     418              :             }
     419      4361188 :         }
     420              :     }
     421              : 
     422       260913 :   if (*live_p && *relevant == vect_unused_in_scope
     423      5654617 :       && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo))
     424              :     {
     425       260108 :       if (dump_enabled_p ())
     426         6225 :         dump_printf_loc (MSG_NOTE, vect_location,
     427              :                          "vec_stmt_relevant_p: stmt live but not relevant.\n");
     428       260108 :       *relevant = vect_used_only_live;
     429              :     }
     430              : 
     431      5393706 :   return (*live_p || *relevant);
     432              : }
     433              : 
     434              : 
     435              : /* Function exist_non_indexing_operands_for_use_p
     436              : 
     437              :    USE is one of the uses attached to STMT_INFO.  Check if USE is
     438              :    used in STMT_INFO for anything other than indexing an array.  */
     439              : 
     440              : static bool
     441      4469879 : exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
     442              : {
     443      4469879 :   tree operand;
     444              : 
     445              :   /* USE corresponds to some operand in STMT.  If there is no data
     446              :      reference in STMT, then any operand that corresponds to USE
     447              :      is not indexing an array.  */
     448      4469879 :   if (!STMT_VINFO_DATA_REF (stmt_info))
     449              :     return true;
     450              : 
     451              :   /* STMT has a data_ref. FORNOW this means that its of one of
     452              :      the following forms:
     453              :      -1- ARRAY_REF = var
     454              :      -2- var = ARRAY_REF
     455              :      (This should have been verified in analyze_data_refs).
     456              : 
     457              :      'var' in the second case corresponds to a def, not a use,
     458              :      so USE cannot correspond to any operands that are not used
     459              :      for array indexing.
     460              : 
     461              :      Therefore, all we need to check is if STMT falls into the
     462              :      first case, and whether var corresponds to USE.  */
     463              : 
     464      1524996 :   gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
     465      1508127 :   if (!assign || !gimple_assign_copy_p (assign))
     466              :     {
     467       818587 :       gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
     468        16869 :       if (call && gimple_call_internal_p (call))
     469              :         {
     470        16869 :           internal_fn ifn = gimple_call_internal_fn (call);
     471        16869 :           int mask_index = internal_fn_mask_index (ifn);
     472        16869 :           if (mask_index >= 0
     473        16869 :               && use == gimple_call_arg (call, mask_index))
     474              :             return true;
     475        11018 :           int els_index = internal_fn_else_index (ifn);
     476        11018 :           if (els_index >= 0
     477        11018 :               && use == gimple_call_arg (call, els_index))
     478              :             return true;
     479         9624 :           int stored_value_index = internal_fn_stored_value_index (ifn);
     480         9624 :           if (stored_value_index >= 0
     481         9624 :               && use == gimple_call_arg (call, stored_value_index))
     482              :             return true;
     483         7449 :           if (internal_gather_scatter_fn_p (ifn)
     484         7449 :               && use == gimple_call_arg (call, 1))
     485              :             return true;
     486              :         }
     487       809167 :       return false;
     488              :     }
     489              : 
     490       706409 :   if (TREE_CODE (gimple_assign_lhs (assign)) == SSA_NAME)
     491              :     return false;
     492       706409 :   operand = gimple_assign_rhs1 (assign);
     493       706409 :   if (TREE_CODE (operand) != SSA_NAME)
     494              :     return false;
     495              : 
     496       611761 :   if (operand == use)
     497       288802 :     return true;
     498              : 
     499              :   return false;
     500              : }
     501              : 
     502              : 
     503              : /*
     504              :    Function process_use.
     505              : 
     506              :    Inputs:
     507              :    - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
     508              :    - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
     509              :      that defined USE.  This is done by calling mark_relevant and passing it
     510              :      the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
     511              :    - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
     512              :      be performed.
     513              : 
     514              :    Outputs:
     515              :    Generally, LIVE_P and RELEVANT are used to define the liveness and
     516              :    relevance info of the DEF_STMT of this USE:
     517              :        STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
     518              :        STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
     519              :    Exceptions:
     520              :    - case 1: If USE is used only for address computations (e.g. array indexing),
     521              :    which does not need to be directly vectorized, then the liveness/relevance
     522              :    of the respective DEF_STMT is left unchanged.
     523              :    - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
     524              :    we skip DEF_STMT cause it had already been processed.
     525              :    - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
     526              :    "relevant" will be modified accordingly.
     527              : 
     528              :    Return true if everything is as expected. Return false otherwise.  */
     529              : 
     530              : static opt_result
     531      4522785 : process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
     532              :              enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
     533              :              bool force)
     534              : {
     535      4522785 :   stmt_vec_info dstmt_vinfo;
     536      4522785 :   enum vect_def_type dt;
     537              : 
     538              :   /* case 1: we are only interested in uses that need to be vectorized.  Uses
     539              :      that are used for address computation are not considered relevant.  */
     540      4522785 :   if (!force && !exist_non_indexing_operands_for_use_p (use, stmt_vinfo))
     541      1226774 :     return opt_result::success ();
     542              : 
     543      3296011 :   if (!vect_is_simple_use (use, loop_vinfo, &dt, &dstmt_vinfo))
     544        36440 :     return opt_result::failure_at (stmt_vinfo->stmt,
     545              :                                    "not vectorized:"
     546              :                                    " unsupported use in stmt.\n");
     547              : 
     548      3259571 :   if (!dstmt_vinfo)
     549       618114 :     return opt_result::success ();
     550              : 
     551      2641457 :   basic_block def_bb = gimple_bb (dstmt_vinfo->stmt);
     552      2641457 :   basic_block bb = gimple_bb (stmt_vinfo->stmt);
     553              : 
     554              :   /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
     555              :      We have to force the stmt live since the epilogue loop needs it to
     556              :      continue computing the reduction.  */
     557      2641457 :   if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
     558       278083 :       && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
     559        86022 :       && gimple_code (dstmt_vinfo->stmt) != GIMPLE_PHI
     560        86022 :       && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
     561      2727479 :       && bb->loop_father == def_bb->loop_father)
     562              :     {
     563        86022 :       if (dump_enabled_p ())
     564         4076 :         dump_printf_loc (MSG_NOTE, vect_location,
     565              :                          "reduc-stmt defining reduc-phi in the same nest.\n");
     566        86022 :       vect_mark_relevant (worklist, dstmt_vinfo, relevant, true);
     567        86022 :       return opt_result::success ();
     568              :     }
     569              : 
     570              :   /* case 3a: outer-loop stmt defining an inner-loop stmt:
     571              :         outer-loop-header-bb:
     572              :                 d = dstmt_vinfo
     573              :         inner-loop:
     574              :                 stmt # use (d)
     575              :         outer-loop-tail-bb:
     576              :                 ...               */
     577      2555435 :   if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
     578              :     {
     579         2141 :       if (dump_enabled_p ())
     580          324 :         dump_printf_loc (MSG_NOTE, vect_location,
     581              :                          "outer-loop def-stmt defining inner-loop stmt.\n");
     582              : 
     583         2141 :       switch (relevant)
     584              :         {
     585            0 :         case vect_unused_in_scope:
     586            0 :           relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
     587              :                       vect_used_in_scope : vect_unused_in_scope;
     588              :           break;
     589              : 
     590          767 :         case vect_used_in_outer_by_reduction:
     591          767 :           gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
     592              :           relevant = vect_used_by_reduction;
     593              :           break;
     594              : 
     595         1088 :         case vect_used_in_outer:
     596         1088 :           gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
     597              :           relevant = vect_used_in_scope;
     598              :           break;
     599              : 
     600              :         case vect_used_in_scope:
     601              :           break;
     602              : 
     603            0 :         default:
     604            0 :           gcc_unreachable ();
     605              :         }
     606              :     }
     607              : 
     608              :   /* case 3b: inner-loop stmt defining an outer-loop stmt:
     609              :         outer-loop-header-bb:
     610              :                 ...
     611              :         inner-loop:
     612              :                 d = dstmt_vinfo
     613              :         outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
     614              :                 stmt # use (d)          */
     615      2553294 :   else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
     616              :     {
     617         2073 :       if (dump_enabled_p ())
     618          629 :         dump_printf_loc (MSG_NOTE, vect_location,
     619              :                          "inner-loop def-stmt defining outer-loop stmt.\n");
     620              : 
     621         2073 :       switch (relevant)
     622              :         {
     623            0 :         case vect_unused_in_scope:
     624      2369599 :           relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
     625            0 :             || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
     626              :                       vect_used_in_outer_by_reduction : vect_unused_in_scope;
     627              :           break;
     628              : 
     629              :         case vect_used_by_reduction:
     630              :         case vect_used_only_live:
     631              :           relevant = vect_used_in_outer_by_reduction;
     632              :           break;
     633              : 
     634              :         case vect_used_in_scope:
     635      2369599 :           relevant = vect_used_in_outer;
     636              :           break;
     637              : 
     638            0 :         default:
     639            0 :           gcc_unreachable ();
     640              :         }
     641              :     }
     642              :   /* We are also not interested in uses on loop PHI backedges that are
     643              :      inductions.  Otherwise we'll needlessly vectorize the IV increment
     644              :      and cause hybrid SLP for SLP inductions.  */
     645      2551221 :   else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
     646       188847 :            && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
     647      2737057 :            && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt,
     648              :                                       loop_latch_edge (bb->loop_father))
     649              :                == use))
     650              :     {
     651       185836 :       if (dump_enabled_p ())
     652         5307 :         dump_printf_loc (MSG_NOTE, vect_location,
     653              :                          "induction value on backedge.\n");
     654       185836 :       return opt_result::success ();
     655              :     }
     656              : 
     657      2369599 :   vect_mark_relevant (worklist, dstmt_vinfo, relevant, false);
     658      2369599 :   return opt_result::success ();
     659              : }
     660              : 
     661              : 
     662              : /* Function vect_mark_stmts_to_be_vectorized.
     663              : 
     664              :    Not all stmts in the loop need to be vectorized. For example:
     665              : 
     666              :      for i...
     667              :        for j...
     668              :    1.    T0 = i + j
     669              :    2.    T1 = a[T0]
     670              : 
     671              :    3.    j = j + 1
     672              : 
     673              :    Stmt 1 and 3 do not need to be vectorized, because loop control and
     674              :    addressing of vectorized data-refs are handled differently.
     675              : 
     676              :    This pass detects such stmts.  */
     677              : 
     678              : opt_result
     679       456698 : vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
     680              : {
     681       456698 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     682       456698 :   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
     683       456698 :   unsigned int nbbs = loop->num_nodes;
     684       456698 :   gimple_stmt_iterator si;
     685       456698 :   unsigned int i;
     686       456698 :   basic_block bb;
     687       456698 :   bool live_p;
     688       456698 :   enum vect_relevant relevant;
     689              : 
     690       456698 :   DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
     691              : 
     692       456698 :   auto_vec<stmt_vec_info, 64> worklist;
     693              : 
     694              :   /* 1. Init worklist.  */
     695      1549252 :   for (i = 0; i < nbbs; i++)
     696              :     {
     697      1103676 :       bb = bbs[i];
     698      2258930 :       for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
     699              :         {
     700      2332462 :           if (virtual_operand_p (gimple_phi_result (gsi_stmt (si))))
     701       239281 :             continue;
     702       926950 :           stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
     703       926950 :           if (dump_enabled_p ())
     704        43771 :             dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G",
     705              :                              phi_info->stmt);
     706              : 
     707       926950 :           if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
     708              :             {
     709        47985 :               if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
     710        10977 :                 return opt_result::failure_at
     711        10977 :                   (*si, "not vectorized: unhandled relevant PHI: %G", *si);
     712        37008 :               vect_mark_relevant (&worklist, phi_info, relevant, live_p);
     713              :             }
     714              :         }
     715      8782171 :       for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si))
     716              :         {
     717      7689617 :           gimple *stmt = gsi_stmt (si);
     718      7689617 :           if (is_gimple_debug (stmt))
     719      3222716 :             continue;
     720      4466901 :           stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
     721      4466901 :           if (dump_enabled_p ())
     722       236106 :               dump_printf_loc (MSG_NOTE, vect_location,
     723              :                                "init: stmt relevant? %G", stmt);
     724              : 
     725      4466901 :           if (gimple_get_lhs (stmt) == NULL_TREE
     726       649271 :               && !is_a <gcond *> (stmt)
     727      4473160 :               && !is_a <gcall *> (stmt))
     728          145 :             return opt_result::failure_at
     729          145 :                 (stmt, "not vectorized: irregular stmt: %G", stmt);
     730              : 
     731      4466756 :           if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
     732       838688 :             vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
     733              :         }
     734              :     }
     735              : 
     736              :   /* 2. Process_worklist */
     737      3312417 :   while (worklist.length () > 0)
     738              :     {
     739      2903283 :       use_operand_p use_p;
     740      2903283 :       ssa_op_iter iter;
     741              : 
     742      2903283 :       stmt_vec_info stmt_vinfo = worklist.pop ();
     743      2903283 :       if (dump_enabled_p ())
     744       151147 :         dump_printf_loc (MSG_NOTE, vect_location,
     745              :                          "worklist: examine stmt: %G", stmt_vinfo->stmt);
     746              : 
     747              :       /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
     748              :          (DEF_STMT) as relevant/irrelevant according to the relevance property
     749              :          of STMT.  */
     750      2903283 :       relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
     751              : 
     752              :       /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
     753              :          propagated as is to the DEF_STMTs of its USEs.
     754              : 
     755              :          One exception is when STMT has been identified as defining a reduction
     756              :          variable; in this case we set the relevance to vect_used_by_reduction.
     757              :          This is because we distinguish between two kinds of relevant stmts -
     758              :          those that are used by a reduction computation, and those that are
     759              :          (also) used by a regular computation.  This allows us later on to
     760              :          identify stmts that are used solely by a reduction, and therefore the
     761              :          order of the results that they produce does not have to be kept.  */
     762              : 
     763      2903283 :       switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
     764              :         {
     765       174169 :           case vect_reduction_def:
     766       174169 :             gcc_assert (relevant != vect_unused_in_scope);
     767       174169 :             if (relevant != vect_unused_in_scope
     768       174169 :                 && relevant != vect_used_in_scope
     769       174169 :                 && relevant != vect_used_by_reduction
     770       174169 :                 && relevant != vect_used_only_live)
     771            0 :               return opt_result::failure_at
     772            0 :                 (stmt_vinfo->stmt, "unsupported use of reduction.\n");
     773              :             break;
     774              : 
     775         2092 :           case vect_nested_cycle:
     776         2092 :             if (relevant != vect_unused_in_scope
     777         2092 :                 && relevant != vect_used_in_outer_by_reduction
     778         1497 :                 && relevant != vect_used_in_outer)
     779            2 :               return opt_result::failure_at
     780            2 :                 (stmt_vinfo->stmt, "unsupported use of nested cycle.\n");
     781              :             break;
     782              : 
     783         1209 :           case vect_double_reduction_def:
     784         1209 :             if (relevant != vect_unused_in_scope
     785         1209 :                 && relevant != vect_used_by_reduction
     786          409 :                 && relevant != vect_used_only_live)
     787            0 :               return opt_result::failure_at
     788            0 :                 (stmt_vinfo->stmt, "unsupported use of double reduction.\n");
     789              :             break;
     790              : 
     791              :           default:
     792              :             break;
     793              :         }
     794              : 
     795      2903281 :       if (is_pattern_stmt_p (stmt_vinfo))
     796              :         {
     797              :           /* Pattern statements are not inserted into the code, so
     798              :              FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
     799              :              have to scan the RHS or function arguments instead.  */
     800       661357 :           if (gassign *assign = dyn_cast <gassign *> (stmt_vinfo->stmt))
     801              :             {
     802       428127 :               enum tree_code rhs_code = gimple_assign_rhs_code (assign);
     803       428127 :               tree op = gimple_assign_rhs1 (assign);
     804              : 
     805       428127 :               i = 1;
     806       428127 :               if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
     807              :                 {
     808            0 :                   opt_result res
     809            0 :                     = process_use (stmt_vinfo, TREE_OPERAND (op, 0),
     810              :                                    loop_vinfo, relevant, &worklist, false);
     811            0 :                   if (!res)
     812            0 :                     return res;
     813            0 :                   res = process_use (stmt_vinfo, TREE_OPERAND (op, 1),
     814              :                                      loop_vinfo, relevant, &worklist, false);
     815            0 :                   if (!res)
     816            0 :                     return res;
     817       428127 :                   i = 2;
     818              :                 }
     819      1233634 :               for (; i < gimple_num_ops (assign); i++)
     820              :                 {
     821       809538 :                   op = gimple_op (assign, i);
     822       809538 :                   if (TREE_CODE (op) == SSA_NAME)
     823              :                     {
     824       615045 :                       opt_result res
     825       615045 :                         = process_use (stmt_vinfo, op, loop_vinfo, relevant,
     826              :                                        &worklist, false);
     827       615045 :                       if (!res)
     828         4031 :                         return res;
     829              :                     }
     830              :                  }
     831              :             }
     832       233230 :           else if (gcond *cond = dyn_cast <gcond *> (stmt_vinfo->stmt))
     833              :             {
     834       226102 :               tree_code rhs_code = gimple_cond_code (cond);
     835       226102 :               gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
     836       226102 :               opt_result res
     837       226102 :                 = process_use (stmt_vinfo, gimple_cond_lhs (cond),
     838              :                                loop_vinfo, relevant, &worklist, false);
     839       226102 :               if (!res)
     840        36442 :                 return res;
     841       226102 :               res = process_use (stmt_vinfo, gimple_cond_rhs (cond),
     842              :                                 loop_vinfo, relevant, &worklist, false);
     843       226102 :               if (!res)
     844            0 :                 return res;
     845              :             }
     846         7128 :           else if (gcall *call = dyn_cast <gcall *> (stmt_vinfo->stmt))
     847              :             {
     848        33335 :               for (i = 0; i < gimple_call_num_args (call); i++)
     849              :                 {
     850        26207 :                   tree arg = gimple_call_arg (call, i);
     851        26207 :                   opt_result res
     852        26207 :                     = process_use (stmt_vinfo, arg, loop_vinfo, relevant,
     853              :                                    &worklist, false);
     854        26207 :                   if (!res)
     855            0 :                     return res;
     856              :                 }
     857              :             }
     858              :           else
     859            0 :             gcc_unreachable ();
     860              :         }
     861              :       else
     862      7841997 :         FOR_EACH_PHI_OR_STMT_USE (use_p, stmt_vinfo->stmt, iter, SSA_OP_USE)
     863              :           {
     864      3376423 :             tree op = USE_FROM_PTR (use_p);
     865      3376423 :             opt_result res
     866      3376423 :               = process_use (stmt_vinfo, op, loop_vinfo, relevant,
     867              :                              &worklist, false);
     868      3376423 :             if (!res)
     869        18274 :               return res;
     870              :           }
     871              : 
     872      2880976 :       if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
     873              :         {
     874        52906 :           gather_scatter_info gs_info;
     875        52906 :           if (!vect_check_gather_scatter (stmt_vinfo,
     876              :                                           STMT_VINFO_VECTYPE (stmt_vinfo),
     877              :                                           loop_vinfo, &gs_info))
     878            0 :             gcc_unreachable ();
     879        52906 :           opt_result res
     880        52906 :             = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
     881              :                            &worklist, true);
     882        52906 :           if (!res)
     883              :             {
     884        14135 :               if (fatal)
     885        14135 :                 *fatal = false;
     886        14135 :               return res;
     887              :             }
     888              :         }
     889              :     } /* while worklist */
     890              : 
     891       409134 :   return opt_result::success ();
     892       456698 : }
     893              : 
     894              : /* Function vect_model_simple_cost.
     895              : 
     896              :    Models cost for simple operations, i.e. those that only emit N operations
     897              :    of the same KIND.  */
     898              : 
     899              : static void
     900       676982 : vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node,
     901              :                         stmt_vector_for_cost *cost_vec,
     902              :                         vect_cost_for_stmt kind = vector_stmt)
     903              : {
     904       676982 :   int inside_cost = 0, prologue_cost = 0;
     905              : 
     906       676982 :   gcc_assert (cost_vec != NULL);
     907              : 
     908       676982 :   n *= vect_get_num_copies (vinfo, node);
     909              : 
     910              :   /* Pass the inside-of-loop statements to the target-specific cost model.  */
     911       676982 :   inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
     912              : 
     913       676982 :   if (dump_enabled_p ())
     914        36163 :     dump_printf_loc (MSG_NOTE, vect_location,
     915              :                      "vect_model_simple_cost: inside_cost = %d, "
     916              :                      "prologue_cost = %d .\n", inside_cost, prologue_cost);
     917       676982 : }
     918              : 
     919              : 
     920              : /* Model cost for type demotion and promotion operations.  PWR is
     921              :    normally zero for single-step promotions and demotions.  It will be
     922              :    one if two-step promotion/demotion is required, and so on.  NCOPIES
     923              :    is the number of vector results (and thus number of instructions)
     924              :    for the narrowest end of the operation chain.  Each additional
     925              :    step doubles the number of instructions required.  If WIDEN_ARITH
     926              :    is true the stmt is doing widening arithmetic.  */
     927              : 
     928              : static void
     929        70426 : vect_model_promotion_demotion_cost (slp_tree slp_node,
     930              :                                     unsigned int ncopies, int pwr,
     931              :                                     stmt_vector_for_cost *cost_vec,
     932              :                                     bool widen_arith)
     933              : {
     934        70426 :   int i;
     935        70426 :   int inside_cost = 0, prologue_cost = 0;
     936              : 
     937       163917 :   for (i = 0; i < pwr + 1; i++)
     938              :     {
     939       185047 :       inside_cost += record_stmt_cost (cost_vec, ncopies,
     940              :                                        widen_arith
     941              :                                        ? vector_stmt : vec_promote_demote,
     942              :                                        slp_node, 0, vect_body);
     943        93491 :       ncopies *= 2;
     944              :     }
     945              : 
     946        70426 :   if (dump_enabled_p ())
     947         6530 :     dump_printf_loc (MSG_NOTE, vect_location,
     948              :                      "vect_model_promotion_demotion_cost: inside_cost = %d, "
     949              :                      "prologue_cost = %d .\n", inside_cost, prologue_cost);
     950        70426 : }
     951              : 
     952              : /* Returns true if the current function returns DECL.  */
     953              : 
     954              : static bool
     955       572662 : cfun_returns (tree decl)
     956              : {
     957       572662 :   edge_iterator ei;
     958       572662 :   edge e;
     959      1127797 :   FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
     960              :     {
     961      1133916 :       greturn *ret = safe_dyn_cast <greturn *> (*gsi_last_bb (e->src));
     962       566958 :       if (!ret)
     963            0 :         continue;
     964       566958 :       if (gimple_return_retval (ret) == decl)
     965              :         return true;
     966              :       /* We often end up with an aggregate copy to the result decl,
     967              :          handle that case as well.  First skip intermediate clobbers
     968              :          though.  */
     969              :       gimple *def = ret;
     970      1707910 :       do
     971              :         {
     972      3415820 :           def = SSA_NAME_DEF_STMT (gimple_vuse (def));
     973              :         }
     974      1707910 :       while (gimple_clobber_p (def));
     975       555822 :       if (is_a <gassign *> (def)
     976        63746 :           && gimple_assign_lhs (def) == gimple_return_retval (ret)
     977       562762 :           && gimple_assign_rhs1 (def) == decl)
     978              :         return true;
     979              :     }
     980              :   return false;
     981              : }
     982              : 
     983              : /* Calculate cost of DR's memory access.  */
     984              : void
     985      1048810 : vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
     986              :                      int ncopies, dr_alignment_support alignment_support_scheme,
     987              :                      int misalignment,
     988              :                      unsigned int *inside_cost,
     989              :                      stmt_vector_for_cost *body_cost_vec)
     990              : {
     991      1048810 :   tree vectype
     992      1048810 :     = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
     993      1048810 :   switch (alignment_support_scheme)
     994              :     {
     995       576875 :     case dr_aligned:
     996       576875 :       {
     997       576875 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
     998              :                                           vector_store, stmt_info, slp_node,
     999              :                                           vectype, 0, vect_body);
    1000              : 
    1001       576875 :         if (dump_enabled_p ())
    1002        15580 :           dump_printf_loc (MSG_NOTE, vect_location,
    1003              :                            "vect_model_store_cost: aligned.\n");
    1004              :         break;
    1005              :       }
    1006              : 
    1007       471935 :     case dr_unaligned_supported:
    1008       471935 :       {
    1009              :         /* Here, we assign an additional cost for the unaligned store.  */
    1010       471935 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
    1011              :                                           unaligned_store, stmt_info, slp_node,
    1012              :                                           vectype, misalignment, vect_body);
    1013       471935 :         if (dump_enabled_p ())
    1014        14187 :           dump_printf_loc (MSG_NOTE, vect_location,
    1015              :                            "vect_model_store_cost: unaligned supported by "
    1016              :                            "hardware.\n");
    1017              :         break;
    1018              :       }
    1019              : 
    1020            0 :     case dr_unaligned_unsupported:
    1021            0 :       {
    1022            0 :         *inside_cost = VECT_MAX_COST;
    1023              : 
    1024            0 :         if (dump_enabled_p ())
    1025            0 :           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1026              :                            "vect_model_store_cost: unsupported access.\n");
    1027              :         break;
    1028              :       }
    1029              : 
    1030            0 :     default:
    1031            0 :       gcc_unreachable ();
    1032              :     }
    1033      1048810 : }
    1034              : 
    1035              : /* Calculate cost of DR's memory access.  */
    1036              : void
    1037       996373 : vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
    1038              :                     int ncopies, dr_alignment_support alignment_support_scheme,
    1039              :                     int misalignment,
    1040              :                     bool add_realign_cost, unsigned int *inside_cost,
    1041              :                     unsigned int *prologue_cost,
    1042              :                     stmt_vector_for_cost *prologue_cost_vec,
    1043              :                     stmt_vector_for_cost *body_cost_vec,
    1044              :                     bool record_prologue_costs)
    1045              : {
    1046       996373 :   tree vectype
    1047       996373 :     = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
    1048       996373 :   switch (alignment_support_scheme)
    1049              :     {
    1050       572113 :     case dr_aligned:
    1051       572113 :       {
    1052       572113 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
    1053              :                                           stmt_info, slp_node, vectype,
    1054              :                                           0, vect_body);
    1055              : 
    1056       572113 :         if (dump_enabled_p ())
    1057        20333 :           dump_printf_loc (MSG_NOTE, vect_location,
    1058              :                            "vect_model_load_cost: aligned.\n");
    1059              : 
    1060              :         break;
    1061              :       }
    1062       363096 :     case dr_unaligned_supported:
    1063       363096 :       {
    1064              :         /* Here, we assign an additional cost for the unaligned load.  */
    1065       363096 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
    1066              :                                           unaligned_load, stmt_info, slp_node,
    1067              :                                           vectype, misalignment, vect_body);
    1068              : 
    1069       363096 :         if (dump_enabled_p ())
    1070        24546 :           dump_printf_loc (MSG_NOTE, vect_location,
    1071              :                            "vect_model_load_cost: unaligned supported by "
    1072              :                            "hardware.\n");
    1073              : 
    1074              :         break;
    1075              :       }
    1076            0 :     case dr_explicit_realign:
    1077            0 :       {
    1078            0 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
    1079              :                                           vector_load, stmt_info, slp_node,
    1080              :                                           vectype, 0, vect_body);
    1081            0 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
    1082              :                                           vec_perm, stmt_info, slp_node,
    1083              :                                           vectype, 0, vect_body);
    1084              : 
    1085              :         /* FIXME: If the misalignment remains fixed across the iterations of
    1086              :            the containing loop, the following cost should be added to the
    1087              :            prologue costs.  */
    1088            0 :         if (targetm.vectorize.builtin_mask_for_load)
    1089            0 :           *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
    1090              :                                             stmt_info, slp_node, vectype,
    1091              :                                             0, vect_body);
    1092              : 
    1093            0 :         if (dump_enabled_p ())
    1094            0 :           dump_printf_loc (MSG_NOTE, vect_location,
    1095              :                            "vect_model_load_cost: explicit realign\n");
    1096              : 
    1097              :         break;
    1098              :       }
    1099            0 :     case dr_explicit_realign_optimized:
    1100            0 :       {
    1101            0 :         if (dump_enabled_p ())
    1102            0 :           dump_printf_loc (MSG_NOTE, vect_location,
    1103              :                            "vect_model_load_cost: unaligned software "
    1104              :                            "pipelined.\n");
    1105              : 
    1106              :         /* Unaligned software pipeline has a load of an address, an initial
    1107              :            load, and possibly a mask operation to "prime" the loop.  However,
    1108              :            if this is an access in a group of loads, which provide grouped
    1109              :            access, then the above cost should only be considered for one
    1110              :            access in the group.  Inside the loop, there is a load op
    1111              :            and a realignment op.  */
    1112              : 
    1113            0 :         if (add_realign_cost && record_prologue_costs)
    1114              :           {
    1115            0 :             *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
    1116              :                                                 vector_stmt, stmt_info,
    1117              :                                                 slp_node, vectype,
    1118              :                                                 0, vect_prologue);
    1119            0 :             if (targetm.vectorize.builtin_mask_for_load)
    1120            0 :               *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
    1121              :                                                   vector_stmt, stmt_info,
    1122              :                                                   slp_node, vectype,
    1123              :                                                   0, vect_prologue);
    1124              :           }
    1125              : 
    1126            0 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
    1127              :                                           stmt_info, slp_node, vectype,
    1128              :                                           0, vect_body);
    1129            0 :         *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
    1130              :                                           stmt_info, slp_node, vectype,
    1131              :                                           0, vect_body);
    1132              : 
    1133            0 :         if (dump_enabled_p ())
    1134            0 :           dump_printf_loc (MSG_NOTE, vect_location,
    1135              :                            "vect_model_load_cost: explicit realign optimized"
    1136              :                            "\n");
    1137              : 
    1138              :         break;
    1139              :       }
    1140              : 
    1141        61164 :     case dr_unaligned_unsupported:
    1142        61164 :       {
    1143        61164 :         *inside_cost = VECT_MAX_COST;
    1144              : 
    1145        61164 :         if (dump_enabled_p ())
    1146          116 :           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1147              :                            "vect_model_load_cost: unsupported access.\n");
    1148              :         break;
    1149              :       }
    1150              : 
    1151            0 :     default:
    1152            0 :       gcc_unreachable ();
    1153              :     }
    1154       996373 : }
    1155              : 
    1156              : /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
    1157              :    the loop preheader for the vectorized stmt STMT_VINFO.  */
    1158              : 
    1159              : static void
    1160         6760 : vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
    1161              :                     gimple_stmt_iterator *gsi)
    1162              : {
    1163         6760 :   if (gsi)
    1164         3480 :     vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
    1165              :   else
    1166         3280 :     vinfo->insert_on_entry (stmt_vinfo, new_stmt);
    1167              : 
    1168         6760 :   if (dump_enabled_p ())
    1169         1860 :     dump_printf_loc (MSG_NOTE, vect_location,
    1170              :                      "created new init_stmt: %G", new_stmt);
    1171         6760 : }
    1172              : 
    1173              : /* Function vect_init_vector.
    1174              : 
    1175              :    Insert a new stmt (INIT_STMT) that initializes a new variable of type
    1176              :    TYPE with the value VAL.  If TYPE is a vector type and VAL does not have
    1177              :    vector type a vector with all elements equal to VAL is created first.
    1178              :    Place the initialization at GSI if it is not NULL.  Otherwise, place the
    1179              :    initialization at the loop preheader.
    1180              :    Return the DEF of INIT_STMT.
    1181              :    It will be used in the vectorization of STMT_INFO.  */
    1182              : 
    1183              : tree
    1184         5042 : vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
    1185              :                   gimple_stmt_iterator *gsi)
    1186              : {
    1187         5042 :   gimple *init_stmt;
    1188         5042 :   tree new_temp;
    1189              : 
    1190              :   /* We abuse this function to push sth to a SSA name with initial 'val'.  */
    1191         5042 :   if (! useless_type_conversion_p (type, TREE_TYPE (val)))
    1192              :     {
    1193         1316 :       gcc_assert (VECTOR_TYPE_P (type));
    1194         1316 :       if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
    1195              :         {
    1196              :           /* Scalar boolean value should be transformed into
    1197              :              all zeros or all ones value before building a vector.  */
    1198           11 :           if (VECTOR_BOOLEAN_TYPE_P (type))
    1199              :             {
    1200            3 :               tree true_val = build_all_ones_cst (TREE_TYPE (type));
    1201            3 :               tree false_val = build_zero_cst (TREE_TYPE (type));
    1202              : 
    1203            3 :               if (CONSTANT_CLASS_P (val))
    1204            0 :                 val = integer_zerop (val) ? false_val : true_val;
    1205              :               else
    1206              :                 {
    1207            3 :                   new_temp = make_ssa_name (TREE_TYPE (type));
    1208            3 :                   init_stmt = gimple_build_assign (new_temp, COND_EXPR,
    1209              :                                                    val, true_val, false_val);
    1210            3 :                   vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
    1211            3 :                   val = new_temp;
    1212              :                 }
    1213              :             }
    1214              :           else
    1215              :             {
    1216            8 :               gimple_seq stmts = NULL;
    1217            8 :               if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
    1218            8 :                 val = gimple_build (&stmts, VIEW_CONVERT_EXPR,
    1219            8 :                                     TREE_TYPE (type), val);
    1220              :               else
    1221              :                 /* ???  Condition vectorization expects us to do
    1222              :                    promotion of invariant/external defs.  */
    1223            0 :                 val = gimple_convert (&stmts, TREE_TYPE (type), val);
    1224           16 :               for (gimple_stmt_iterator gsi2 = gsi_start (stmts);
    1225           16 :                    !gsi_end_p (gsi2); )
    1226              :                 {
    1227            8 :                   init_stmt = gsi_stmt (gsi2);
    1228            8 :                   gsi_remove (&gsi2, false);
    1229            8 :                   vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
    1230              :                 }
    1231              :             }
    1232              :         }
    1233         1316 :       val = build_vector_from_val (type, val);
    1234              :     }
    1235              : 
    1236         5042 :   new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
    1237         5042 :   init_stmt = gimple_build_assign (new_temp, val);
    1238         5042 :   vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
    1239         5042 :   return new_temp;
    1240              : }
    1241              : 
    1242              : 
    1243              : /* Get vectorized definitions for OP0 and OP1.  */
    1244              : 
    1245              : void
    1246       194630 : vect_get_vec_defs (vec_info *, slp_tree slp_node,
    1247              :                    bool op0, vec<tree> *vec_oprnds0,
    1248              :                    bool op1, vec<tree> *vec_oprnds1,
    1249              :                    bool op2, vec<tree> *vec_oprnds2,
    1250              :                    bool op3, vec<tree> *vec_oprnds3)
    1251              : {
    1252       194630 :   if (op0)
    1253       193022 :     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
    1254       194630 :   if (op1)
    1255       143175 :     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
    1256       194630 :   if (op2)
    1257         9475 :     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
    1258       194630 :   if (op3)
    1259            0 :     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
    1260       194630 : }
    1261              : 
    1262              : /* Helper function called by vect_finish_replace_stmt and
    1263              :    vect_finish_stmt_generation.  Set the location of the new
    1264              :    statement and create and return a stmt_vec_info for it.  */
    1265              : 
    1266              : static void
    1267      1436282 : vect_finish_stmt_generation_1 (vec_info *,
    1268              :                                stmt_vec_info stmt_info, gimple *vec_stmt)
    1269              : {
    1270      1436282 :   if (dump_enabled_p ())
    1271       151648 :     dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
    1272              : 
    1273      1436282 :   if (stmt_info)
    1274              :     {
    1275      1407550 :       gimple_set_location (vec_stmt, gimple_location (stmt_info->stmt));
    1276              : 
    1277              :       /* While EH edges will generally prevent vectorization, stmt might
    1278              :          e.g. be in a must-not-throw region.  Ensure newly created stmts
    1279              :          that could throw are part of the same region.  */
    1280      1407550 :       int lp_nr = lookup_stmt_eh_lp (stmt_info->stmt);
    1281      1407550 :       if (lp_nr != 0 && stmt_could_throw_p (cfun, vec_stmt))
    1282           48 :         add_stmt_to_eh_lp (vec_stmt, lp_nr);
    1283              :     }
    1284              :   else
    1285        28732 :     gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
    1286      1436282 : }
    1287              : 
    1288              : /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
    1289              :    which sets the same scalar result as STMT_INFO did.  Create and return a
    1290              :    stmt_vec_info for VEC_STMT.  */
    1291              : 
    1292              : void
    1293          977 : vect_finish_replace_stmt (vec_info *vinfo,
    1294              :                           stmt_vec_info stmt_info, gimple *vec_stmt)
    1295              : {
    1296          977 :   gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
    1297          977 :   gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
    1298              : 
    1299          977 :   gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
    1300          977 :   gsi_replace (&gsi, vec_stmt, true);
    1301              : 
    1302          977 :   vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
    1303          977 : }
    1304              : 
    1305              : /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
    1306              :    before *GSI.  Create and return a stmt_vec_info for VEC_STMT.  */
    1307              : 
    1308              : void
    1309      1435305 : vect_finish_stmt_generation (vec_info *vinfo,
    1310              :                              stmt_vec_info stmt_info, gimple *vec_stmt,
    1311              :                              gimple_stmt_iterator *gsi)
    1312              : {
    1313      1435305 :   gcc_assert (!stmt_info || gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
    1314              : 
    1315      1435305 :   if (!gsi_end_p (*gsi)
    1316      2869665 :       && gimple_has_mem_ops (vec_stmt))
    1317              :     {
    1318      1434360 :       gimple *at_stmt = gsi_stmt (*gsi);
    1319      1434360 :       tree vuse = gimple_vuse (at_stmt);
    1320      1428099 :       if (vuse && TREE_CODE (vuse) == SSA_NAME)
    1321              :         {
    1322      1281663 :           tree vdef = gimple_vdef (at_stmt);
    1323      1281663 :           gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
    1324      1281663 :           gimple_set_modified (vec_stmt, true);
    1325              :           /* If we have an SSA vuse and insert a store, update virtual
    1326              :              SSA form to avoid triggering the renamer.  Do so only
    1327              :              if we can easily see all uses - which is what almost always
    1328              :              happens with the way vectorized stmts are inserted.  */
    1329       759307 :           if ((vdef && TREE_CODE (vdef) == SSA_NAME)
    1330      2040934 :               && ((is_gimple_assign (vec_stmt)
    1331       758417 :                    && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
    1332        64780 :                   || (is_gimple_call (vec_stmt)
    1333          854 :                       && (!(gimple_call_flags (vec_stmt)
    1334          854 :                             & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
    1335            3 :                           || (gimple_call_lhs (vec_stmt)
    1336            3 :                               && !is_gimple_reg (gimple_call_lhs (vec_stmt)))))))
    1337              :             {
    1338       695342 :               tree new_vdef = copy_ssa_name (vuse, vec_stmt);
    1339       695342 :               gimple_set_vdef (vec_stmt, new_vdef);
    1340       695342 :               SET_USE (gimple_vuse_op (at_stmt), new_vdef);
    1341              :             }
    1342              :         }
    1343              :     }
    1344      1435305 :   gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
    1345      1435305 :   vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
    1346      1435305 : }
    1347              : 
    1348              : /* We want to vectorize a call to combined function CFN with function
    1349              :    decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
    1350              :    as the types of all inputs.  Check whether this is possible using
    1351              :    an internal function, returning its code if so or IFN_LAST if not.  */
    1352              : 
    1353              : static internal_fn
    1354        18575 : vectorizable_internal_function (combined_fn cfn, tree fndecl,
    1355              :                                 tree vectype_out, tree vectype_in)
    1356              : {
    1357        18575 :   internal_fn ifn;
    1358        18575 :   if (internal_fn_p (cfn))
    1359        14728 :     ifn = as_internal_fn (cfn);
    1360              :   else
    1361         3847 :     ifn = associated_internal_fn (fndecl);
    1362        18575 :   if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
    1363              :     {
    1364        14759 :       const direct_internal_fn_info &info = direct_internal_fn (ifn);
    1365        14759 :       if (info.vectorizable)
    1366              :         {
    1367        14759 :           bool same_size_p = TYPE_SIZE (vectype_in) == TYPE_SIZE (vectype_out);
    1368        14759 :           tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
    1369        14759 :           tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
    1370              : 
    1371              :           /* The type size of both the vectype_in and vectype_out should be
    1372              :              exactly the same when vectype_out isn't participating the optab.
    1373              :              While there is no restriction for type size when vectype_out
    1374              :              is part of the optab query.  */
    1375        14759 :           if (type0 != vectype_out && type1 != vectype_out && !same_size_p)
    1376        18575 :             return IFN_LAST;
    1377              : 
    1378        14722 :           if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
    1379              :                                               OPTIMIZE_FOR_SPEED))
    1380         8304 :             return ifn;
    1381              :         }
    1382              :     }
    1383              :   return IFN_LAST;
    1384              : }
    1385              : 
    1386              : 
    1387              : static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
    1388              :                                   gimple_stmt_iterator *);
    1389              : 
    1390              : /* Check whether a load or store statement in the loop described by
    1391              :    LOOP_VINFO is possible in a loop using partial vectors.  This is
    1392              :    testing whether the vectorizer pass has the appropriate support,
    1393              :    as well as whether the target does.
    1394              : 
    1395              :    VLS_TYPE says whether the statement is a load or store and VECTYPE
    1396              :    is the type of the vector being loaded or stored.  SLP_NODE is the SLP
    1397              :    node that contains the statement, or null if none.  MEMORY_ACCESS_TYPE
    1398              :    says how the load or store is going to be implemented and GROUP_SIZE
    1399              :    is the number of load or store statements in the containing group.
    1400              :    If the access is a gather load or scatter store, GS_INFO describes
    1401              :    its arguments.  If the load or store is conditional, SCALAR_MASK is the
    1402              :    condition under which it occurs.
    1403              : 
    1404              :    Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
    1405              :    vectors is not supported, otherwise record the required rgroup control
    1406              :    types.
    1407              : 
    1408              :    If partial vectors can be used and ELSVALS is nonzero the supported
    1409              :    else values will be added to the vector ELSVALS points to.  */
    1410              : 
    1411              : static void
    1412       304212 : check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
    1413              :                                       slp_tree slp_node,
    1414              :                                       vec_load_store_type vls_type,
    1415              :                                       int group_size,
    1416              :                                       vect_load_store_data *ls,
    1417              :                                       slp_tree mask_node,
    1418              :                                       vec<int> *elsvals = nullptr)
    1419              : {
    1420       304212 :   vect_memory_access_type memory_access_type = ls->memory_access_type;
    1421              : 
    1422              :   /* Invariant loads need no special support.  */
    1423       304212 :   if (memory_access_type == VMAT_INVARIANT)
    1424        30465 :     return;
    1425              : 
    1426              :   /* Figure whether the mask is uniform.  scalar_mask is used to
    1427              :      populate the scalar_cond_masked_set.  */
    1428       302987 :   tree scalar_mask = NULL_TREE;
    1429       302987 :   if (mask_node)
    1430         4952 :     for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
    1431              :       {
    1432         2507 :         tree def = vect_get_slp_scalar_def (mask_node, i);
    1433         2507 :         if (!def
    1434         2507 :             || (scalar_mask && def != scalar_mask))
    1435              :           {
    1436              :             scalar_mask = NULL;
    1437              :             break;
    1438              :           }
    1439              :         else
    1440         2486 :           scalar_mask = def;
    1441              :       }
    1442              : 
    1443       302987 :   unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
    1444       302987 :   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
    1445       302987 :   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
    1446       302987 :   machine_mode vecmode = TYPE_MODE (vectype);
    1447       302987 :   bool is_load = (vls_type == VLS_LOAD);
    1448       302987 :   if (memory_access_type == VMAT_LOAD_STORE_LANES)
    1449              :     {
    1450            0 :       nvectors /= group_size;
    1451            0 :       internal_fn ifn
    1452            0 :         = (is_load ? vect_load_lanes_supported (vectype, group_size, true,
    1453              :                                                 elsvals)
    1454            0 :                    : vect_store_lanes_supported (vectype, group_size, true));
    1455            0 :       if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
    1456            0 :         vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
    1457            0 :       else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
    1458            0 :         vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
    1459              :                                scalar_mask);
    1460              :       else
    1461              :         {
    1462            0 :           if (dump_enabled_p ())
    1463            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1464              :                              "can't operate on partial vectors because"
    1465              :                              " the target doesn't have an appropriate"
    1466              :                              " load/store-lanes instruction.\n");
    1467            0 :           LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    1468              :         }
    1469              :       return;
    1470              :     }
    1471              : 
    1472       302987 :   if (mat_gather_scatter_p (memory_access_type))
    1473              :     {
    1474         1633 :       internal_fn ifn = (is_load
    1475         1633 :                          ? IFN_MASK_GATHER_LOAD
    1476              :                          : IFN_MASK_SCATTER_STORE);
    1477          342 :       internal_fn len_ifn = (is_load
    1478              :                              ? IFN_MASK_LEN_GATHER_LOAD
    1479              :                              : IFN_MASK_LEN_SCATTER_STORE);
    1480         1633 :       stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node);
    1481         1633 :       tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr)
    1482         1633 :                           ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0])
    1483         1633 :                           : ls->strided_offset_vectype);
    1484         1633 :       tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr));
    1485         1633 :       int scale = SLP_TREE_GS_SCALE (slp_node);
    1486              : 
    1487              :       /* The following "supported" checks just verify what we established in
    1488              :          get_load_store_type and don't try different offset types.
    1489              :          Therefore, off_vectype must be a supported offset type.  In case
    1490              :          we chose a different one use this instead.  */
    1491         1633 :       if (ls->supported_offset_vectype)
    1492            0 :         off_vectype = ls->supported_offset_vectype;
    1493              :       /* Same for scale.  */
    1494         1633 :       if (ls->supported_scale)
    1495            0 :         scale = ls->supported_scale;
    1496              : 
    1497         1633 :       if (internal_gather_scatter_fn_supported_p (len_ifn, vectype,
    1498              :                                                   memory_type,
    1499              :                                                   off_vectype, scale,
    1500              :                                                   elsvals))
    1501            0 :         vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
    1502         1633 :       else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
    1503              :                                                        memory_type,
    1504              :                                                        off_vectype, scale,
    1505              :                                                        elsvals)
    1506         1633 :                || memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
    1507          567 :         vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
    1508              :                                scalar_mask);
    1509              :       else
    1510              :         {
    1511         1066 :           if (dump_enabled_p ())
    1512           24 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1513              :                              "can't operate on partial vectors because"
    1514              :                              " the target doesn't have an appropriate"
    1515              :                              " gather load or scatter store instruction.\n");
    1516         1066 :           LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    1517              :         }
    1518              :       return;
    1519              :     }
    1520              : 
    1521       301354 :   if (memory_access_type != VMAT_CONTIGUOUS)
    1522              :     {
    1523              :       /* Element X of the data must come from iteration i * VF + X of the
    1524              :          scalar loop.  We need more work to support other mappings.  */
    1525        27386 :       if (dump_enabled_p ())
    1526          758 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1527              :                          "can't operate on partial vectors because an"
    1528              :                          " access isn't contiguous.\n");
    1529        27386 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    1530        27386 :       return;
    1531              :     }
    1532              : 
    1533       273968 :   if (!VECTOR_MODE_P (vecmode))
    1534              :     {
    1535            0 :       if (dump_enabled_p ())
    1536            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1537              :                          "can't operate on partial vectors when emulating"
    1538              :                          " vector operations.\n");
    1539            0 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    1540            0 :       return;
    1541              :     }
    1542              : 
    1543       273968 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    1544       273968 :   poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    1545              : 
    1546              :   /* For permuted SLP loads the group accessed in one vector iteration
    1547              :      might not fill a whole number of vectors, for example when the
    1548              :      load permutation duplicates elements.  The loop masks and lengths
    1549              :      are computed based on NVECTORS * NUNITS / VF scalars per iteration
    1550              :      which would then enable more elements than the scalar loop accesses.  */
    1551       273968 :   if (!constant_multiple_p (group_size * vf, nunits, &nvectors))
    1552              :     {
    1553          221 :       if (dump_enabled_p ())
    1554           75 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1555              :                          "can't operate on partial vectors because the"
    1556              :                          " access does not cover a whole number of"
    1557              :                          " vectors.\n");
    1558          221 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    1559          221 :       return;
    1560              :     }
    1561              : 
    1562       273747 :   machine_mode mask_mode;
    1563       273747 :   machine_mode vmode;
    1564       273747 :   bool using_partial_vectors_p = false;
    1565       273747 :   if (get_len_load_store_mode
    1566       273747 :       (vecmode, is_load, nullptr, elsvals).exists (&vmode))
    1567              :     {
    1568            0 :       unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
    1569            0 :       vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
    1570            0 :       using_partial_vectors_p = true;
    1571              :     }
    1572       273747 :   else if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
    1573       273747 :            && can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
    1574              :                                          elsvals))
    1575              :     {
    1576        92435 :       vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask);
    1577        92435 :       using_partial_vectors_p = true;
    1578              :     }
    1579              : 
    1580        92435 :   if (!using_partial_vectors_p)
    1581              :     {
    1582       181312 :       if (dump_enabled_p ())
    1583        12883 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1584              :                          "can't operate on partial vectors because the"
    1585              :                          " target doesn't have the appropriate partial"
    1586              :                          " vectorization load or store.\n");
    1587       181312 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    1588              :     }
    1589              : }
    1590              : 
    1591              : /* Return the mask input to a masked load or store.  VEC_MASK is the vectorized
    1592              :    form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
    1593              :    that needs to be applied to all loads and stores in a vectorized loop.
    1594              :    Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
    1595              :    otherwise return VEC_MASK & LOOP_MASK.
    1596              : 
    1597              :    MASK_TYPE is the type of both masks.  If new statements are needed,
    1598              :    insert them before GSI.  */
    1599              : 
    1600              : tree
    1601         1629 : prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
    1602              :                   tree vec_mask, gimple_stmt_iterator *gsi)
    1603              : {
    1604         1629 :   gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
    1605         1629 :   if (!loop_mask)
    1606              :     return vec_mask;
    1607              : 
    1608          145 :   gcc_assert (TREE_TYPE (loop_mask) == mask_type);
    1609              : 
    1610          145 :   if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
    1611              :     return vec_mask;
    1612              : 
    1613          145 :   tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
    1614          145 :   gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
    1615              :                                           vec_mask, loop_mask);
    1616              : 
    1617          145 :   gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
    1618          145 :   return and_res;
    1619              : }
    1620              : 
    1621              : /* Determine whether we can use a gather load or scatter store to vectorize
    1622              :    strided load or store STMT_INFO by truncating the current offset to a
    1623              :    smaller width.  We need to be able to construct an offset vector:
    1624              : 
    1625              :      { 0, X, X*2, X*3, ... }
    1626              : 
    1627              :    without loss of precision, where X is STMT_INFO's DR_STEP.
    1628              : 
    1629              :    Return true if this is possible, describing the gather load or scatter
    1630              :    store in GS_INFO.  MASKED_P is true if the load or store is conditional.
    1631              : 
    1632              :    If we can use gather/scatter and ELSVALS is nonzero the supported
    1633              :    else values will be stored in the vector ELSVALS points to.  */
    1634              : 
    1635              : static bool
    1636        65671 : vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
    1637              :                                      loop_vec_info loop_vinfo, bool masked_p,
    1638              :                                      gather_scatter_info *gs_info,
    1639              :                                      vec<int> *elsvals)
    1640              : {
    1641        65671 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    1642        65671 :   data_reference *dr = dr_info->dr;
    1643        65671 :   tree step = DR_STEP (dr);
    1644        65671 :   if (TREE_CODE (step) != INTEGER_CST)
    1645              :     {
    1646              :       /* ??? Perhaps we could use range information here?  */
    1647        29644 :       if (dump_enabled_p ())
    1648          205 :         dump_printf_loc (MSG_NOTE, vect_location,
    1649              :                          "cannot truncate variable step.\n");
    1650              :       return false;
    1651              :     }
    1652              : 
    1653              :   /* Get the number of bits in an element.  */
    1654        36027 :   scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
    1655        36027 :   unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
    1656              : 
    1657              :   /* Set COUNT to the upper limit on the number of elements - 1.
    1658              :      Start with the maximum vectorization factor.  */
    1659        36027 :   unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
    1660              : 
    1661              :   /* Try lowering COUNT to the number of scalar latch iterations.  Peeling
    1662              :      for alignment by masking shifts the active lanes up, and we cannot tell
    1663              :      yet whether peeling will be done by masking, or whether masks will be
    1664              :      needed at all, so be conservative and do not lower COUNT when peeling
    1665              :      for alignment.  */
    1666        36027 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    1667        36027 :   widest_int max_iters;
    1668        36027 :   if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
    1669        36004 :       && max_loop_iterations (loop, &max_iters)
    1670        71257 :       && max_iters < count)
    1671         2146 :     count = max_iters.to_shwi ();
    1672              : 
    1673              :   /* Try scales of 1 and the element size.  */
    1674        36027 :   unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
    1675        36027 :   wi::overflow_type overflow = wi::OVF_NONE;
    1676       108081 :   for (int i = 0; i < 2; ++i)
    1677              :     {
    1678        72054 :       unsigned int scale = scales[i];
    1679        72054 :       widest_int factor;
    1680        72054 :       if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
    1681            0 :         continue;
    1682              : 
    1683              :       /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE.  */
    1684        72054 :       widest_int range = wi::mul (count, factor, SIGNED, &overflow);
    1685        72054 :       if (overflow)
    1686            0 :         continue;
    1687        72054 :       signop sign = range >= 0 ? UNSIGNED : SIGNED;
    1688        72054 :       unsigned int min_offset_bits = wi::min_precision (range, sign);
    1689              : 
    1690              :       /* Find the narrowest viable offset type.  */
    1691        72054 :       unsigned int offset_bits = 1U << ceil_log2 (min_offset_bits);
    1692        72054 :       tree offset_type = build_nonstandard_integer_type (offset_bits,
    1693              :                                                          sign == UNSIGNED);
    1694              : 
    1695              :       /* See whether the target supports the operation with an offset
    1696              :          no narrower than OFFSET_TYPE.  */
    1697        72054 :       tree memory_type = TREE_TYPE (DR_REF (dr));
    1698        72054 :       tree tmp_offset_vectype;
    1699        72054 :       int tmp_scale;
    1700        72054 :       if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
    1701              :                                      vectype, memory_type, offset_type,
    1702              :                                      scale, &tmp_scale,
    1703              :                                      &gs_info->ifn, &gs_info->offset_vectype,
    1704              :                                      &tmp_offset_vectype, elsvals)
    1705        72054 :           || gs_info->ifn == IFN_LAST)
    1706        72054 :         continue;
    1707              : 
    1708            0 :       gs_info->decl = NULL_TREE;
    1709              :       /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
    1710              :          but we don't need to store that here.  */
    1711            0 :       gs_info->base = NULL_TREE;
    1712            0 :       gs_info->alias_ptr = build_int_cst
    1713            0 :         (reference_alias_ptr_type (DR_REF (dr)),
    1714            0 :          get_object_alignment (DR_REF (dr)));
    1715            0 :       gs_info->element_type = TREE_TYPE (vectype);
    1716            0 :       gs_info->offset = fold_convert (offset_type, step);
    1717            0 :       gs_info->scale = scale;
    1718            0 :       gs_info->memory_type = memory_type;
    1719            0 :       return true;
    1720        72054 :     }
    1721              : 
    1722        36027 :   if (overflow && dump_enabled_p ())
    1723            0 :     dump_printf_loc (MSG_NOTE, vect_location,
    1724              :                      "truncating gather/scatter offset to %d bits"
    1725              :                      " might change its value.\n", element_bits);
    1726              : 
    1727              :   return false;
    1728        65671 : }
    1729              : 
    1730              : /* Return true if we can use gather/scatter or strided internal functions
    1731              :    to vectorize STMT_INFO, which is a grouped or strided load or store
    1732              :    with multiple lanes and will be implemented by a type-punned access
    1733              :    of a vector with element size that matches the number of lanes.
    1734              : 
    1735              :    MASKED_P is true if load or store is conditional.
    1736              :    When returning true, fill in GS_INFO with the information required to
    1737              :    perform the operation.  Also, store the punning type in PUNNED_VECTYPE.
    1738              : 
    1739              :    If successful and ELSVALS is nonzero the supported
    1740              :    else values will be stored in the vector ELSVALS points to.  */
    1741              : 
    1742              : static bool
    1743         4658 : vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype,
    1744              :                          loop_vec_info loop_vinfo, bool masked_p,
    1745              :                          unsigned int nelts,
    1746              :                          gather_scatter_info *info, vec<int> *elsvals,
    1747              :                          tree *pun_vectype)
    1748              : {
    1749         4658 :   data_reference *dr = dr_info->dr;
    1750              : 
    1751              :   /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
    1752              :      multiple gathers/scatter.  */
    1753         4658 :   if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
    1754              :     return false;
    1755              : 
    1756              :   /* Pun the vectype with one of the same size but an element spanning
    1757              :      NELTS elements of VECTYPE.
    1758              :      The punned type of a V16QI with NELTS = 4 would be V4SI.
    1759              :      */
    1760         4017 :   tree tmp;
    1761         4017 :   unsigned int pieces;
    1762         4017 :   if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces)
    1763         4017 :       || pieces <= 1)
    1764         2003 :     return false;
    1765              : 
    1766         2014 :   *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true);
    1767              : 
    1768         2014 :   if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype))
    1769              :     return false;
    1770              : 
    1771         1868 :   internal_fn ifn;
    1772         1868 :   tree offset_vectype = *pun_vectype;
    1773              : 
    1774         1261 :   internal_fn strided_ifn = DR_IS_READ (dr)
    1775         1868 :     ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
    1776              : 
    1777              :   /* Check if we have a gather/scatter with the new type.  We're just trying
    1778              :      with the type itself as offset for now.  If not, check if we have a
    1779              :      strided load/store.  These have fewer constraints (for example no offset
    1780              :      type must exist) so it is possible that even though a gather/scatter is
    1781              :      not available we still have a strided load/store.  */
    1782         1868 :   bool ok = false;
    1783         1868 :   tree tmp_vectype;
    1784         1868 :   int tmp_scale;
    1785         1868 :   if (vect_gather_scatter_fn_p
    1786         1868 :       (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
    1787         1868 :        TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
    1788              :        &offset_vectype, &tmp_vectype, elsvals))
    1789              :     ok = true;
    1790         1868 :   else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
    1791              :                                             elsvals))
    1792              :     {
    1793              :       /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
    1794              :          will switch back to the strided variants.  */
    1795            0 :       ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
    1796              :         IFN_MASK_LEN_SCATTER_STORE;
    1797            0 :       ok = true;
    1798              :     }
    1799              : 
    1800            0 :   if (ok)
    1801              :     {
    1802            0 :       info->ifn = ifn;
    1803            0 :       info->decl = NULL_TREE;
    1804            0 :       info->base = dr->ref;
    1805            0 :       info->alias_ptr = build_int_cst
    1806            0 :         (reference_alias_ptr_type (DR_REF (dr)),
    1807            0 :          get_object_alignment (DR_REF (dr)));
    1808            0 :       info->element_type = TREE_TYPE (*pun_vectype);
    1809            0 :       info->offset_vectype = offset_vectype;
    1810              :       /* No need to set the offset, vect_get_strided_load_store_ops
    1811              :          will do that.  */
    1812            0 :       info->scale = 1;
    1813            0 :       info->memory_type = TREE_TYPE (DR_REF (dr));
    1814            0 :       return true;
    1815              :     }
    1816              : 
    1817              :   return false;
    1818              : }
    1819              : 
    1820              : 
    1821              : /* Return true if we can use gather/scatter internal functions to
    1822              :    vectorize STMT_INFO, which is a grouped or strided load or store.
    1823              :    MASKED_P is true if load or store is conditional.  When returning
    1824              :    true, fill in GS_INFO with the information required to perform the
    1825              :    operation.
    1826              : 
    1827              :    If we can use gather/scatter and ELSVALS is nonzero the supported
    1828              :    else values will be stored in the vector ELSVALS points to.  */
    1829              : 
    1830              : static bool
    1831        65671 : vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
    1832              :                                     loop_vec_info loop_vinfo, bool masked_p,
    1833              :                                     gather_scatter_info *gs_info,
    1834              :                                     vec<int> *elsvals,
    1835              :                                     unsigned int group_size,
    1836              :                                     bool single_element_p)
    1837              : {
    1838        65671 :   if (!vect_check_gather_scatter (stmt_info, vectype,
    1839              :                                   loop_vinfo, gs_info, elsvals)
    1840        65671 :       || gs_info->ifn == IFN_LAST)
    1841              :     {
    1842        65671 :       if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
    1843              :                                                 masked_p, gs_info, elsvals))
    1844              :         return false;
    1845              :     }
    1846              : 
    1847            0 :   if (!single_element_p
    1848            0 :       && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
    1849              :                                                    gs_info->scale,
    1850              :                                                    group_size))
    1851              :     return false;
    1852              : 
    1853            0 :   if (dump_enabled_p ())
    1854            0 :     dump_printf_loc (MSG_NOTE, vect_location,
    1855              :                      "using gather/scatter for strided/grouped access,"
    1856              :                      " scale = %d\n", gs_info->scale);
    1857              : 
    1858              :   return true;
    1859              : }
    1860              : 
    1861              : /* STMT_INFO is a non-strided load or store, meaning that it accesses
    1862              :    elements with a known constant step.  Return -1 if that step
    1863              :    is negative, 0 if it is zero, and 1 if it is greater than zero.  */
    1864              : 
    1865              : int
    1866      1544118 : compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
    1867              : {
    1868      1544118 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    1869      1544118 :   return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
    1870      1544118 :                                size_zero_node);
    1871              : }
    1872              : 
    1873              : /* If the target supports a permute mask that reverses the elements in
    1874              :    a vector of type VECTYPE, return that mask, otherwise return null.  */
    1875              : 
    1876              : tree
    1877         9452 : perm_mask_for_reverse (tree vectype)
    1878              : {
    1879         9452 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    1880              : 
    1881              :   /* The encoding has a single stepped pattern.  */
    1882         9452 :   vec_perm_builder sel (nunits, 1, 3);
    1883        47260 :   for (int i = 0; i < 3; ++i)
    1884        28356 :     sel.quick_push (nunits - 1 - i);
    1885              : 
    1886         9452 :   vec_perm_indices indices (sel, 1, nunits);
    1887         9452 :   if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
    1888              :                              indices))
    1889              :     return NULL_TREE;
    1890         8224 :   return vect_gen_perm_mask_checked (vectype, indices);
    1891         9452 : }
    1892              : 
    1893              : /* A subroutine of get_load_store_type, with a subset of the same
    1894              :    arguments.  Handle the case where STMT_INFO is a load or store that
    1895              :    accesses consecutive elements with a negative step.  Sets *POFFSET
    1896              :    to the offset to be applied to the DR for the first access.  */
    1897              : 
    1898              : static vect_memory_access_type
    1899        12434 : get_negative_load_store_type (vec_info *vinfo,
    1900              :                               stmt_vec_info stmt_info, tree vectype,
    1901              :                               vec_load_store_type vls_type,
    1902              :                               unsigned int ncopies, poly_int64 *poffset)
    1903              : {
    1904        12434 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    1905        12434 :   dr_alignment_support alignment_support_scheme;
    1906              : 
    1907        12434 :   if (ncopies > 1)
    1908              :     {
    1909            0 :       if (dump_enabled_p ())
    1910            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1911              :                          "multiple types with negative step.\n");
    1912              :       return VMAT_ELEMENTWISE;
    1913              :     }
    1914              : 
    1915              :   /* For backward running DRs the first access in vectype actually is
    1916              :      N-1 elements before the address of the DR.  */
    1917        12434 :   *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
    1918        12434 :               * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
    1919              : 
    1920        12434 :   int misalignment = dr_misalignment (dr_info, vectype, *poffset);
    1921        12434 :   alignment_support_scheme
    1922        12434 :     = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
    1923        12434 :   if (alignment_support_scheme != dr_aligned
    1924        12434 :       && alignment_support_scheme != dr_unaligned_supported)
    1925              :     {
    1926         4436 :       if (dump_enabled_p ())
    1927            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1928              :                          "negative step but alignment required.\n");
    1929         4436 :       *poffset = 0;
    1930         4436 :       return VMAT_ELEMENTWISE;
    1931              :     }
    1932              : 
    1933         7998 :   if (vls_type == VLS_STORE_INVARIANT)
    1934              :     {
    1935         1184 :       if (dump_enabled_p ())
    1936           21 :         dump_printf_loc (MSG_NOTE, vect_location,
    1937              :                          "negative step with invariant source;"
    1938              :                          " no permute needed.\n");
    1939              :       return VMAT_CONTIGUOUS_DOWN;
    1940              :     }
    1941              : 
    1942         6814 :   if (!perm_mask_for_reverse (vectype))
    1943              :     {
    1944         1228 :       if (dump_enabled_p ())
    1945           52 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1946              :                          "negative step and reversing not supported.\n");
    1947         1228 :       *poffset = 0;
    1948         1228 :       return VMAT_ELEMENTWISE;
    1949              :     }
    1950              : 
    1951              :   return VMAT_CONTIGUOUS_REVERSE;
    1952              : }
    1953              : 
    1954              : /* STMT_INFO is either a masked or unconditional store.  Return the value
    1955              :    being stored.  */
    1956              : 
    1957              : tree
    1958            0 : vect_get_store_rhs (stmt_vec_info stmt_info)
    1959              : {
    1960            0 :   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
    1961              :     {
    1962            0 :       gcc_assert (gimple_assign_single_p (assign));
    1963            0 :       return gimple_assign_rhs1 (assign);
    1964              :     }
    1965            0 :   if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
    1966              :     {
    1967            0 :       internal_fn ifn = gimple_call_internal_fn (call);
    1968            0 :       int index = internal_fn_stored_value_index (ifn);
    1969            0 :       gcc_assert (index >= 0);
    1970            0 :       return gimple_call_arg (call, index);
    1971              :     }
    1972            0 :   gcc_unreachable ();
    1973              : }
    1974              : 
    1975              : /* Function VECTOR_VECTOR_COMPOSITION_TYPE
    1976              : 
    1977              :    This function returns a vector type which can be composed with NELTS pieces,
    1978              :    whose type is recorded in PTYPE.  VTYPE should be a vector type, and has the
    1979              :    same vector size as the return vector.  It checks target whether supports
    1980              :    pieces-size vector mode for construction firstly, if target fails to, check
    1981              :    pieces-size scalar mode for construction further.  It returns NULL_TREE if
    1982              :    fails to find the available composition.  If the caller only wants scalar
    1983              :    pieces where PTYPE e.g. is a possible gather/scatter element type
    1984              :    SCALAR_PTYPE_ONLY must be true.
    1985              : 
    1986              :    For example, for (vtype=V16QI, nelts=4), we can probably get:
    1987              :      - V16QI with PTYPE V4QI.
    1988              :      - V4SI with PTYPE SI.
    1989              :      - NULL_TREE.  */
    1990              : 
    1991              : static tree
    1992        12838 : vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype,
    1993              :                                 bool scalar_ptype_only)
    1994              : {
    1995        12838 :   gcc_assert (VECTOR_TYPE_P (vtype));
    1996        12838 :   gcc_assert (known_gt (nelts, 0U));
    1997              : 
    1998        12838 :   machine_mode vmode = TYPE_MODE (vtype);
    1999        12838 :   if (!VECTOR_MODE_P (vmode))
    2000              :     return NULL_TREE;
    2001              : 
    2002              :   /* When we are asked to compose the vector from its components let
    2003              :      that happen directly.  */
    2004        12838 :   if (known_eq (TYPE_VECTOR_SUBPARTS (vtype), nelts))
    2005              :     {
    2006         6095 :       *ptype = TREE_TYPE (vtype);
    2007         6095 :       return vtype;
    2008              :     }
    2009              : 
    2010        13486 :   poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
    2011         6743 :   unsigned int pbsize;
    2012         6743 :   if (constant_multiple_p (vbsize, nelts, &pbsize))
    2013              :     {
    2014              :       /* First check if vec_init optab supports construction from
    2015              :          vector pieces directly.  */
    2016         6743 :       scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype));
    2017        13486 :       poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode);
    2018         6743 :       machine_mode rmode;
    2019         6743 :       if (!scalar_ptype_only
    2020         8350 :           && related_vector_mode (vmode, elmode, inelts).exists (&rmode)
    2021        11010 :           && (convert_optab_handler (vec_init_optab, vmode, rmode)
    2022              :               != CODE_FOR_nothing))
    2023              :         {
    2024         3621 :           *ptype = build_vector_type (TREE_TYPE (vtype), inelts);
    2025         3621 :           return vtype;
    2026              :         }
    2027              : 
    2028              :       /* Otherwise check if exists an integer type of the same piece size and
    2029              :          if vec_init optab supports construction from it directly.  */
    2030         3122 :       if (int_mode_for_size (pbsize, 0).exists (&elmode)
    2031         3122 :           && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
    2032              :         {
    2033         2938 :           if (scalar_ptype_only
    2034         2938 :               || convert_optab_handler (vec_init_optab, rmode, elmode)
    2035              :               != CODE_FOR_nothing)
    2036              :             {
    2037         2938 :               *ptype = build_nonstandard_integer_type (pbsize, 1);
    2038         2938 :               return build_vector_type (*ptype, nelts);
    2039              :             }
    2040              :         }
    2041              :     }
    2042              : 
    2043              :   return NULL_TREE;
    2044              : }
    2045              : 
    2046              : /* Check if the load permutation of NODE only refers to a consecutive
    2047              :    subset of the group indices.  We also require the length of the
    2048              :    permutation to be a power of two.
    2049              :    Such load permutations can be elided in strided access schemes as
    2050              :    we can "jump over" the gap they leave.  */
    2051              : 
    2052              : static bool
    2053        46339 : has_consecutive_load_permutation (slp_tree node)
    2054              : {
    2055        46339 :   load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
    2056        46339 :   if (!perm.exists ()
    2057         2183 :       || perm.length () <= 1
    2058        46848 :       || !pow2p_hwi (perm.length ()))
    2059              :     return false;
    2060              : 
    2061          489 :   return vect_load_perm_consecutive_p (node);
    2062              : }
    2063              : 
    2064              : 
    2065              : /* Analyze load or store SLP_NODE of type VLS_TYPE.  Return true
    2066              :    if there is a memory access type that the vectorized form can use,
    2067              :    storing it in *MEMORY_ACCESS_TYPE if so.  If we decide to use gathers
    2068              :    or scatters, fill in GS_INFO accordingly.  In addition
    2069              :    *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
    2070              :    the target does not support the alignment scheme.  *MISALIGNMENT
    2071              :    is set according to the alignment of the access (including
    2072              :    DR_MISALIGNMENT_UNKNOWN when it is unknown).
    2073              : 
    2074              :    MASKED_P is true if the statement is conditional on a vectorized mask.
    2075              :    VECTYPE is the vector type that the vectorized statements will use.
    2076              : 
    2077              :    If ELSVALS is nonzero the supported else values will be stored in the
    2078              :    vector ELSVALS points to.  */
    2079              : 
    2080              : static bool
    2081      1421714 : get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
    2082              :                      tree vectype, slp_tree slp_node,
    2083              :                      bool masked_p, vec_load_store_type vls_type,
    2084              :                      vect_load_store_data *ls)
    2085              : {
    2086      1421714 :   vect_memory_access_type *memory_access_type = &ls->memory_access_type;
    2087      1421714 :   poly_int64 *poffset = &ls->poffset;
    2088      1421714 :   dr_alignment_support *alignment_support_scheme
    2089              :     = &ls->alignment_support_scheme;
    2090      1421714 :   int *misalignment = &ls->misalignment;
    2091      1421714 :   internal_fn *lanes_ifn = &ls->lanes_ifn;
    2092      1421714 :   vec<int> *elsvals = &ls->elsvals;
    2093      1421714 :   tree *ls_type = &ls->ls_type;
    2094      1421714 :   bool *slp_perm = &ls->slp_perm;
    2095      1421714 :   unsigned *n_perms = &ls->n_perms;
    2096      1421714 :   unsigned *n_loads = &ls->n_loads;
    2097      1421714 :   tree *supported_offset_vectype = &ls->supported_offset_vectype;
    2098      1421714 :   int *supported_scale = &ls->supported_scale;
    2099      1421714 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    2100      1421714 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    2101      1421714 :   class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
    2102      1421714 :   stmt_vec_info first_stmt_info;
    2103      1421714 :   unsigned int group_size;
    2104      1421714 :   unsigned HOST_WIDE_INT gap;
    2105      1421714 :   bool single_element_p;
    2106      1421714 :   poly_int64 neg_ldst_offset = 0;
    2107              : 
    2108      1421714 :   *misalignment = DR_MISALIGNMENT_UNKNOWN;
    2109      1421714 :   *poffset = 0;
    2110      1421714 :   *ls_type = NULL_TREE;
    2111      1421714 :   *slp_perm = false;
    2112      1421714 :   *n_perms = -1U;
    2113      1421714 :   *n_loads = -1U;
    2114      1421714 :   ls->subchain_p = false;
    2115              : 
    2116      1421714 :   bool perm_ok = true;
    2117      1421714 :   poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
    2118              : 
    2119      1421714 :   if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
    2120        78907 :     perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
    2121        78907 :                                             vf, true, n_perms, n_loads);
    2122              : 
    2123      1421714 :   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2124              :     {
    2125       899684 :       first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
    2126       899684 :       group_size = DR_GROUP_SIZE (first_stmt_info);
    2127       899684 :       gap = DR_GROUP_GAP (first_stmt_info);
    2128       899684 :       single_element_p = (stmt_info == first_stmt_info
    2129       899684 :                           && !DR_GROUP_NEXT_ELEMENT (stmt_info));
    2130              :     }
    2131              :   else
    2132              :     {
    2133              :       first_stmt_info = stmt_info;
    2134              :       group_size = 1;
    2135              :       gap = 0;
    2136              :       single_element_p = true;
    2137              :     }
    2138      1421714 :   dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
    2139              : 
    2140              :   /* There can only be a gap at the end of the group if the stride is
    2141              :      known at compile time.  */
    2142      1421714 :   gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
    2143              : 
    2144              :   /* For SLP vectorization we directly vectorize a subchain
    2145              :      without permutation.  */
    2146      1421714 :   if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
    2147      1342807 :     first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]);
    2148              : 
    2149      1421714 :   if (STMT_VINFO_STRIDED_P (first_stmt_info))
    2150              :     {
    2151              :       /* Try to use consecutive accesses of as many elements as possible,
    2152              :          separated by the stride, until we have a complete vector.
    2153              :          Fall back to scalar accesses if that isn't possible.  */
    2154        46339 :       *memory_access_type = VMAT_STRIDED_SLP;
    2155              : 
    2156              :       /* If the load permutation is consecutive we can reduce the group to
    2157              :          the elements the permutation accesses.  Then we release the
    2158              :          permutation.  */
    2159        46339 :       if (has_consecutive_load_permutation (slp_node))
    2160              :         {
    2161           51 :           ls->subchain_p = true;
    2162           51 :           group_size = SLP_TREE_LANES (slp_node);
    2163           51 :           SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
    2164              :         }
    2165              :     }
    2166      1375375 :   else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    2167              :     {
    2168        11249 :       slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
    2169        11249 :       tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
    2170        11249 :       int scale = SLP_TREE_GS_SCALE (slp_node);
    2171        11249 :       tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
    2172        11249 :       tree tem;
    2173        11249 :       if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
    2174              :                                     masked_p, vectype, memory_type,
    2175              :                                     offset_vectype, scale, supported_scale,
    2176              :                                     &ls->gs.ifn, &tem,
    2177              :                                     supported_offset_vectype, elsvals))
    2178              :         {
    2179            0 :           if (dump_enabled_p ())
    2180              :             {
    2181            0 :               dump_printf_loc (MSG_NOTE, vect_location,
    2182              :                                "gather/scatter with required "
    2183              :                                "offset type "
    2184              :                                "%T and offset scale %d.\n",
    2185              :                                offset_vectype, scale);
    2186            0 :               if (*supported_offset_vectype)
    2187            0 :                 dump_printf_loc (MSG_NOTE, vect_location,
    2188              :                                  " target supports offset type %T.\n",
    2189              :                                  *supported_offset_vectype);
    2190            0 :               if (*supported_scale)
    2191            0 :                 dump_printf_loc (MSG_NOTE, vect_location,
    2192              :                                  " target supports offset scale %d.\n",
    2193              :                                  *supported_scale);
    2194              :             }
    2195            0 :           *memory_access_type = VMAT_GATHER_SCATTER_IFN;
    2196              :         }
    2197        11249 :       else if (vls_type == VLS_LOAD
    2198        11249 :                ? (targetm.vectorize.builtin_gather
    2199         9842 :                   && (ls->gs.decl
    2200         9842 :                         = targetm.vectorize.builtin_gather (vectype,
    2201         9842 :                                                             TREE_TYPE
    2202              :                                                               (offset_vectype),
    2203              :                                                             scale)))
    2204         1407 :                : (targetm.vectorize.builtin_scatter
    2205         1407 :                   && (ls->gs.decl
    2206         1407 :                         = targetm.vectorize.builtin_scatter (vectype,
    2207         1407 :                                                              TREE_TYPE
    2208              :                                                                (offset_vectype),
    2209              :                                                              scale))))
    2210          569 :         *memory_access_type = VMAT_GATHER_SCATTER_LEGACY;
    2211              :       else
    2212              :         {
    2213              :           /* GATHER_SCATTER_EMULATED_P.  */
    2214        10680 :           if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
    2215        10680 :               || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
    2216        10680 :               || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
    2217        10680 :               || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
    2218        14158 :                                        TYPE_VECTOR_SUBPARTS (vectype)))
    2219              :             {
    2220         3478 :               if (dump_enabled_p ())
    2221          466 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2222              :                                  "unsupported vector types for emulated "
    2223              :                                  "gather.\n");
    2224         3478 :               return false;
    2225              :             }
    2226         7202 :           *memory_access_type = VMAT_GATHER_SCATTER_EMULATED;
    2227              :         }
    2228              :     }
    2229              :   else
    2230              :     {
    2231      1364126 :       int cmp = compare_step_with_zero (vinfo, stmt_info);
    2232      1364126 :       if (cmp < 0)
    2233              :         {
    2234        12618 :           if (single_element_p)
    2235              :             /* ???  The VMAT_CONTIGUOUS_REVERSE code generation is
    2236              :                only correct for single element "interleaving" SLP.  */
    2237        12434 :             *memory_access_type = get_negative_load_store_type
    2238        12434 :                 (vinfo, stmt_info, vectype, vls_type, 1,
    2239              :                  &neg_ldst_offset);
    2240              :           else
    2241              :             /* We can fall back to VMAT_STRIDED_SLP since that does
    2242              :                not care whether the stride between the group instances
    2243              :                is positive or negative.  */
    2244          184 :             *memory_access_type = VMAT_STRIDED_SLP;
    2245              :         }
    2246      1351508 :       else if (cmp == 0 && loop_vinfo)
    2247              :         {
    2248         5144 :           gcc_assert (vls_type == VLS_LOAD);
    2249         5144 :           *memory_access_type = VMAT_INVARIANT;
    2250              :         }
    2251              :       /* Try using LOAD/STORE_LANES.  */
    2252      1346364 :       else if (slp_node->ldst_lanes
    2253      1346364 :                && (*lanes_ifn
    2254            0 :                    = (vls_type == VLS_LOAD
    2255            0 :                       ? vect_load_lanes_supported (vectype, group_size,
    2256              :                                                    masked_p, elsvals)
    2257            0 :                       : vect_store_lanes_supported (vectype, group_size,
    2258              :                                                     masked_p))) != IFN_LAST)
    2259            0 :         *memory_access_type = VMAT_LOAD_STORE_LANES;
    2260      1346364 :       else if (!loop_vinfo && slp_node->avoid_stlf_fail)
    2261              :         {
    2262           66 :           *memory_access_type = VMAT_ELEMENTWISE;
    2263           66 :           if (dump_enabled_p ())
    2264            2 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2265              :                              "using element-wise load to avoid disrupting "
    2266              :                              "cross iteration store-to-load forwarding\n");
    2267              :         }
    2268              :       else
    2269      1346298 :         *memory_access_type = VMAT_CONTIGUOUS;
    2270              : 
    2271              :       /* True if the vectorized statements would access beyond the last
    2272              :          statement in the group.  */
    2273      1364126 :       bool overrun_p = false;
    2274              : 
    2275              :       /* True if we can cope with such overrun by peeling for gaps, so that
    2276              :          there is at least one final scalar iteration after the vector loop.  */
    2277      2728252 :       bool can_overrun_p = (!masked_p
    2278      1364126 :                             && vls_type == VLS_LOAD
    2279       552738 :                             && loop_vinfo
    2280       408811 :                             && !loop->inner
    2281      1771085 :                             && (*memory_access_type != VMAT_STRIDED_SLP
    2282           80 :                                 || cmp > 0));
    2283              : 
    2284              :       /* If this is single-element interleaving with an element
    2285              :          distance that leaves unused vector loads around fall back
    2286              :          to elementwise access if possible - we otherwise least
    2287              :          create very sub-optimal code in that case (and
    2288              :          blow up memory, see PR65518).  */
    2289      1364126 :       if (loop_vinfo
    2290      1364126 :           && single_element_p
    2291       501783 :           && (*memory_access_type == VMAT_CONTIGUOUS
    2292        17578 :               || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
    2293      1865909 :           && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
    2294              :         {
    2295        18285 :           *memory_access_type = VMAT_ELEMENTWISE;
    2296        18285 :           if (dump_enabled_p ())
    2297          198 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2298              :                              "single-element interleaving not supported "
    2299              :                              "for not adjacent vector loads, using "
    2300              :                              "elementwise access\n");
    2301              :         }
    2302              : 
    2303              :       /* Also fall back to elementwise access in case we did not lower a
    2304              :          permutation and cannot code generate it.  */
    2305      1364126 :       if (loop_vinfo
    2306       552884 :           && *memory_access_type != VMAT_ELEMENTWISE
    2307       528935 :           && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
    2308      1390063 :           && !perm_ok)
    2309              :         {
    2310         2106 :           *memory_access_type = VMAT_ELEMENTWISE;
    2311         2106 :           if (dump_enabled_p ())
    2312          248 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2313              :                              "permutation not supported, using elementwise "
    2314              :                              "access\n");
    2315              :         }
    2316              : 
    2317       552884 :       overrun_p = (loop_vinfo && gap != 0
    2318      1407911 :                    && *memory_access_type != VMAT_ELEMENTWISE);
    2319      1364126 :       if (overrun_p && vls_type != VLS_LOAD)
    2320              :         {
    2321            0 :           if (dump_enabled_p ())
    2322            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2323              :                              "Grouped store with gaps requires"
    2324              :                              " non-consecutive accesses\n");
    2325           23 :           return false;
    2326              :         }
    2327              : 
    2328      1364126 :       unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
    2329      1364126 :       poly_int64 off = 0;
    2330      1364126 :       if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
    2331         5427 :         off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
    2332              : 
    2333              :       /* An overrun is fine if the trailing elements are smaller
    2334              :          than the alignment boundary B.  Every vector access will
    2335              :          be a multiple of B and so we are guaranteed to access a
    2336              :          non-gap element in the same B-sized block.  */
    2337      1364126 :       if (overrun_p
    2338      1364126 :           && gap < (vect_known_alignment_in_bytes (first_dr_info,
    2339        23325 :                                                    vectype, off) / dr_size))
    2340              :         overrun_p = false;
    2341              : 
    2342              :       /* When we have a contiguous access across loop iterations
    2343              :          but the access in the loop doesn't cover the full vector
    2344              :          we can end up with no gap recorded but still excess
    2345              :          elements accessed, see PR103116.  Make sure we peel for
    2346              :          gaps if necessary and sufficient and give up if not.
    2347              : 
    2348              :          If there is a combination of the access not covering the full
    2349              :          vector and a gap recorded then we may need to peel twice.  */
    2350      1364126 :       bool large_vector_overrun_p = false;
    2351      1364126 :       if (loop_vinfo
    2352       552884 :           && (*memory_access_type == VMAT_CONTIGUOUS
    2353        37989 :               || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
    2354       520322 :           && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
    2355      1387573 :           && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo),
    2356              :                           nunits))
    2357              :         large_vector_overrun_p = overrun_p = true;
    2358              : 
    2359              :       /* If the gap splits the vector in half and the target
    2360              :          can do half-vector operations avoid the epilogue peeling
    2361              :          by simply loading half of the vector only.  Usually
    2362              :          the construction with an upper zero half will be elided.  */
    2363      1364126 :       dr_alignment_support alss;
    2364      1364126 :       int misalign = dr_misalignment (first_dr_info, vectype, off);
    2365      1364126 :       tree half_vtype;
    2366      1364126 :       poly_uint64 remain;
    2367      1364126 :       unsigned HOST_WIDE_INT tem, num;
    2368      1364126 :       if (overrun_p
    2369      1364126 :           && !masked_p
    2370        17795 :           && (*memory_access_type == VMAT_CONTIGUOUS
    2371          360 :               || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
    2372        17781 :           && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
    2373              :                                                       vectype, misalign)))
    2374              :               == dr_aligned
    2375        15246 :               || alss == dr_unaligned_supported)
    2376        10028 :           && can_div_trunc_p (group_size
    2377        10028 :                               * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
    2378              :                               nunits, &tem, &remain)
    2379      1374154 :           && (known_eq (remain, 0u)
    2380         7525 :               || (known_ne (remain, 0u)
    2381         5825 :                   && constant_multiple_p (nunits, remain, &num)
    2382      1364126 :                   && (vector_vector_composition_type (vectype, num, &half_vtype)
    2383              :                       != NULL_TREE))))
    2384              :         overrun_p = false;
    2385              : 
    2386      1364126 :       if (overrun_p && !can_overrun_p)
    2387              :         {
    2388           20 :           if (dump_enabled_p ())
    2389           12 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2390              :                              "Peeling for outer loop is not supported\n");
    2391              :           return false;
    2392              :         }
    2393              : 
    2394              :       /* Peeling for gaps assumes that a single scalar iteration
    2395              :          is enough to make sure the last vector iteration doesn't
    2396              :          access excess elements.  */
    2397      1364106 :       if (overrun_p
    2398      1364106 :           && (!can_div_trunc_p (group_size
    2399         9447 :                                 * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
    2400              :                                 nunits, &tem, &remain)
    2401         9447 :               || maybe_lt (remain + group_size, nunits)))
    2402              :         {
    2403              :           /* But peeling a single scalar iteration is enough if
    2404              :              we can use the next power-of-two sized partial
    2405              :              access and that is sufficiently small to be covered
    2406              :              by the single scalar iteration.  */
    2407           16 :           unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size;
    2408           16 :           if (masked_p
    2409           16 :               || !nunits.is_constant (&cnunits)
    2410           16 :               || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf)
    2411           16 :               || (((cremain = (group_size * cvf - gap) % cnunits), true)
    2412           16 :                   && ((cpart_size = (1 << ceil_log2 (cremain))), true)
    2413           16 :                   && (cremain + group_size < cpart_size
    2414           16 :                       || (vector_vector_composition_type (vectype,
    2415           13 :                                                          cnunits / cpart_size,
    2416              :                                                          &half_vtype)
    2417              :                           == NULL_TREE))))
    2418              :             {
    2419              :               /* If all fails we can still resort to niter masking unless
    2420              :                  the vectors used are too big, so enforce the use of
    2421              :                  partial vectors.  */
    2422            3 :               if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    2423            3 :                   && !large_vector_overrun_p)
    2424              :                 {
    2425            0 :                   if (dump_enabled_p ())
    2426            0 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2427              :                                      "peeling for gaps insufficient for "
    2428              :                                      "access unless using partial "
    2429              :                                      "vectors\n");
    2430            0 :                   LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
    2431              :                 }
    2432              :               else
    2433              :                 {
    2434            3 :                   if (dump_enabled_p ())
    2435            3 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2436              :                                      "peeling for gaps insufficient for "
    2437              :                                      "access\n");
    2438           23 :                   return false;
    2439              :                 }
    2440              :             }
    2441           13 :           else if (large_vector_overrun_p)
    2442              :             {
    2443           13 :               if (dump_enabled_p ())
    2444           12 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2445              :                                  "can't operate on partial vectors because "
    2446              :                                  "only unmasked loads handle access "
    2447              :                                  "shortening required because of gaps at "
    2448              :                                  "the end of the access\n");
    2449           13 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2450              :             }
    2451              :         }
    2452              : 
    2453      1364103 :       if (overrun_p)
    2454              :         {
    2455         9444 :           gcc_assert (can_overrun_p);
    2456         9444 :           if (dump_enabled_p ())
    2457          511 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2458              :                              "Data access with gaps requires scalar "
    2459              :                              "epilogue loop\n");
    2460         9444 :           LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
    2461              :         }
    2462              :     }
    2463              : 
    2464              :   /* As a last resort, trying using a gather load or scatter store.
    2465              : 
    2466              :      ??? Although the code can handle all group sizes correctly,
    2467              :      it probably isn't a win to use separate strided accesses based
    2468              :      on nearby locations.  Or, even if it's a win over scalar code,
    2469              :      it might not be a win over vectorizing at a lower VF, if that
    2470              :      allows us to use contiguous accesses.  */
    2471      1418213 :   vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED;
    2472      1418213 :   if (loop_vinfo
    2473       606971 :       && (*memory_access_type == VMAT_ELEMENTWISE
    2474       606971 :           || *memory_access_type == VMAT_STRIDED_SLP))
    2475              :     {
    2476        72559 :       gather_scatter_info gs_info;
    2477        72559 :       tree tem;
    2478        72559 :       if (SLP_TREE_LANES (slp_node) == 1
    2479        67707 :           && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
    2480        22099 :               || single_element_p)
    2481       138230 :           && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
    2482              :                                                  masked_p, &gs_info, elsvals,
    2483              :                                                  group_size, single_element_p))
    2484              :         {
    2485              :           /* vect_use_strided_gather_scatters_p does not save the actually
    2486              :              supported scale and offset type so do that here.
    2487              :              We need it later in check_load_store_for_partial_vectors
    2488              :              where we only check if the given internal function is supported
    2489              :              (to choose whether to use the IFN, LEGACY, or EMULATED flavor
    2490              :              of gather/scatter) and don't re-do the full analysis.  */
    2491            0 :           tree tmp;
    2492            0 :           gcc_assert (vect_gather_scatter_fn_p
    2493              :                       (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype,
    2494              :                        gs_info.memory_type, TREE_TYPE (gs_info.offset),
    2495              :                        gs_info.scale, supported_scale, &gs_info.ifn,
    2496              :                        &tmp, supported_offset_vectype, elsvals));
    2497              : 
    2498            0 :           SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
    2499            0 :           SLP_TREE_GS_BASE (slp_node) = error_mark_node;
    2500            0 :           ls->gs.ifn = gs_info.ifn;
    2501            0 :           ls->strided_offset_vectype = gs_info.offset_vectype;
    2502            0 :           *memory_access_type = VMAT_GATHER_SCATTER_IFN;
    2503              :         }
    2504        72559 :       else if (SLP_TREE_LANES (slp_node) > 1
    2505              :                && !masked_p
    2506         4852 :                && !single_element_p
    2507        77217 :                && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info),
    2508              :                                            vectype, loop_vinfo,
    2509              :                                            masked_p, group_size,
    2510              :                                            &gs_info, elsvals, &tem))
    2511              :         {
    2512            0 :           SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
    2513            0 :           SLP_TREE_GS_BASE (slp_node) = error_mark_node;
    2514            0 :           grouped_gather_fallback = *memory_access_type;
    2515            0 :           *memory_access_type = VMAT_GATHER_SCATTER_IFN;
    2516            0 :           ls->gs.ifn = gs_info.ifn;
    2517            0 :           vectype = *ls_type = tem;
    2518            0 :           ls->strided_offset_vectype = gs_info.offset_vectype;
    2519              :         }
    2520              :     }
    2521              : 
    2522      1418213 :   if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
    2523      1418213 :       || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
    2524         6607 :     *poffset = neg_ldst_offset;
    2525              : 
    2526      1418213 :   if (*memory_access_type == VMAT_ELEMENTWISE
    2527      1392092 :       || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY
    2528      1391523 :       || *memory_access_type == VMAT_STRIDED_SLP
    2529      1345019 :       || *memory_access_type == VMAT_INVARIANT)
    2530              :     {
    2531        78338 :       *alignment_support_scheme = dr_unaligned_supported;
    2532        78338 :       *misalignment = DR_MISALIGNMENT_UNKNOWN;
    2533              :     }
    2534              :   else
    2535              :     {
    2536      1339875 :       if (mat_gather_scatter_p (*memory_access_type)
    2537              :           && !first_dr_info)
    2538              :         *misalignment = DR_MISALIGNMENT_UNKNOWN;
    2539              :       else
    2540      1339875 :         *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
    2541      1339875 :       *alignment_support_scheme
    2542      1339875 :         = vect_supportable_dr_alignment
    2543      1339875 :            (vinfo, first_dr_info, vectype, *misalignment,
    2544              :             mat_gather_scatter_p (*memory_access_type));
    2545      1339875 :       if (grouped_gather_fallback != VMAT_UNINITIALIZED
    2546            0 :           && *alignment_support_scheme != dr_aligned
    2547            0 :           && *alignment_support_scheme != dr_unaligned_supported)
    2548              :         {
    2549              :           /* No supportable alignment for a grouped gather, fall back to the
    2550              :              original memory access type.  Even though VMAT_STRIDED_SLP might
    2551              :              also try aligned vector loads it can still choose vector
    2552              :              construction from scalars.  */
    2553            0 :           *memory_access_type = grouped_gather_fallback;
    2554            0 :           *alignment_support_scheme = dr_unaligned_supported;
    2555            0 :           *misalignment = DR_MISALIGNMENT_UNKNOWN;
    2556              :         }
    2557              :     }
    2558              : 
    2559      1418213 :   if ((*memory_access_type == VMAT_ELEMENTWISE
    2560      1418213 :        || *memory_access_type == VMAT_STRIDED_SLP)
    2561              :       && !nunits.is_constant ())
    2562              :     {
    2563              :       if (dump_enabled_p ())
    2564              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2565              :                          "Not using elementwise accesses due to variable "
    2566              :                          "vectorization factor.\n");
    2567              :       return false;
    2568              :     }
    2569              : 
    2570              :   /* Checks if all scalar iterations are known to be inbounds.  */
    2571      1418213 :   bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
    2572              : 
    2573              :   /* Check if we support the operation if early breaks are needed.  Here we
    2574              :      must ensure that we don't access any more than the scalar code would
    2575              :      have.  A masked operation would ensure this, so for these load types
    2576              :      force masking.  */
    2577      1418213 :   if (loop_vinfo
    2578       606971 :       && dr_safe_speculative_read_required (stmt_info)
    2579      1618911 :       && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
    2580              :     {
    2581       200698 :       if (mat_gather_scatter_p (*memory_access_type)
    2582       200698 :           || *memory_access_type == VMAT_STRIDED_SLP)
    2583              :         {
    2584         9542 :           if (dump_enabled_p ())
    2585            8 :             dump_printf_loc (MSG_NOTE, vect_location,
    2586              :                              "early break not supported: cannot peel for "
    2587              :                              "alignment. With non-contiguous memory vectorization"
    2588              :                              " could read out of bounds at %G ",
    2589              :                              STMT_VINFO_STMT (stmt_info));
    2590         9542 :           if (inbounds)
    2591            0 :             LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
    2592              :           else
    2593              :             return false;
    2594              :         }
    2595              :       /* Block-level alignment: Even though individual accesses of
    2596              :          VMAT_ELEMENTWISE type do not cause alignment problems, loading the
    2597              :          whole vector's worth of values in a speculative early-break context
    2598              :          might cross a page boundary.  Set the alignment scheme to `dr_aligned'
    2599              :          here in order to force checking of whether such accesses meet
    2600              :          alignment criteria.  */
    2601       191156 :       else if (*memory_access_type == VMAT_ELEMENTWISE && !inbounds)
    2602        15078 :         *alignment_support_scheme = dr_aligned;
    2603              :     }
    2604              : 
    2605              :   /* If this DR needs alignment for correctness, we must ensure the target
    2606              :      alignment is a constant power-of-two multiple of the amount read per
    2607              :      vector iteration or force masking.  */
    2608      1408671 :   if (dr_safe_speculative_read_required (stmt_info)
    2609      1408671 :       && (*alignment_support_scheme == dr_aligned
    2610       112902 :           && !mat_gather_scatter_p (*memory_access_type)))
    2611              :     {
    2612              :       /* We can only peel for loops, of course.  */
    2613       112902 :       gcc_checking_assert (loop_vinfo);
    2614              : 
    2615       112902 :       poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    2616       112902 :       poly_uint64 read_amount
    2617       112902 :         = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
    2618       112902 :       if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2619       112902 :         read_amount *= group_size;
    2620              : 
    2621       112902 :       auto target_alignment
    2622       112902 :         = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
    2623       112902 :       if (!multiple_p (target_alignment, read_amount))
    2624              :         {
    2625        12832 :           if (dump_enabled_p ())
    2626              :             {
    2627           28 :               dump_printf_loc (MSG_NOTE, vect_location,
    2628              :                                "desired alignment not met, target was ");
    2629           28 :               dump_dec (MSG_NOTE, target_alignment);
    2630           28 :               dump_printf (MSG_NOTE, " previously, but read amount is ");
    2631           28 :               dump_dec (MSG_NOTE, read_amount);
    2632           28 :               dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info));
    2633              :             }
    2634        15117 :           return false;
    2635              :         }
    2636              : 
    2637              :       /* When using a group access the first element may be aligned but the
    2638              :          subsequent loads may not be.  For LOAD_LANES since the loads are based
    2639              :          on the first DR then all loads in the group are aligned.  For
    2640              :          non-LOAD_LANES this is not the case. In particular a load + blend when
    2641              :          there are gaps can have the non first loads issued unaligned, even
    2642              :          partially overlapping the memory of the first load in order to simplify
    2643              :          the blend.  This is what the x86_64 backend does for instance.  As
    2644              :          such only the first load in the group is aligned, the rest are not.
    2645              :          Because of this the permutes may break the alignment requirements that
    2646              :          have been set, and as such we should for now, reject them.  */
    2647       100070 :       if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
    2648              :         {
    2649         2285 :           if (dump_enabled_p ())
    2650           87 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2651              :                              "loads with load permutations not supported for "
    2652              :                              "speculative early break loads for %G",
    2653              :                              STMT_VINFO_STMT (stmt_info));
    2654              :           return false;
    2655              :         }
    2656              : 
    2657              :       /* Reject vectorization if we know the read mount per vector iteration
    2658              :          exceeds the min page size.  */
    2659        97785 :       if (known_gt (read_amount, (unsigned) param_min_pagesize))
    2660              :         {
    2661            0 :           if (dump_enabled_p ())
    2662              :             {
    2663            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2664              :                                "alignment required for correctness (");
    2665            0 :               dump_dec (MSG_MISSED_OPTIMIZATION, read_amount);
    2666            0 :               dump_printf (MSG_NOTE, ") may exceed page size.\n");
    2667              :             }
    2668              :           return false;
    2669              :         }
    2670              : 
    2671        97785 :       if (!vf.is_constant ())
    2672              :         {
    2673              :           /* For VLA modes, we need a runtime check to ensure any speculative
    2674              :              read amount does not exceed the page size.  Here we record the max
    2675              :              possible read amount for the check.  */
    2676              :           if (maybe_gt (read_amount,
    2677              :                         LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo)))
    2678              :             LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount;
    2679              : 
    2680              :           /* For VLA modes, we must use partial vectors.  */
    2681              :           LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
    2682              :         }
    2683              :     }
    2684              : 
    2685      1393554 :   if (*alignment_support_scheme == dr_unaligned_unsupported)
    2686              :     {
    2687        72984 :       if (dump_enabled_p ())
    2688          256 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2689              :                          "unsupported unaligned access\n");
    2690              :       return false;
    2691              :     }
    2692              : 
    2693              :   /* FIXME: At the moment the cost model seems to underestimate the
    2694              :      cost of using elementwise accesses.  This check preserves the
    2695              :      traditional behavior until that can be fixed.  */
    2696      1320570 :   if (*memory_access_type == VMAT_ELEMENTWISE
    2697        15260 :       && !STMT_VINFO_STRIDED_P (first_stmt_info)
    2698      1335830 :       && !(STMT_VINFO_GROUPED_ACCESS (stmt_info)
    2699        10019 :            && single_element_p
    2700         9376 :            && !pow2p_hwi (group_size)))
    2701              :     {
    2702         9337 :       if (dump_enabled_p ())
    2703          364 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2704              :                          "not falling back to elementwise accesses\n");
    2705              :       return false;
    2706              :     }
    2707              : 
    2708              :   /* For BB vectorization build up the vector from existing scalar defs.  */
    2709      1311233 :   if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE)
    2710              :     return false;
    2711              : 
    2712              :   /* Some loads need to explicitly permute the loaded data if there
    2713              :      is a load permutation.  Among those are:
    2714              :       - VMAT_ELEMENTWISE.
    2715              :       - VMAT_STRIDED_SLP.
    2716              :       - VMAT_GATHER_SCATTER:
    2717              :         - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1).
    2718              :         - Grouped strided gather (ditto but for #lanes > 1).
    2719              : 
    2720              :      For VMAT_ELEMENTWISE we can fold the load permutation into the
    2721              :      individual indices we access directly, eliding the permutation.
    2722              :      Strided gather only allows load permutations for the
    2723              :      single-element case.  */
    2724              : 
    2725      1311233 :   if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
    2726      1311233 :       && !(*memory_access_type == VMAT_ELEMENTWISE
    2727        51238 :            || (mat_gather_scatter_p (*memory_access_type)
    2728            0 :                && SLP_TREE_LANES (slp_node) == 1
    2729            0 :                && single_element_p)))
    2730              :     {
    2731        51238 :       if (!loop_vinfo)
    2732              :         {
    2733              :           /* In BB vectorization we may not actually use a loaded vector
    2734              :              accessing elements in excess of DR_GROUP_SIZE.  */
    2735        32195 :           stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
    2736        32195 :           group_info = DR_GROUP_FIRST_ELEMENT (group_info);
    2737        32195 :           unsigned HOST_WIDE_INT nunits;
    2738        32195 :           unsigned j, k, maxk = 0;
    2739       112603 :           FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
    2740        80408 :             if (k > maxk)
    2741              :               maxk = k;
    2742        32195 :           tree vectype = SLP_TREE_VECTYPE (slp_node);
    2743        58471 :           if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
    2744        32195 :               || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
    2745              :             {
    2746         5919 :               if (dump_enabled_p ())
    2747           41 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2748              :                                  "BB vectorization with gaps at the end of "
    2749              :                                  "a load is not supported\n");
    2750      1421714 :               return false;
    2751              :             }
    2752              :         }
    2753              : 
    2754        45319 :       if (!perm_ok)
    2755              :         {
    2756         2317 :           if (dump_enabled_p ())
    2757            2 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION,
    2758              :                              vect_location,
    2759              :                              "unsupported load permutation\n");
    2760              :           return false;
    2761              :         }
    2762              : 
    2763        43002 :       *slp_perm = true;
    2764              :     }
    2765              : 
    2766              :   return true;
    2767              : }
    2768              : 
    2769              : /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
    2770              :    conditional operation STMT_INFO.  When returning true, store the mask
    2771              :    in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
    2772              :    the vectorized mask in *MASK_VECTYPE_OUT.  */
    2773              : 
    2774              : static bool
    2775        12605 : vect_check_scalar_mask (vec_info *vinfo,
    2776              :                         slp_tree slp_node, unsigned mask_index,
    2777              :                         slp_tree *mask_node,
    2778              :                         vect_def_type *mask_dt_out, tree *mask_vectype_out)
    2779              : {
    2780        12605 :   enum vect_def_type mask_dt;
    2781        12605 :   tree mask_vectype;
    2782        12605 :   slp_tree mask_node_1;
    2783        12605 :   tree mask_;
    2784        12605 :   if (!vect_is_simple_use (vinfo, slp_node, mask_index,
    2785              :                            &mask_, &mask_node_1, &mask_dt, &mask_vectype))
    2786              :     {
    2787            0 :       if (dump_enabled_p ())
    2788            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2789              :                          "mask use not simple.\n");
    2790              :       return false;
    2791              :     }
    2792              : 
    2793        12605 :   if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
    2794        12605 :       && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
    2795              :     {
    2796            0 :       if (dump_enabled_p ())
    2797            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2798              :                          "mask argument is not a boolean.\n");
    2799              :       return false;
    2800              :     }
    2801              : 
    2802        12605 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    2803        12605 :   if (!mask_vectype)
    2804           19 :     mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
    2805              :                                                   mask_node_1);
    2806              : 
    2807        12605 :   if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
    2808              :     {
    2809            0 :       if (dump_enabled_p ())
    2810            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2811              :                          "could not find an appropriate vector mask type.\n");
    2812              :       return false;
    2813              :     }
    2814              : 
    2815        12605 :   if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
    2816        25210 :                 TYPE_VECTOR_SUBPARTS (vectype)))
    2817              :     {
    2818            0 :       if (dump_enabled_p ())
    2819            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2820              :                          "vector mask type %T"
    2821              :                          " does not match vector data type %T.\n",
    2822              :                          mask_vectype, vectype);
    2823              : 
    2824              :       return false;
    2825              :     }
    2826              : 
    2827        12605 :   *mask_dt_out = mask_dt;
    2828        12605 :   *mask_vectype_out = mask_vectype;
    2829        12605 :   *mask_node = mask_node_1;
    2830        12605 :   return true;
    2831              : }
    2832              : 
    2833              : 
    2834              : /* Return true if stored value is suitable for vectorizing store
    2835              :    statement STMT_INFO.  When returning true, store the scalar stored
    2836              :    in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
    2837              :    the type of the vectorized store value in
    2838              :    *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT.  */
    2839              : 
    2840              : static bool
    2841      1387948 : vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
    2842              :                       slp_tree slp_node, slp_tree *rhs_node,
    2843              :                       vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
    2844              :                       vec_load_store_type *vls_type_out)
    2845              : {
    2846      1387948 :   int op_no = 0;
    2847      1387948 :   if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
    2848              :     {
    2849         1883 :       if (gimple_call_internal_p (call)
    2850         1883 :           && internal_store_fn_p (gimple_call_internal_fn (call)))
    2851         1883 :         op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
    2852              :     }
    2853      1387948 :   op_no = vect_slp_child_index_for_operand (stmt_info, op_no);
    2854              : 
    2855      1387948 :   enum vect_def_type rhs_dt;
    2856      1387948 :   tree rhs_vectype;
    2857      1387948 :   tree rhs;
    2858      1387948 :   if (!vect_is_simple_use (vinfo, slp_node, op_no,
    2859              :                            &rhs, rhs_node, &rhs_dt, &rhs_vectype))
    2860              :     {
    2861            0 :       if (dump_enabled_p ())
    2862            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2863              :                          "use not simple.\n");
    2864              :       return false;
    2865              :     }
    2866              : 
    2867              :   /* In the case this is a store from a constant make sure
    2868              :      native_encode_expr can handle it.  */
    2869      1387948 :   if (rhs_dt == vect_constant_def
    2870      1387948 :       && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
    2871              :     {
    2872            0 :       if (dump_enabled_p ())
    2873            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2874              :                          "cannot encode constant as a byte sequence.\n");
    2875              :       return false;
    2876              :     }
    2877              : 
    2878      1387948 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    2879      1387948 :   if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
    2880              :     {
    2881           24 :       if (dump_enabled_p ())
    2882           24 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2883              :                          "incompatible vector types.\n");
    2884              :       return false;
    2885              :     }
    2886              : 
    2887      1387924 :   *rhs_dt_out = rhs_dt;
    2888      1387924 :   *rhs_vectype_out = rhs_vectype;
    2889      1387924 :   if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
    2890      1026777 :     *vls_type_out = VLS_STORE_INVARIANT;
    2891              :   else
    2892       361147 :     *vls_type_out = VLS_STORE;
    2893              :   return true;
    2894              : }
    2895              : 
    2896              : /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
    2897              :    Note that we support masks with floating-point type, in which case the
    2898              :    floats are interpreted as a bitmask.  */
    2899              : 
    2900              : static tree
    2901          155 : vect_build_all_ones_mask (vec_info *vinfo,
    2902              :                           stmt_vec_info stmt_info, tree masktype)
    2903              : {
    2904          155 :   if (TREE_CODE (masktype) == INTEGER_TYPE)
    2905           98 :     return build_int_cst (masktype, -1);
    2906           57 :   else if (VECTOR_BOOLEAN_TYPE_P (masktype)
    2907          114 :            || TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
    2908              :     {
    2909            8 :       tree mask = build_int_cst (TREE_TYPE (masktype), -1);
    2910            8 :       mask = build_vector_from_val (masktype, mask);
    2911            8 :       return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
    2912              :     }
    2913           49 :   else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
    2914              :     {
    2915              :       REAL_VALUE_TYPE r;
    2916              :       long tmp[6];
    2917          343 :       for (int j = 0; j < 6; ++j)
    2918          294 :         tmp[j] = -1;
    2919           49 :       real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
    2920           49 :       tree mask = build_real (TREE_TYPE (masktype), r);
    2921           49 :       mask = build_vector_from_val (masktype, mask);
    2922           49 :       return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
    2923              :     }
    2924            0 :   gcc_unreachable ();
    2925              : }
    2926              : 
    2927              : /* Build an all-zero merge value of type VECTYPE while vectorizing
    2928              :    STMT_INFO as a gather load.  */
    2929              : 
    2930              : static tree
    2931          154 : vect_build_zero_merge_argument (vec_info *vinfo,
    2932              :                                 stmt_vec_info stmt_info, tree vectype)
    2933              : {
    2934          154 :   tree merge;
    2935          154 :   if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
    2936           49 :     merge = build_int_cst (TREE_TYPE (vectype), 0);
    2937          105 :   else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
    2938              :     {
    2939              :       REAL_VALUE_TYPE r;
    2940              :       long tmp[6];
    2941          735 :       for (int j = 0; j < 6; ++j)
    2942          630 :         tmp[j] = 0;
    2943          105 :       real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
    2944          105 :       merge = build_real (TREE_TYPE (vectype), r);
    2945              :     }
    2946              :   else
    2947            0 :     gcc_unreachable ();
    2948          154 :   merge = build_vector_from_val (vectype, merge);
    2949          154 :   return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
    2950              : }
    2951              : 
    2952              : /* Return the corresponding else value for an else value constant
    2953              :    ELSVAL with type TYPE.  */
    2954              : 
    2955              : tree
    2956         1860 : vect_get_mask_load_else (int elsval, tree type)
    2957              : {
    2958         1860 :   tree els;
    2959         1860 :   if (elsval == MASK_LOAD_ELSE_UNDEFINED)
    2960              :     {
    2961            0 :       tree tmp = create_tmp_var (type);
    2962              :       /* No need to warn about anything.  */
    2963            0 :       TREE_NO_WARNING (tmp) = 1;
    2964            0 :       els = get_or_create_ssa_default_def (cfun, tmp);
    2965              :     }
    2966         1860 :   else if (elsval == MASK_LOAD_ELSE_M1)
    2967            0 :     els = build_minus_one_cst (type);
    2968         1860 :   else if (elsval == MASK_LOAD_ELSE_ZERO)
    2969         1860 :     els = build_zero_cst (type);
    2970              :   else
    2971            0 :     gcc_unreachable ();
    2972              : 
    2973         1860 :   return els;
    2974              : }
    2975              : 
    2976              : /* Build a gather load call while vectorizing STMT_INFO.  Insert new
    2977              :    instructions before GSI and add them to VEC_STMT.  GS_INFO describes
    2978              :    the gather load operation.  If the load is conditional, MASK is the
    2979              :    vectorized condition, otherwise MASK is null.  PTR is the base
    2980              :    pointer and OFFSET is the vectorized offset.  */
    2981              : 
    2982              : static gimple *
    2983          337 : vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
    2984              :                                  slp_tree slp_node, tree vectype,
    2985              :                                  gimple_stmt_iterator *gsi, tree decl,
    2986              :                                  tree ptr, tree offset, tree mask)
    2987              : {
    2988          337 :   tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
    2989          337 :   tree rettype = TREE_TYPE (TREE_TYPE (decl));
    2990          337 :   tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
    2991          337 :   /* ptrtype */ arglist = TREE_CHAIN (arglist);
    2992          337 :   tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
    2993          337 :   tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
    2994          337 :   tree scaletype = TREE_VALUE (arglist);
    2995          337 :   tree var;
    2996          337 :   gcc_checking_assert (types_compatible_p (srctype, rettype)
    2997              :                        && (!mask
    2998              :                            || TREE_CODE (masktype) == INTEGER_TYPE
    2999              :                            || types_compatible_p (srctype, masktype)));
    3000              : 
    3001          337 :   tree op = offset;
    3002          337 :   if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
    3003              :     {
    3004           99 :       gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
    3005              :                             TYPE_VECTOR_SUBPARTS (idxtype)));
    3006           99 :       var = vect_get_new_ssa_name (idxtype, vect_simple_var);
    3007           99 :       op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
    3008           99 :       gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
    3009           99 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3010           99 :       op = var;
    3011              :     }
    3012              : 
    3013          337 :   tree src_op = NULL_TREE;
    3014          337 :   tree mask_op = NULL_TREE;
    3015          337 :   if (mask)
    3016              :     {
    3017          183 :       if (!useless_type_conversion_p (masktype, TREE_TYPE (mask)))
    3018              :         {
    3019          183 :           tree utype, optype = TREE_TYPE (mask);
    3020          183 :           if (VECTOR_TYPE_P (masktype)
    3021          183 :               || TYPE_MODE (masktype) == TYPE_MODE (optype))
    3022              :             utype = masktype;
    3023              :           else
    3024            6 :             utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
    3025          183 :           var = vect_get_new_ssa_name (utype, vect_scalar_var);
    3026          183 :           tree mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask);
    3027          183 :           gassign *new_stmt
    3028          183 :               = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
    3029          183 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3030          183 :           mask_arg = var;
    3031          183 :           if (!useless_type_conversion_p (masktype, utype))
    3032              :             {
    3033            6 :               gcc_assert (TYPE_PRECISION (utype)
    3034              :                           <= TYPE_PRECISION (masktype));
    3035            6 :               var = vect_get_new_ssa_name (masktype, vect_scalar_var);
    3036            6 :               new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
    3037            6 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3038            6 :               mask_arg = var;
    3039              :             }
    3040          183 :           src_op = build_zero_cst (srctype);
    3041          183 :           mask_op = mask_arg;
    3042              :         }
    3043              :       else
    3044              :         {
    3045              :           src_op = mask;
    3046              :           mask_op = mask;
    3047              :         }
    3048              :     }
    3049              :   else
    3050              :     {
    3051          154 :       src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
    3052          154 :       mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
    3053              :     }
    3054              : 
    3055          337 :   tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
    3056          337 :   gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op,
    3057              :                                         mask_op, scale);
    3058              : 
    3059          337 :   if (!useless_type_conversion_p (vectype, rettype))
    3060              :     {
    3061           52 :       gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
    3062              :                             TYPE_VECTOR_SUBPARTS (rettype)));
    3063           52 :       op = vect_get_new_ssa_name (rettype, vect_simple_var);
    3064           52 :       gimple_call_set_lhs (new_stmt, op);
    3065           52 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3066           52 :       op = build1 (VIEW_CONVERT_EXPR, vectype, op);
    3067           52 :       new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, op);
    3068              :     }
    3069              : 
    3070          337 :   return new_stmt;
    3071              : }
    3072              : 
    3073              : /* Build a scatter store call while vectorizing STMT_INFO.  Insert new
    3074              :    instructions before GSI.  GS_INFO describes the scatter store operation.
    3075              :    PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
    3076              :    vectorized data to store.
    3077              :    If the store is conditional, MASK is the vectorized condition, otherwise
    3078              :    MASK is null.  */
    3079              : 
    3080              : static gimple *
    3081          161 : vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
    3082              :                                    slp_tree slp_node,
    3083              :                                    gimple_stmt_iterator *gsi,
    3084              :                                    tree decl,
    3085              :                                    tree ptr, tree offset, tree oprnd, tree mask)
    3086              : {
    3087          161 :   tree rettype = TREE_TYPE (TREE_TYPE (decl));
    3088          161 :   tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
    3089          161 :   /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist);
    3090          161 :   tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
    3091          161 :   tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
    3092          161 :   tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
    3093          161 :   tree scaletype = TREE_VALUE (arglist);
    3094          161 :   gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
    3095              :                        && TREE_CODE (rettype) == VOID_TYPE);
    3096              : 
    3097          161 :   tree mask_arg = NULL_TREE;
    3098          161 :   if (mask)
    3099              :     {
    3100          110 :       mask_arg = mask;
    3101          110 :       tree optype = TREE_TYPE (mask_arg);
    3102          110 :       tree utype;
    3103          110 :       if (TYPE_MODE (masktype) == TYPE_MODE (optype))
    3104              :         utype = masktype;
    3105              :       else
    3106            8 :         utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
    3107          110 :       tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
    3108          110 :       mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
    3109          110 :       gassign *new_stmt
    3110          110 :         = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
    3111          110 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3112          110 :       mask_arg = var;
    3113          110 :       if (!useless_type_conversion_p (masktype, utype))
    3114              :         {
    3115            8 :           gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
    3116            8 :           tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
    3117            8 :           new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
    3118            8 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3119            8 :           mask_arg = var;
    3120              :         }
    3121              :     }
    3122              :   else
    3123              :     {
    3124           51 :       mask_arg = build_int_cst (masktype, -1);
    3125           51 :       mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
    3126              :     }
    3127              : 
    3128          161 :   tree src = oprnd;
    3129          161 :   if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
    3130              :     {
    3131            0 :       gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
    3132              :                             TYPE_VECTOR_SUBPARTS (srctype)));
    3133            0 :       tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
    3134            0 :       src = build1 (VIEW_CONVERT_EXPR, srctype, src);
    3135            0 :       gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
    3136            0 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3137            0 :       src = var;
    3138              :     }
    3139              : 
    3140          161 :   tree op = offset;
    3141          161 :   if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
    3142              :     {
    3143           16 :       gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
    3144              :                             TYPE_VECTOR_SUBPARTS (idxtype)));
    3145           16 :       tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
    3146           16 :       op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
    3147           16 :       gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
    3148           16 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3149           16 :       op = var;
    3150              :     }
    3151              : 
    3152          161 :   tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
    3153          161 :   gcall *new_stmt
    3154          161 :     = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale);
    3155          161 :   return new_stmt;
    3156              : }
    3157              : 
    3158              : /* Prepare the base and offset in GS_INFO for vectorization.
    3159              :    Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
    3160              :    to the vectorized offset argument for the first copy of STMT_INFO.
    3161              :    STMT_INFO is the statement described by GS_INFO and LOOP is the
    3162              :    containing loop.  */
    3163              : 
    3164              : static void
    3165         1252 : vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
    3166              :                              tree *dataref_ptr, vec<tree> *vec_offset)
    3167              : {
    3168         1252 :   gimple_seq stmts = NULL;
    3169         1252 :   *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node),
    3170              :                                        &stmts, true, NULL_TREE);
    3171         1252 :   if (stmts != NULL)
    3172              :     {
    3173         1023 :       basic_block new_bb;
    3174         1023 :       edge pe = loop_preheader_edge (loop);
    3175         1023 :       new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
    3176         1023 :       gcc_assert (!new_bb);
    3177              :     }
    3178         1252 :   vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
    3179         1252 : }
    3180              : 
    3181              : /* Prepare to implement a grouped or strided load or store using
    3182              :    the gather load or scatter store operation described by GS_INFO.
    3183              :    STMT_INFO is the load or store statement.
    3184              : 
    3185              :    Set *DR_STEP to the amount that should be added to pointer base address
    3186              :    to get to the next iteration's base address.
    3187              :    Set *DR_BUMP to the amount that should be added to the base
    3188              :    address after each copy of the vectorized statement in a grouped read.
    3189              :    Set *VEC_OFFSET to an invariant offset vector in which element I has the
    3190              :    value I * DR_STEP / SCALE.  */
    3191              : 
    3192              : static void
    3193            0 : vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
    3194              :                                  tree vectype, tree offset_vectype,
    3195              :                                  loop_vec_info loop_vinfo,
    3196              :                                  gimple_stmt_iterator *gsi,
    3197              :                                  tree *dr_step, tree *dr_bump,
    3198              :                                  tree *vec_offset)
    3199              : {
    3200            0 :   struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
    3201              : 
    3202            0 :   tree dr_step_temp
    3203            0 :     = size_binop (MULT_EXPR,
    3204              :                   fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
    3205              :                   LOOP_VINFO_IV_INCREMENT (loop_vinfo));
    3206            0 :   *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
    3207            0 :              ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
    3208            0 :              : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
    3209              :                                          true, GSI_SAME_STMT);
    3210            0 :   tree bump = size_binop (MULT_EXPR,
    3211              :                           fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
    3212              :                           size_int (TYPE_VECTOR_SUBPARTS (vectype)));
    3213            0 :   *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
    3214              : 
    3215            0 :   internal_fn ifn
    3216            0 :     = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
    3217            0 :   if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
    3218              :     {
    3219            0 :       *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo,
    3220              :                                                    unshare_expr (DR_STEP (dr)));
    3221            0 :       return;
    3222              :     }
    3223              : 
    3224              :   /* The offset given in GS_INFO can have pointer type, so use the element
    3225              :      type of the vector instead.  */
    3226            0 :   tree offset_type = TREE_TYPE (offset_vectype);
    3227              : 
    3228              :   /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type.  */
    3229            0 :   tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)),
    3230              :                           ssize_int (SLP_TREE_GS_SCALE (node)));
    3231            0 :   step = fold_convert (offset_type, step);
    3232              : 
    3233              :   /* Create {0, X, X*2, X*3, ...}.  */
    3234            0 :   tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype,
    3235              :                              build_zero_cst (offset_type), step);
    3236            0 :   *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
    3237              : }
    3238              : 
    3239              : /* Return the amount that should be added to a vector pointer, represented by
    3240              :    DR_INFO, to increment to the next vectorized iteration.  */
    3241              : 
    3242              : static tree
    3243       698568 : vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
    3244              :                         vect_memory_access_type memory_access_type)
    3245              : {
    3246       698568 :   if (memory_access_type == VMAT_INVARIANT)
    3247            0 :     return size_zero_node;
    3248              : 
    3249       698568 :   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
    3250              : 
    3251              :   /* For BB SLP there is no next iteration.  */
    3252       698568 :   if (!loop_vinfo)
    3253       559420 :     return build_zero_cst (sizetype);
    3254              : 
    3255       139148 :   tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
    3256              : 
    3257              :   /* gather/scatter never reach here.  */
    3258       139148 :   gcc_assert (!mat_gather_scatter_p (memory_access_type));
    3259              : 
    3260       139148 :   tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
    3261              : 
    3262       139148 :   return fold_build2 (MULT_EXPR, sizetype, iv_increment,
    3263              :                       fold_convert (sizetype, step));
    3264              : }
    3265              : 
    3266              : /* Return the amount that should be added to a vector pointer to move
    3267              :    to the next or previous copy of AGGR_TYPE.  DR_INFO is the data reference
    3268              :    being vectorized and MEMORY_ACCESS_TYPE describes the type of
    3269              :    vectorization.  */
    3270              : 
    3271              : static tree
    3272       699080 : vect_get_data_ptr_bump (vec_info *vinfo,
    3273              :                         dr_vec_info *dr_info, tree aggr_type,
    3274              :                         vect_memory_access_type memory_access_type)
    3275              : {
    3276       699080 :   if (memory_access_type == VMAT_INVARIANT)
    3277            0 :     return size_zero_node;
    3278              : 
    3279       699080 :   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
    3280              :   /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value
    3281              :      should never be needed.  */
    3282       139660 :   if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
    3283              :     return NULL_TREE;
    3284              : 
    3285       699080 :   tree iv_step = TYPE_SIZE_UNIT (aggr_type);
    3286       699080 :   tree step = vect_dr_behavior (vinfo, dr_info)->step;
    3287       699080 :   if (tree_int_cst_sgn (step) == -1)
    3288         2899 :     iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
    3289              :   return iv_step;
    3290              : }
    3291              : 
    3292              : /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}.  */
    3293              : 
    3294              : static bool
    3295          156 : vectorizable_bswap (vec_info *vinfo,
    3296              :                     stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    3297              :                     slp_tree slp_node,
    3298              :                     slp_tree *slp_op,
    3299              :                     tree vectype_in, stmt_vector_for_cost *cost_vec)
    3300              : {
    3301          156 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    3302          156 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    3303              : 
    3304          156 :   if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
    3305              :     {
    3306            0 :       if (dump_enabled_p ())
    3307            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3308              :                          "mismatched vector sizes %T and %T\n",
    3309              :                          vectype_in, vectype);
    3310              :       return false;
    3311              :     }
    3312              : 
    3313          156 :   tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
    3314          156 :   if (! char_vectype)
    3315              :     return false;
    3316              : 
    3317          156 :   poly_uint64 num_bytes = TYPE_VECTOR_SUBPARTS (char_vectype);
    3318          156 :   unsigned word_bytes;
    3319          156 :   if (!constant_multiple_p (num_bytes, nunits, &word_bytes))
    3320              :     return false;
    3321              : 
    3322              :   /* The encoding uses one stepped pattern for each byte in the word.  */
    3323          156 :   vec_perm_builder elts (num_bytes, word_bytes, 3);
    3324          780 :   for (unsigned i = 0; i < 3; ++i)
    3325         2772 :     for (unsigned j = 0; j < word_bytes; ++j)
    3326         2304 :       elts.quick_push ((i + 1) * word_bytes - j - 1);
    3327              : 
    3328          156 :   vec_perm_indices indices (elts, 1, num_bytes);
    3329          156 :   machine_mode vmode = TYPE_MODE (char_vectype);
    3330          156 :   if (!can_vec_perm_const_p (vmode, vmode, indices))
    3331              :     return false;
    3332              : 
    3333           61 :   if (cost_vec)
    3334              :     {
    3335           45 :       if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
    3336              :         {
    3337            0 :           if (dump_enabled_p ())
    3338            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3339              :                              "incompatible vector types for invariants\n");
    3340              :           return false;
    3341              :         }
    3342              : 
    3343           45 :       SLP_TREE_TYPE (slp_node) = call_vec_info_type;
    3344           45 :       DUMP_VECT_SCOPE ("vectorizable_bswap");
    3345           45 :       record_stmt_cost (cost_vec,
    3346              :                         1, vector_stmt, slp_node, 0, vect_prologue);
    3347           45 :       record_stmt_cost (cost_vec,
    3348           45 :                         vect_get_num_copies (vinfo, slp_node),
    3349              :                         vec_perm, slp_node, 0, vect_body);
    3350           45 :       return true;
    3351              :     }
    3352              : 
    3353           16 :   tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
    3354              : 
    3355              :   /* Transform.  */
    3356           16 :   vec<tree> vec_oprnds = vNULL;
    3357           16 :   vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
    3358              :   /* Arguments are ready. create the new vector stmt.  */
    3359           16 :   unsigned i;
    3360           16 :   tree vop;
    3361           48 :   FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
    3362              :     {
    3363           16 :       gimple *new_stmt;
    3364           16 :       tree tem = make_ssa_name (char_vectype);
    3365           16 :       new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
    3366              :                                                    char_vectype, vop));
    3367           16 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3368           16 :       tree tem2 = make_ssa_name (char_vectype);
    3369           16 :       new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
    3370              :                                       tem, tem, bswap_vconst);
    3371           16 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3372           16 :       tem = make_ssa_name (vectype);
    3373           16 :       new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
    3374              :                                                    vectype, tem2));
    3375           16 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3376           16 :       slp_node->push_vec_def (new_stmt);
    3377              :     }
    3378              : 
    3379           16 :   vec_oprnds.release ();
    3380           16 :   return true;
    3381          156 : }
    3382              : 
    3383              : /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
    3384              :    integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
    3385              :    in a single step.  On success, store the binary pack code in
    3386              :    *CONVERT_CODE.  */
    3387              : 
    3388              : static bool
    3389          197 : simple_integer_narrowing (tree vectype_out, tree vectype_in,
    3390              :                           code_helper *convert_code)
    3391              : {
    3392          394 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
    3393          394 :       || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
    3394              :     return false;
    3395              : 
    3396           81 :   code_helper code;
    3397           81 :   int multi_step_cvt = 0;
    3398           81 :   auto_vec <tree, 8> interm_types;
    3399          121 :   if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
    3400              :                                         &code, &multi_step_cvt, &interm_types)
    3401           81 :       || multi_step_cvt)
    3402              :     return false;
    3403              : 
    3404           41 :   *convert_code = code;
    3405           41 :   return true;
    3406           81 : }
    3407              : 
    3408              : /* Function vectorizable_call.
    3409              : 
    3410              :    Check if STMT_INFO performs a function call that can be vectorized.
    3411              :    If COST_VEC is passed, calculate costs but don't change anything,
    3412              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    3413              :    it, and insert it at GSI.
    3414              :    Return true if STMT_INFO is vectorizable in this way.  */
    3415              : 
    3416              : static bool
    3417      2678571 : vectorizable_call (vec_info *vinfo,
    3418              :                    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    3419              :                    slp_tree slp_node,
    3420              :                    stmt_vector_for_cost *cost_vec)
    3421              : {
    3422      2678571 :   gcall *stmt;
    3423      2678571 :   tree vec_dest;
    3424      2678571 :   tree scalar_dest;
    3425      2678571 :   tree vec_oprnd0 = NULL_TREE;
    3426      2678571 :   tree vectype_out, vectype_in;
    3427      2678571 :   poly_uint64 nunits_in;
    3428      2678571 :   poly_uint64 nunits_out;
    3429      2678571 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    3430      2678571 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    3431      2678571 :   tree fndecl, new_temp, rhs_type;
    3432      2678571 :   enum vect_def_type dt[5]
    3433              :     = { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
    3434              :         vect_unknown_def_type, vect_unknown_def_type };
    3435      2678571 :   tree vectypes[ARRAY_SIZE (dt)] = {};
    3436      2678571 :   slp_tree slp_op[ARRAY_SIZE (dt)] = {};
    3437      2678571 :   auto_vec<tree, 8> vargs;
    3438      2678571 :   enum { NARROW, NONE, WIDEN } modifier;
    3439      2678571 :   size_t i, nargs;
    3440      2678571 :   tree clz_ctz_arg1 = NULL_TREE;
    3441              : 
    3442      2678571 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    3443              :     return false;
    3444              : 
    3445      2678571 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    3446       251571 :       && cost_vec)
    3447              :     return false;
    3448              : 
    3449              :   /* Is STMT_INFO a vectorizable call?   */
    3450      2706651 :   stmt = dyn_cast <gcall *> (stmt_info->stmt);
    3451        28080 :   if (!stmt)
    3452              :     return false;
    3453              : 
    3454        28080 :   if (gimple_call_internal_p (stmt)
    3455        28080 :       && (internal_load_fn_p (gimple_call_internal_fn (stmt))
    3456        17481 :           || internal_store_fn_p (gimple_call_internal_fn (stmt))))
    3457              :     /* Handled by vectorizable_load and vectorizable_store.  */
    3458              :     return false;
    3459              : 
    3460        24571 :   if (gimple_call_lhs (stmt) == NULL_TREE
    3461        24571 :       || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
    3462              :     return false;
    3463              : 
    3464        24557 :   gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
    3465              : 
    3466        24557 :   vectype_out = SLP_TREE_VECTYPE (slp_node);
    3467              : 
    3468              :   /* Process function arguments.  */
    3469        24557 :   rhs_type = NULL_TREE;
    3470        24557 :   vectype_in = NULL_TREE;
    3471        24557 :   nargs = gimple_call_num_args (stmt);
    3472              : 
    3473              :   /* Bail out if the function has more than four arguments, we do not have
    3474              :      interesting builtin functions to vectorize with more than two arguments
    3475              :      except for fma (cond_fma has more).  No arguments is also not good.  */
    3476        24557 :   if (nargs == 0 || nargs > 5)
    3477              :     return false;
    3478              : 
    3479              :   /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic.  */
    3480        24477 :   combined_fn cfn = gimple_call_combined_fn (stmt);
    3481        24477 :   if (cfn == CFN_GOMP_SIMD_LANE)
    3482              :     {
    3483         3205 :       nargs = 0;
    3484         3205 :       rhs_type = unsigned_type_node;
    3485              :     }
    3486              :   /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
    3487              :      argument just says whether it is well-defined at zero or not and what
    3488              :      value should be returned for it.  */
    3489        24477 :   if ((cfn == CFN_CLZ || cfn == CFN_CTZ) && nargs == 2)
    3490              :     {
    3491          194 :       nargs = 1;
    3492          194 :       clz_ctz_arg1 = gimple_call_arg (stmt, 1);
    3493              :     }
    3494              : 
    3495        24477 :   int mask_opno = -1;
    3496        24477 :   if (internal_fn_p (cfn))
    3497              :     {
    3498              :       /* We can only handle direct internal masked calls here,
    3499              :          vectorizable_simd_clone_call is for the rest.  */
    3500        19744 :       if (cfn == CFN_MASK_CALL)
    3501              :         return false;
    3502        19573 :       mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
    3503              :     }
    3504              : 
    3505        75052 :   for (i = 0; i < nargs; i++)
    3506              :     {
    3507        52859 :       if ((int) i == mask_opno)
    3508              :         {
    3509         7926 :           if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
    3510              :                                        &slp_op[i], &dt[i], &vectypes[i]))
    3511              :             return false;
    3512         7926 :           continue;
    3513              :         }
    3514              : 
    3515        44933 :       if (!vect_is_simple_use (vinfo, slp_node,
    3516              :                                i, &slp_op[i], &dt[i], &vectypes[i]))
    3517              :         {
    3518            0 :           if (dump_enabled_p ())
    3519            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3520              :                              "use not simple.\n");
    3521              :           return false;
    3522              :         }
    3523              : 
    3524              :       /* We can only handle calls with arguments of the same type.  */
    3525        44933 :       tree op = gimple_call_arg (stmt, i);
    3526        44933 :       if (rhs_type
    3527        44933 :           && !types_compatible_p (rhs_type, TREE_TYPE (op)))
    3528              :         {
    3529         2113 :           if (dump_enabled_p ())
    3530          226 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3531              :                              "argument types differ.\n");
    3532              :           return false;
    3533              :         }
    3534        42820 :       if (!rhs_type)
    3535        21101 :         rhs_type = TREE_TYPE (op);
    3536              : 
    3537        42820 :       if (!vectype_in)
    3538        22402 :         vectype_in = vectypes[i];
    3539        20418 :       else if (vectypes[i]
    3540        20418 :                && !types_compatible_p (vectypes[i], vectype_in))
    3541              :         {
    3542            0 :           if (dump_enabled_p ())
    3543            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3544              :                              "argument vector types differ.\n");
    3545              :           return false;
    3546              :         }
    3547              :     }
    3548              :   /* If all arguments are external or constant defs, infer the vector type
    3549              :      from the scalar type.  */
    3550        22193 :   if (!vectype_in)
    3551         5842 :     vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
    3552        22193 :   if (!cost_vec)
    3553         4611 :     gcc_assert (vectype_in);
    3554        17582 :   if (!vectype_in)
    3555              :     {
    3556         1102 :       if (dump_enabled_p ())
    3557            2 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3558              :                          "no vectype for scalar type %T\n", rhs_type);
    3559              : 
    3560              :       return false;
    3561              :     }
    3562              : 
    3563              :   /* During vect-patterns we remembered the defined-at-zero value in the
    3564              :      optional second argument for the natural vector mode.  Now, once we have
    3565              :      the final vector mode, verify that they coincide. If not, bail out.  */
    3566        21091 :   if (clz_ctz_arg1)
    3567              :     {
    3568          194 :       int val;
    3569          194 :       int d = (cfn == CFN_CLZ
    3570          194 :                ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val)
    3571          199 :                : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val));
    3572          194 :       if (d != 2 || val != tree_to_shwi (clz_ctz_arg1))
    3573              :         return false;
    3574              :     }
    3575              : 
    3576        42130 :   if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
    3577        21065 :       != VECTOR_BOOLEAN_TYPE_P (vectype_in))
    3578              :     {
    3579           12 :       if (dump_enabled_p ())
    3580           12 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3581              :                          "mixed mask and nonmask vector types\n");
    3582              :       return false;
    3583              :     }
    3584              : 
    3585        21053 :   if (vect_emulated_vector_p (vectype_in)
    3586        21053 :       || vect_emulated_vector_p (vectype_out))
    3587              :     {
    3588            0 :       if (dump_enabled_p ())
    3589            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3590              :                          "use emulated vector type for call\n");
    3591              :       return false;
    3592              :     }
    3593              : 
    3594              :   /* FORNOW */
    3595        21053 :   nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
    3596        21053 :   nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
    3597        21053 :   if (known_eq (nunits_in * 2, nunits_out))
    3598              :     modifier = NARROW;
    3599        20457 :   else if (known_eq (nunits_out, nunits_in))
    3600              :     modifier = NONE;
    3601           50 :   else if (known_eq (nunits_out * 2, nunits_in))
    3602              :     modifier = WIDEN;
    3603              :   else
    3604              :     return false;
    3605              : 
    3606              :   /* We only handle functions that do not read or clobber memory.  */
    3607        42106 :   if (gimple_vuse (stmt))
    3608              :     {
    3609         1427 :       if (dump_enabled_p ())
    3610           18 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3611              :                          "function reads from or writes to memory.\n");
    3612              :       return false;
    3613              :     }
    3614              : 
    3615              :   /* For now, we only vectorize functions if a target specific builtin
    3616              :      is available.  TODO -- in some cases, it might be profitable to
    3617              :      insert the calls for pieces of the vector, in order to be able
    3618              :      to vectorize other operations in the loop.  */
    3619        19626 :   fndecl = NULL_TREE;
    3620        19626 :   internal_fn ifn = IFN_LAST;
    3621        19626 :   tree callee = gimple_call_fndecl (stmt);
    3622              : 
    3623              :   /* First try using an internal function.  */
    3624        19626 :   code_helper convert_code = MAX_TREE_CODES;
    3625        19626 :   if (cfn != CFN_LAST
    3626        19626 :       && (modifier == NONE
    3627          209 :           || (modifier == NARROW
    3628          197 :               && simple_integer_narrowing (vectype_out, vectype_in,
    3629              :                                            &convert_code))))
    3630        18575 :     ifn = vectorizable_internal_function (cfn, callee, vectype_out,
    3631              :                                           vectype_in);
    3632              : 
    3633              :   /* Check if the operation traps.  */
    3634        19626 :   bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
    3635        19626 :   if (could_trap && cost_vec && loop_vinfo)
    3636              :     {
    3637              :       /* If the operation can trap it must be conditional, otherwise fail.  */
    3638          577 :       internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
    3639          577 :                              ? ifn : get_conditional_internal_fn (ifn));
    3640          577 :       internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
    3641          577 :       if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
    3642              :         {
    3643              :           /* We assume that BB SLP fills all lanes, so no inactive lanes can
    3644              :              cause issues.  */
    3645          136 :           if ((cond_fn == IFN_LAST
    3646           76 :                || !direct_internal_fn_supported_p (cond_fn, vectype_out,
    3647              :                                                    OPTIMIZE_FOR_SPEED))
    3648          212 :               && (cond_len_fn == IFN_LAST
    3649           76 :                   || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
    3650              :                                                       OPTIMIZE_FOR_SPEED)))
    3651              :             {
    3652          136 :               if (dump_enabled_p ())
    3653           10 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3654              :                                  "can't use a fully-masked loop because no"
    3655              :                                  " conditional operation is available.\n");
    3656          136 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    3657              :             }
    3658              :         }
    3659              :     }
    3660              : 
    3661              :   /* If that fails, try asking for a target-specific built-in function.  */
    3662        19626 :   if (ifn == IFN_LAST)
    3663              :     {
    3664        11322 :       if (cfn != CFN_LAST)
    3665        10439 :         fndecl = targetm.vectorize.builtin_vectorized_function
    3666        10439 :           (cfn, vectype_out, vectype_in);
    3667          883 :       else if (callee && fndecl_built_in_p (callee, BUILT_IN_MD))
    3668           24 :         fndecl = targetm.vectorize.builtin_md_vectorized_function
    3669           24 :           (callee, vectype_out, vectype_in);
    3670              :     }
    3671              : 
    3672        19626 :   if (ifn == IFN_LAST && !fndecl)
    3673              :     {
    3674        10179 :       if (cfn == CFN_GOMP_SIMD_LANE
    3675         3205 :           && SLP_TREE_LANES (slp_node) == 1
    3676         3205 :           && loop_vinfo
    3677         3205 :           && LOOP_VINFO_LOOP (loop_vinfo)->simduid
    3678         3205 :           && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
    3679        16589 :           && LOOP_VINFO_LOOP (loop_vinfo)->simduid
    3680         3205 :              == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
    3681              :         {
    3682              :           /* We can handle IFN_GOMP_SIMD_LANE by returning a
    3683              :              { 0, 1, 2, ... vf - 1 } vector.  */
    3684         3205 :           gcc_assert (nargs == 0);
    3685              :         }
    3686         6974 :       else if (modifier == NONE
    3687         6974 :                && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
    3688         6607 :                    || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
    3689         6538 :                    || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
    3690         6485 :                    || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
    3691          156 :         return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
    3692          156 :                                    slp_op, vectype_in, cost_vec);
    3693              :       else
    3694              :         {
    3695         6818 :           if (dump_enabled_p ())
    3696          301 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3697              :                              "function is not vectorizable.\n");
    3698              :           return false;
    3699              :         }
    3700              :     }
    3701              : 
    3702        12652 :   int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
    3703        12652 :   internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
    3704        12652 :                          ? ifn : get_conditional_internal_fn (ifn));
    3705        12652 :   internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
    3706        12652 :   vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
    3707        10437 :   vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
    3708        12652 :   unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
    3709        12652 :   if (cost_vec) /* transformation not required.  */
    3710              :     {
    3711        24954 :       for (i = 0; i < nargs; ++i)
    3712        16897 :         if (!vect_maybe_update_slp_op_vectype (slp_op[i],
    3713        16897 :                                                vectypes[i]
    3714              :                                                ? vectypes[i] : vectype_in))
    3715              :           {
    3716            0 :             if (dump_enabled_p ())
    3717            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3718              :                                "incompatible vector types for invariants\n");
    3719              :             return false;
    3720              :           }
    3721         8057 :       SLP_TREE_TYPE (slp_node) = call_vec_info_type;
    3722         8057 :       DUMP_VECT_SCOPE ("vectorizable_call");
    3723         8057 :       vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
    3724              : 
    3725         8057 :       if (loop_vinfo
    3726         6828 :           && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    3727         4668 :           && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
    3728              :         {
    3729         2618 :           if (reduc_idx >= 0
    3730         1631 :               && (cond_fn == IFN_LAST
    3731         1631 :                   || !direct_internal_fn_supported_p (cond_fn, vectype_out,
    3732              :                                                       OPTIMIZE_FOR_SPEED))
    3733         2630 :               && (cond_len_fn == IFN_LAST
    3734           12 :                   || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
    3735              :                                                       OPTIMIZE_FOR_SPEED)))
    3736              :             {
    3737           12 :               if (dump_enabled_p ())
    3738            6 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3739              :                                  "can't use a fully-masked loop because no"
    3740              :                                  " conditional operation is available.\n");
    3741           12 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    3742              :             }
    3743              :           else
    3744              :             {
    3745         2606 :               tree scalar_mask = NULL_TREE;
    3746         2606 :               if (mask_opno >= 0)
    3747         2606 :                 scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
    3748         2606 :               if (cond_len_fn != IFN_LAST
    3749         2606 :                   && direct_internal_fn_supported_p (cond_len_fn, vectype_out,
    3750              :                                                      OPTIMIZE_FOR_SPEED))
    3751            0 :                 vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
    3752              :                                       1);
    3753              :               else
    3754         2606 :                 vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
    3755              :                                        scalar_mask);
    3756              :             }
    3757              :         }
    3758         8057 :       return true;
    3759              :     }
    3760              : 
    3761              :   /* Transform.  */
    3762              : 
    3763         4595 :   if (dump_enabled_p ())
    3764          428 :     dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
    3765              : 
    3766              :   /* Handle def.  */
    3767         4595 :   scalar_dest = gimple_call_lhs (stmt);
    3768         4595 :   vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
    3769              : 
    3770         4595 :   bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
    3771         3609 :   bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
    3772         4595 :   unsigned int vect_nargs = nargs;
    3773         4595 :   if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
    3774              :     {
    3775            0 :       ifn = cond_len_fn;
    3776              :       /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS.  */
    3777            0 :       vect_nargs += 2;
    3778              :       /* But unless there's a mask argument already we need that
    3779              :          as well, and an else value.  */
    3780            0 :       if (mask_opno == -1)
    3781            0 :         vect_nargs += 2;
    3782              :     }
    3783         4595 :   else if (masked_loop_p && mask_opno == -1 && (reduc_idx >= 0 || could_trap))
    3784              :     {
    3785            0 :       ifn = cond_fn;
    3786            0 :       vect_nargs += 2;
    3787              :     }
    3788         4595 :   int len_opno = internal_fn_len_index (ifn);
    3789         4595 :   if (clz_ctz_arg1)
    3790           59 :     ++vect_nargs;
    3791              : 
    3792         4595 :   if (modifier == NONE || ifn != IFN_LAST)
    3793              :     {
    3794         4563 :       tree prev_res = NULL_TREE;
    3795         4563 :       vargs.safe_grow (vect_nargs, true);
    3796         4563 :       auto_vec<vec<tree> > vec_defs (nargs);
    3797              : 
    3798              :       /* Build argument list for the vectorized call.  */
    3799         4563 :       if (cfn == CFN_GOMP_SIMD_LANE)
    3800              :         {
    3801         3306 :           for (i = 0; i < nvectors; ++i)
    3802              :             {
    3803              :               /* ???  For multi-lane SLP we'd need to build
    3804              :                  { 0, 0, .., 1, 1, ... }.  */
    3805         1707 :               tree cst = build_index_vector (vectype_out,
    3806              :                                              i * nunits_out, 1);
    3807         1707 :               tree new_var
    3808         1707 :                 = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
    3809         1707 :               gimple *init_stmt = gimple_build_assign (new_var, cst);
    3810         1707 :               vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
    3811         1707 :               new_temp = make_ssa_name (vec_dest);
    3812         1707 :               gimple *new_stmt = gimple_build_assign (new_temp, new_var);
    3813         1707 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3814         1707 :               slp_node->push_vec_def (new_stmt);
    3815              :             }
    3816              :         }
    3817              :       else
    3818              :         {
    3819         2964 :           vec<tree> vec_oprnds0;
    3820         2964 :           vect_get_slp_defs (vinfo, slp_node, &vec_defs);
    3821         2964 :           vec_oprnds0 = vec_defs[0];
    3822              : 
    3823              :           /* Arguments are ready.  Create the new vector stmt.  */
    3824         6134 :           FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
    3825              :             {
    3826         3170 :               int varg = 0;
    3827              :               /* Add the mask if necessary.  */
    3828           38 :               if ((masked_loop_p || len_loop_p) && mask_opno == -1
    3829         3172 :                   && internal_fn_mask_index (ifn) != -1)
    3830              :                 {
    3831            0 :                   gcc_assert (internal_fn_mask_index (ifn) == varg);
    3832            0 :                   if (masked_loop_p)
    3833              :                     {
    3834            0 :                       unsigned int vec_num = vec_oprnds0.length ();
    3835            0 :                       vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
    3836              :                                                           masks, vec_num,
    3837              :                                                           vectype_out, i);
    3838              :                     }
    3839              :                   else
    3840              :                     {
    3841            0 :                       tree mask_vectype = truth_type_for (vectype_out);
    3842            0 :                       vargs[varg++] = vect_build_all_ones_mask (loop_vinfo,
    3843              :                                                                 stmt_info,
    3844              :                                                                 mask_vectype);
    3845              :                     }
    3846              :                 }
    3847         3170 :               size_t k;
    3848        11307 :               for (k = 0; k < nargs; k++)
    3849              :                 {
    3850         8137 :                   vec<tree> vec_oprndsk = vec_defs[k];
    3851         8137 :                   vargs[varg++] = vec_oprndsk[i];
    3852              :                 }
    3853              :               /* Add the else value if necessary.  */
    3854           38 :               if ((masked_loop_p || len_loop_p) && mask_opno == -1
    3855         3172 :                   && internal_fn_else_index (ifn) != -1)
    3856              :                 {
    3857            0 :                   gcc_assert (internal_fn_else_index (ifn) == varg);
    3858            0 :                   if (reduc_idx >= 0)
    3859            0 :                     vargs[varg++] = vargs[reduc_idx + 1];
    3860              :                   else
    3861              :                     {
    3862            0 :                       auto else_value = targetm.preferred_else_value
    3863            0 :                         (ifn, vectype_out, varg - 1, &vargs[1]);
    3864            0 :                       vargs[varg++] = else_value;
    3865              :                     }
    3866              :                 }
    3867         3170 :               if (clz_ctz_arg1)
    3868           59 :                 vargs[varg++] = clz_ctz_arg1;
    3869              : 
    3870         3170 :               gimple *new_stmt;
    3871         3170 :               if (modifier == NARROW)
    3872              :                 {
    3873              :                   /* We don't define any narrowing conditional functions
    3874              :                      at present.  */
    3875            0 :                   gcc_assert (mask_opno < 0);
    3876            0 :                   tree half_res = make_ssa_name (vectype_in);
    3877            0 :                   gcall *call = gimple_build_call_internal_vec (ifn, vargs);
    3878            0 :                   gimple_call_set_lhs (call, half_res);
    3879            0 :                   gimple_call_set_nothrow (call, true);
    3880            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    3881            0 :                   if ((i & 1) == 0)
    3882              :                     {
    3883            0 :                       prev_res = half_res;
    3884            0 :                       continue;
    3885              :                     }
    3886            0 :                   new_temp = make_ssa_name (vec_dest);
    3887            0 :                   new_stmt = vect_gimple_build (new_temp, convert_code,
    3888              :                                                 prev_res, half_res);
    3889            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    3890              :                 }
    3891              :               else
    3892              :                 {
    3893         3170 :                   if (len_opno >= 0 && len_loop_p)
    3894              :                     {
    3895            0 :                       unsigned int vec_num = vec_oprnds0.length ();
    3896            0 :                       tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
    3897              :                                                     vec_num, vectype_out, i, 1, true);
    3898            0 :                       signed char biasval
    3899            0 :                         = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    3900            0 :                       tree bias = build_int_cst (intQI_type_node, biasval);
    3901            0 :                       vargs[len_opno] = len;
    3902            0 :                       vargs[len_opno + 1] = bias;
    3903              :                     }
    3904         3170 :                   else if (mask_opno >= 0 && masked_loop_p)
    3905              :                     {
    3906           36 :                       unsigned int vec_num = vec_oprnds0.length ();
    3907           36 :                       tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    3908              :                                                       vec_num, vectype_out, i);
    3909           36 :                       vargs[mask_opno]
    3910           72 :                         = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
    3911           36 :                                             vargs[mask_opno], gsi);
    3912              :                     }
    3913              : 
    3914         3170 :                   gcall *call;
    3915         3170 :                   if (ifn != IFN_LAST)
    3916         2861 :                     call = gimple_build_call_internal_vec (ifn, vargs);
    3917              :                   else
    3918          309 :                     call = gimple_build_call_vec (fndecl, vargs);
    3919         3170 :                   new_temp = make_ssa_name (vec_dest, call);
    3920         3170 :                   gimple_call_set_lhs (call, new_temp);
    3921         3170 :                   gimple_call_set_nothrow (call, true);
    3922         3170 :                   vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    3923         3170 :                   new_stmt = call;
    3924              :                 }
    3925         3170 :               slp_node->push_vec_def (new_stmt);
    3926              :             }
    3927              :         }
    3928              : 
    3929        12123 :       for (i = 0; i < nargs; i++)
    3930              :         {
    3931         7560 :           vec<tree> vec_oprndsi = vec_defs[i];
    3932         7560 :           vec_oprndsi.release ();
    3933              :         }
    3934         4563 :     }
    3935           32 :   else if (modifier == NARROW)
    3936              :     {
    3937           32 :       auto_vec<vec<tree> > vec_defs (nargs);
    3938              :       /* We don't define any narrowing conditional functions at present.  */
    3939           32 :       gcc_assert (mask_opno < 0);
    3940              : 
    3941              :       /* Build argument list for the vectorized call.  */
    3942           32 :       vargs.create (nargs * 2);
    3943              : 
    3944           32 :       vect_get_slp_defs (vinfo, slp_node, &vec_defs);
    3945           32 :       vec<tree> vec_oprnds0 = vec_defs[0];
    3946              : 
    3947              :       /* Arguments are ready.  Create the new vector stmt.  */
    3948           64 :       for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
    3949              :         {
    3950           32 :           size_t k;
    3951           32 :           vargs.truncate (0);
    3952           96 :           for (k = 0; k < nargs; k++)
    3953              :             {
    3954           32 :               vec<tree> vec_oprndsk = vec_defs[k];
    3955           32 :               vargs.quick_push (vec_oprndsk[i]);
    3956           32 :               vargs.quick_push (vec_oprndsk[i + 1]);
    3957              :             }
    3958           32 :           gcall *call;
    3959           32 :           if (ifn != IFN_LAST)
    3960              :             call = gimple_build_call_internal_vec (ifn, vargs);
    3961              :           else
    3962           32 :             call = gimple_build_call_vec (fndecl, vargs);
    3963           32 :           new_temp = make_ssa_name (vec_dest, call);
    3964           32 :           gimple_call_set_lhs (call, new_temp);
    3965           32 :           gimple_call_set_nothrow (call, true);
    3966           32 :           vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    3967           32 :           slp_node->push_vec_def (call);
    3968              :         }
    3969              : 
    3970           64 :       for (i = 0; i < nargs; i++)
    3971              :         {
    3972           32 :           vec<tree> vec_oprndsi = vec_defs[i];
    3973           32 :           vec_oprndsi.release ();
    3974              :         }
    3975           32 :     }
    3976              :   else
    3977              :     /* No current target implements this case.  */
    3978              :     return false;
    3979              : 
    3980         4595 :   vargs.release ();
    3981              : 
    3982         4595 :   return true;
    3983      2678571 : }
    3984              : 
    3985              : 
    3986              : struct simd_call_arg_info
    3987              : {
    3988              :   tree vectype;
    3989              :   tree op;
    3990              :   HOST_WIDE_INT linear_step;
    3991              :   enum vect_def_type dt;
    3992              :   unsigned int align;
    3993              :   bool simd_lane_linear;
    3994              : };
    3995              : 
    3996              : /* Helper function of vectorizable_simd_clone_call.  If OP, an SSA_NAME,
    3997              :    is linear within simd lane (but not within whole loop), note it in
    3998              :    *ARGINFO.  */
    3999              : 
    4000              : static void
    4001           19 : vect_simd_lane_linear (tree op, class loop *loop,
    4002              :                        struct simd_call_arg_info *arginfo)
    4003              : {
    4004           19 :   gimple *def_stmt = SSA_NAME_DEF_STMT (op);
    4005              : 
    4006           19 :   if (!is_gimple_assign (def_stmt)
    4007           19 :       || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
    4008           35 :       || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
    4009              :     return;
    4010              : 
    4011           16 :   tree base = gimple_assign_rhs1 (def_stmt);
    4012           16 :   HOST_WIDE_INT linear_step = 0;
    4013           16 :   tree v = gimple_assign_rhs2 (def_stmt);
    4014           64 :   while (TREE_CODE (v) == SSA_NAME)
    4015              :     {
    4016           48 :       tree t;
    4017           48 :       def_stmt = SSA_NAME_DEF_STMT (v);
    4018           48 :       if (is_gimple_assign (def_stmt))
    4019           32 :         switch (gimple_assign_rhs_code (def_stmt))
    4020              :           {
    4021            0 :           case PLUS_EXPR:
    4022            0 :             t = gimple_assign_rhs2 (def_stmt);
    4023            0 :             if (linear_step || TREE_CODE (t) != INTEGER_CST)
    4024              :               return;
    4025            0 :             base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
    4026            0 :             v = gimple_assign_rhs1 (def_stmt);
    4027            0 :             continue;
    4028           16 :           case MULT_EXPR:
    4029           16 :             t = gimple_assign_rhs2 (def_stmt);
    4030           16 :             if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
    4031              :               return;
    4032           16 :             linear_step = tree_to_shwi (t);
    4033           16 :             v = gimple_assign_rhs1 (def_stmt);
    4034           16 :             continue;
    4035           16 :           CASE_CONVERT:
    4036           16 :             t = gimple_assign_rhs1 (def_stmt);
    4037           16 :             if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
    4038           16 :                 || (TYPE_PRECISION (TREE_TYPE (v))
    4039           16 :                     < TYPE_PRECISION (TREE_TYPE (t))))
    4040              :               return;
    4041           16 :             if (!linear_step)
    4042            0 :               linear_step = 1;
    4043           16 :             v = t;
    4044           16 :             continue;
    4045              :           default:
    4046              :             return;
    4047              :           }
    4048           16 :       else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
    4049           16 :                && loop->simduid
    4050           12 :                && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
    4051           28 :                && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
    4052              :                    == loop->simduid))
    4053              :         {
    4054           12 :           if (!linear_step)
    4055            0 :             linear_step = 1;
    4056           12 :           arginfo->linear_step = linear_step;
    4057           12 :           arginfo->op = base;
    4058           12 :           arginfo->simd_lane_linear = true;
    4059              :         }
    4060              :       return;
    4061              :     }
    4062              : }
    4063              : 
    4064              : /* Function vectorizable_simd_clone_call.
    4065              : 
    4066              :    Check if STMT_INFO performs a function call that can be vectorized
    4067              :    by calling a simd clone of the function.
    4068              :    If COST_VEC is passed, calculate costs but don't change anything,
    4069              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    4070              :    it, and insert it at GSI.
    4071              :    Return true if STMT_INFO is vectorizable in this way.  */
    4072              : 
    4073              : static bool
    4074      2666240 : vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
    4075              :                               gimple_stmt_iterator *gsi,
    4076              :                               slp_tree slp_node,
    4077              :                               stmt_vector_for_cost *cost_vec)
    4078              : {
    4079      2666240 :   tree vec_dest;
    4080      2666240 :   tree scalar_dest;
    4081      2666240 :   tree vec_oprnd0 = NULL_TREE;
    4082      2666240 :   tree vectype;
    4083      2666240 :   poly_uint64 nunits;
    4084      2666240 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    4085      2666240 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    4086      2666240 :   class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
    4087      2666240 :   tree fndecl, new_temp;
    4088      2666240 :   int j;
    4089      2666240 :   auto_vec<simd_call_arg_info> arginfo;
    4090      2666240 :   vec<tree> vargs = vNULL;
    4091      2666240 :   size_t i, nargs;
    4092      2666240 :   tree rtype, ratype;
    4093      2666240 :   vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
    4094      2666240 :   int masked_call_offset = 0;
    4095              : 
    4096              :   /* Is STMT a vectorizable call?   */
    4097      2666240 :   gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
    4098        16932 :   if (!stmt)
    4099              :     return false;
    4100              : 
    4101        16932 :   fndecl = gimple_call_fndecl (stmt);
    4102        16932 :   if (fndecl == NULL_TREE
    4103        16932 :       && gimple_call_internal_p (stmt, IFN_MASK_CALL))
    4104              :     {
    4105          240 :       fndecl = gimple_call_arg (stmt, 0);
    4106          240 :       gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
    4107          240 :       fndecl = TREE_OPERAND (fndecl, 0);
    4108          240 :       gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
    4109              :       masked_call_offset = 1;
    4110              :     }
    4111        16692 :   if (fndecl == NULL_TREE)
    4112              :     return false;
    4113              : 
    4114         6570 :   struct cgraph_node *node = cgraph_node::get (fndecl);
    4115         6570 :   if (node == NULL || node->simd_clones == NULL)
    4116              :     return false;
    4117              : 
    4118         1556 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    4119              :     return false;
    4120              : 
    4121         1556 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    4122            0 :       && cost_vec)
    4123              :     return false;
    4124              : 
    4125         1556 :   if (gimple_call_lhs (stmt)
    4126         1556 :       && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
    4127              :     return false;
    4128              : 
    4129         1556 :   gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
    4130              : 
    4131         1556 :   vectype = SLP_TREE_VECTYPE (slp_node);
    4132              : 
    4133      2667796 :   if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
    4134              :     return false;
    4135              : 
    4136              :   /* Process function arguments.  */
    4137         1556 :   nargs = gimple_call_num_args (stmt) - masked_call_offset;
    4138              : 
    4139              :   /* Bail out if the function has zero arguments.  */
    4140         1556 :   if (nargs == 0)
    4141              :     return false;
    4142              : 
    4143         1492 :   vect_simd_clone_data _data;
    4144         1492 :   vect_simd_clone_data &data = slp_node->get_data (_data);
    4145         1492 :   vec<tree>& simd_clone_info = data.simd_clone_info;
    4146         1492 :   arginfo.reserve (nargs, true);
    4147         1492 :   auto_vec<slp_tree> slp_op;
    4148         1492 :   slp_op.safe_grow_cleared (nargs);
    4149              : 
    4150         5791 :   for (i = 0; i < nargs; i++)
    4151              :     {
    4152         2815 :       simd_call_arg_info thisarginfo;
    4153         2815 :       affine_iv iv;
    4154         2815 :       tree op;
    4155              : 
    4156         2815 :       thisarginfo.linear_step = 0;
    4157         2815 :       thisarginfo.align = 0;
    4158         2815 :       thisarginfo.op = NULL_TREE;
    4159         2815 :       thisarginfo.simd_lane_linear = false;
    4160              : 
    4161         5630 :       int op_no = vect_slp_child_index_for_operand (stmt_info,
    4162         2815 :                                                     i + masked_call_offset);
    4163         5630 :       if (!vect_is_simple_use (vinfo, slp_node,
    4164         2815 :                                op_no, &op, &slp_op[i],
    4165              :                                &thisarginfo.dt, &thisarginfo.vectype)
    4166         2815 :           || thisarginfo.dt == vect_uninitialized_def)
    4167              :         {
    4168            0 :           if (dump_enabled_p ())
    4169            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    4170              :                              "use not simple.\n");
    4171            8 :           return false;
    4172              :         }
    4173              : 
    4174         2815 :       if (thisarginfo.dt == vect_constant_def
    4175         2815 :           || thisarginfo.dt == vect_external_def)
    4176              :         {
    4177              :           /* With SLP we determine the vector type of constants/externals
    4178              :              at analysis time, handling conflicts via
    4179              :              vect_maybe_update_slp_op_vectype.  At transform time
    4180              :              we have a vector type recorded for SLP.  */
    4181          690 :           gcc_assert (cost_vec
    4182              :                       || thisarginfo.vectype != NULL_TREE);
    4183              :           if (cost_vec)
    4184              :             {
    4185          558 :               thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
    4186          558 :                                                                  TREE_TYPE (op),
    4187              :                                                                  slp_node);
    4188              :               /* ???  While for example uniform arguments might not need
    4189              :                  a vector type we are not set up to handle this case
    4190              :                  right now.  */
    4191          558 :               if (!thisarginfo.vectype)
    4192              :                 {
    4193            8 :                   if (dump_enabled_p ())
    4194            8 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    4195              :                                      "no vector type for argument.\n");
    4196              :                   return false;
    4197              :                 }
    4198              :             }
    4199              :         }
    4200              :       else
    4201         2125 :         gcc_assert (thisarginfo.vectype != NULL_TREE);
    4202              : 
    4203              :       /* For linear arguments, the analyze phase should have saved
    4204              :          the base and step.  */
    4205         2675 :       if (!cost_vec
    4206         1666 :           && i * 3 + 4 <= simd_clone_info.length ()
    4207         2885 :           && simd_clone_info[i * 3 + 2])
    4208              :         {
    4209          118 :           thisarginfo.linear_step = tree_to_shwi (simd_clone_info[i * 3 + 2]);
    4210          118 :           thisarginfo.op = simd_clone_info[i * 3 + 1];
    4211          118 :           thisarginfo.simd_lane_linear
    4212          118 :             = (simd_clone_info[i * 3 + 3] == boolean_true_node);
    4213              :           /* If loop has been peeled for alignment, we need to adjust it.  */
    4214          118 :           tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
    4215          118 :           tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
    4216          118 :           if (n1 != n2 && !thisarginfo.simd_lane_linear)
    4217              :             {
    4218            0 :               tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
    4219            0 :               tree step = simd_clone_info[i * 3 + 2];
    4220            0 :               tree opt = TREE_TYPE (thisarginfo.op);
    4221            0 :               bias = fold_convert (TREE_TYPE (step), bias);
    4222            0 :               bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
    4223            0 :               thisarginfo.op
    4224            0 :                 = fold_build2 (POINTER_TYPE_P (opt)
    4225              :                                ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
    4226              :                                thisarginfo.op, bias);
    4227              :             }
    4228              :         }
    4229         2689 :       else if (cost_vec
    4230         1974 :                && thisarginfo.dt != vect_constant_def
    4231         1846 :                && thisarginfo.dt != vect_external_def
    4232         1424 :                && loop_vinfo
    4233         1407 :                && SLP_TREE_LANES (slp_node) == 1
    4234         1383 :                && TREE_CODE (op) == SSA_NAME
    4235         2766 :                && simple_iv (loop, loop_containing_stmt (stmt), op,
    4236              :                              &iv, false)
    4237         2905 :                && tree_fits_shwi_p (iv.step))
    4238              :         {
    4239          216 :           thisarginfo.linear_step = tree_to_shwi (iv.step);
    4240          216 :           thisarginfo.op = iv.base;
    4241              :         }
    4242         2473 :       else if ((thisarginfo.dt == vect_constant_def
    4243         2473 :                 || thisarginfo.dt == vect_external_def)
    4244          682 :                && SLP_TREE_LANES (slp_node) == 1
    4245         2781 :                && POINTER_TYPE_P (TREE_TYPE (op)))
    4246           86 :         thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
    4247              :       /* Addresses of array elements indexed by GOMP_SIMD_LANE are
    4248              :          linear too.  */
    4249         2807 :       if (SLP_TREE_LANES (slp_node) == 1
    4250         2363 :           && POINTER_TYPE_P (TREE_TYPE (op))
    4251          200 :           && !thisarginfo.linear_step
    4252          116 :           && cost_vec
    4253           62 :           && thisarginfo.dt != vect_constant_def
    4254           62 :           && thisarginfo.dt != vect_external_def
    4255           19 :           && loop_vinfo
    4256         2826 :           && TREE_CODE (op) == SSA_NAME)
    4257           19 :         vect_simd_lane_linear (op, loop, &thisarginfo);
    4258              : 
    4259         2807 :       if (!vectype)
    4260           12 :         vectype = thisarginfo.vectype;
    4261         2807 :       arginfo.quick_push (thisarginfo);
    4262              :     }
    4263              : 
    4264         1484 :   poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
    4265         1484 :   unsigned group_size = SLP_TREE_LANES (slp_node);
    4266         1484 :   unsigned int badness = 0;
    4267         1484 :   unsigned int badness_inbranch = 0;
    4268         1484 :   struct cgraph_node *bestn = NULL;
    4269         1484 :   struct cgraph_node *bestn_inbranch = NULL;
    4270         1484 :   if (!cost_vec)
    4271          382 :     bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
    4272          382 :              ? data.clone_inbranch : data.clone);
    4273              :   else
    4274         6302 :     for (struct cgraph_node *n = node->simd_clones; n != NULL;
    4275         5200 :          n = n->simdclone->next_clone)
    4276              :       {
    4277         5200 :         unsigned int this_badness = 0;
    4278         5200 :         unsigned int num_calls;
    4279              :         /* The number of arguments in the call and the number of parameters in
    4280              :            the simdclone should match.  However, when the simdclone is
    4281              :            'inbranch', it could have one more parameter than nargs when using
    4282              :            an inbranch simdclone to call a non-inbranch call, either in a
    4283              :            non-masked loop using a all true constant mask, or inside a masked
    4284              :            loop using it's mask.  */
    4285         5200 :         size_t simd_nargs = n->simdclone->nargs;
    4286         5200 :         if (!masked_call_offset && n->simdclone->inbranch)
    4287         2291 :           simd_nargs--;
    4288         5200 :         if (!constant_multiple_p (vf * group_size, n->simdclone->simdlen,
    4289              :                                   &num_calls)
    4290         2071 :             || (!n->simdclone->inbranch && (masked_call_offset > 0))
    4291         1863 :             || (nargs != simd_nargs))
    4292         3337 :           continue;
    4293         1863 :         if (num_calls != 1)
    4294         1276 :           this_badness += floor_log2 (num_calls) * 4096;
    4295         1863 :         if (n->simdclone->inbranch)
    4296          801 :           this_badness += 8192;
    4297              : 
    4298              :         /* If SLP_TREE_VECTYPE has not been set yet pass the general vector
    4299              :            mode,  which for targets that use it will determine what ISA we can
    4300              :            vectorize this code with.  */
    4301         1863 :         machine_mode vector_mode = vinfo->vector_mode;
    4302         1863 :         if (vectype)
    4303         1863 :           vector_mode = TYPE_MODE (vectype);
    4304         1863 :         int target_badness = targetm.simd_clone.usable (n, vector_mode);
    4305         1863 :         if (target_badness < 0)
    4306          390 :           continue;
    4307         1473 :         this_badness += target_badness * 512;
    4308         4376 :         for (i = 0; i < nargs; i++)
    4309              :           {
    4310         3178 :             switch (n->simdclone->args[i].arg_type)
    4311              :               {
    4312         2216 :               case SIMD_CLONE_ARG_TYPE_VECTOR:
    4313         2216 :                 if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type))
    4314              :                   /* Vector mask arguments are not supported.  */
    4315              :                   i = -1;
    4316         2208 :                 else if (!useless_type_conversion_p
    4317         2208 :                          (n->simdclone->args[i].orig_type,
    4318         2208 :                           TREE_TYPE (gimple_call_arg (stmt,
    4319              :                                                       i + masked_call_offset))))
    4320              :                   i = -1;
    4321         2208 :                 else if (arginfo[i].dt == vect_constant_def
    4322         2100 :                          || arginfo[i].dt == vect_external_def
    4323         4244 :                          || arginfo[i].linear_step)
    4324          400 :                   this_badness += 64;
    4325              :                 break;
    4326          310 :               case SIMD_CLONE_ARG_TYPE_UNIFORM:
    4327          310 :                 if ((arginfo[i].dt != vect_constant_def
    4328          145 :                      && arginfo[i].dt != vect_external_def)
    4329          410 :                     || SLP_TREE_LANES (slp_node) != 1)
    4330              :                   i = -1;
    4331              :                 break;
    4332          328 :               case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
    4333          328 :               case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
    4334          328 :                 if (arginfo[i].dt == vect_constant_def
    4335          328 :                     || arginfo[i].dt == vect_external_def
    4336          328 :                     || (arginfo[i].linear_step
    4337          328 :                         != n->simdclone->args[i].linear_step))
    4338              :                   i = -1;
    4339              :                 break;
    4340              :               case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
    4341              :               case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
    4342              :               case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
    4343              :               case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
    4344              :               case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
    4345              :               case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
    4346              :                 /* FORNOW */
    4347              :                 i = -1;
    4348              :                 break;
    4349          324 :               case SIMD_CLONE_ARG_TYPE_MASK:
    4350          324 :                 if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
    4351          285 :                     && n->simdclone->mask_mode != VOIDmode)
    4352              :                   i = -1;
    4353              :                 /* While we can create a traditional data vector from
    4354              :                    an incoming integer mode mask we have no good way to
    4355              :                    force generate an integer mode mask from a traditional
    4356              :                    boolean vector input.  */
    4357          324 :                 else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
    4358          324 :                          && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
    4359              :                   i = -1;
    4360          318 :                 else if (n->simdclone->mask_mode == VOIDmode
    4361              :                          /* FORNOW we only have partial support for vector-type
    4362              :                             masks that can't hold all of simdlen. */
    4363          603 :                          && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type),
    4364          467 :                                        TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
    4365              :                              /* Verify we can compute the mask argument.  */
    4366          115 :                              || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type,
    4367          115 :                                                          arginfo[i].vectype)))
    4368              :                   i = -1;
    4369          136 :                 else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
    4370              :                          /* FORNOW we only have partial support for
    4371              :                             integer-type masks that represent the same number
    4372              :                             of lanes as the vectorized mask inputs.  */
    4373          169 :                          && maybe_ne (exact_div (n->simdclone->simdlen,
    4374              :                                                  n->simdclone->args[i].linear_step),
    4375           33 :                                       TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
    4376              :                   i = -1;
    4377          112 :                 else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
    4378          112 :                          && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
    4379           12 :                   this_badness += 2048;
    4380              :                 break;
    4381              :               }
    4382           24 :             if (i == (size_t) -1)
    4383              :               break;
    4384         2903 :             if (n->simdclone->args[i].alignment > arginfo[i].align)
    4385              :               {
    4386              :                 i = -1;
    4387              :                 break;
    4388              :               }
    4389         2903 :             if (arginfo[i].align)
    4390          110 :               this_badness += (exact_log2 (arginfo[i].align)
    4391          160 :                                - exact_log2 (n->simdclone->args[i].alignment));
    4392              :           }
    4393         1473 :         if (i == (size_t) -1)
    4394          275 :           continue;
    4395         1198 :         if (masked_call_offset == 0
    4396         1086 :             && n->simdclone->inbranch
    4397          351 :             && n->simdclone->nargs > nargs)
    4398              :           {
    4399          351 :             gcc_assert (n->simdclone->args[n->simdclone->nargs - 1].arg_type ==
    4400              :                         SIMD_CLONE_ARG_TYPE_MASK);
    4401          351 :             if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
    4402          335 :                 && n->simdclone->mask_mode != VOIDmode)
    4403            0 :               continue;
    4404              :             /* Penalize using a masked SIMD clone in a non-masked loop, that is
    4405              :                not in a branch, as we'd have to construct an all-true mask.  */
    4406          351 :             this_badness += 64;
    4407              :           }
    4408         1198 :         if (bestn == NULL || this_badness < badness)
    4409              :           {
    4410          861 :             bestn = n;
    4411          861 :             badness = this_badness;
    4412              :           }
    4413         1198 :         if (n->simdclone->inbranch
    4414          463 :             && (bestn_inbranch == NULL || this_badness < badness_inbranch))
    4415              :           {
    4416         5200 :             bestn_inbranch = n;
    4417         5200 :             badness_inbranch = this_badness;
    4418              :           }
    4419              :       }
    4420              : 
    4421         1484 :   if (bestn == NULL)
    4422              :     return false;
    4423              : 
    4424          883 :   fndecl = bestn->decl;
    4425          883 :   nunits = bestn->simdclone->simdlen;
    4426          883 :   int ncopies = vector_unroll_factor (vf * group_size, nunits);
    4427              : 
    4428              :   /* If the function isn't const, only allow it in simd loops where user
    4429              :      has asserted that at least nunits consecutive iterations can be
    4430              :      performed using SIMD instructions.  */
    4431          866 :   if ((loop == NULL || maybe_lt ((unsigned) loop->safelen, nunits))
    4432         1072 :       && gimple_vuse (stmt))
    4433              :     return false;
    4434              : 
    4435              :   /* ncopies is the number of SIMD clone calls we create, since simdlen
    4436              :      is not necessarily matching nunits of the vector types used, track
    4437              :      that in ncopies_in.  */
    4438          883 :   int ncopies_in = vect_get_num_vectors (vf * group_size, vectype);
    4439              : 
    4440              :   /* Sanity check: make sure that at least one copy of the vectorized stmt
    4441              :      needs to be generated.  */
    4442          883 :   gcc_assert (ncopies >= 1);
    4443              : 
    4444          883 :   if (cost_vec) /* transformation not required.  */
    4445              :     {
    4446         1642 :       for (unsigned i = 0; i < nargs; ++i)
    4447         1141 :         if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
    4448              :           {
    4449            0 :             if (dump_enabled_p ())
    4450            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    4451              :                                "incompatible vector types for invariants\n");
    4452              :             return false;
    4453              :           }
    4454              : 
    4455          501 :       if (!bestn_inbranch && loop_vinfo)
    4456              :         {
    4457          271 :           if (dump_enabled_p ()
    4458          271 :               && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
    4459          193 :             dump_printf_loc (MSG_NOTE, vect_location,
    4460              :                              "can't use a fully-masked loop because no"
    4461              :                              " masked simd clone was available.\n");
    4462          271 :           LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    4463              :         }
    4464              : 
    4465              :       /* When the original call is pure or const but the SIMD ABI dictates
    4466              :          an aggregate return we will have to use a virtual definition and
    4467              :          in a loop eventually even need to add a virtual PHI.  That's
    4468              :          not straight-forward so allow to fix this up via renaming.  */
    4469          501 :       if (gimple_call_lhs (stmt)
    4470          495 :           && !gimple_vdef (stmt)
    4471          888 :           && TREE_CODE (TREE_TYPE (TREE_TYPE (bestn->decl))) == ARRAY_TYPE)
    4472           36 :         vinfo->any_known_not_updated_vssa = true;
    4473              :       /* ???  For SLP code-gen we end up inserting after the last
    4474              :          vector argument def rather than at the original call position
    4475              :          so automagic virtual operand updating doesn't work.  */
    4476         1002 :       if (gimple_vuse (stmt))
    4477          151 :         vinfo->any_known_not_updated_vssa = true;
    4478              : 
    4479          501 :       data.clone = bestn;
    4480          501 :       data.clone_inbranch = bestn_inbranch;
    4481              : 
    4482          501 :       simd_clone_info.safe_push (NULL_TREE);
    4483         1799 :       for (i = 0;
    4484         2729 :            i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++)
    4485              :         {
    4486         1298 :           if (loop_vinfo
    4487         1274 :               && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    4488          480 :               && (bestn_inbranch->simdclone->args[i].arg_type
    4489              :                   == SIMD_CLONE_ARG_TYPE_MASK))
    4490              :             {
    4491          173 :               if (masked_call_offset)
    4492              :                 /* When there is an explicit mask we require the
    4493              :                    number of elements to match up.  */
    4494           52 :                 vect_record_loop_mask (loop_vinfo,
    4495              :                                        &LOOP_VINFO_MASKS (loop_vinfo),
    4496              :                                        ncopies_in, vectype, NULL_TREE);
    4497              :               else
    4498              :                 {
    4499              :                   /* When there is no explicit mask on the call we have
    4500              :                      more relaxed requirements.  */
    4501          121 :                   tree masktype;
    4502          121 :                   poly_uint64 callee_nelements;
    4503          121 :                   if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode))
    4504              :                     {
    4505           16 :                       callee_nelements
    4506           16 :                         = exact_div (bestn_inbranch->simdclone->simdlen,
    4507              :                                      bestn_inbranch->simdclone->args[i].linear_step);
    4508           16 :                       masktype = get_related_vectype_for_scalar_type
    4509           16 :                           (vinfo->vector_mode, TREE_TYPE (vectype),
    4510              :                            callee_nelements);
    4511              :                     }
    4512              :                   else
    4513              :                     {
    4514          105 :                       masktype = bestn_inbranch->simdclone->args[i].vector_type;
    4515          105 :                       if (VECTOR_BOOLEAN_TYPE_P (masktype))
    4516              :                         {
    4517            0 :                           if (VECTOR_MODE_P (TYPE_MODE (masktype)))
    4518              :                             /* The aarch64 port will add custom attributes to
    4519              :                                types for SVE simdclones which make the types
    4520              :                                different.  We should use canonincal types for
    4521              :                                masks within the vectorizer, hence we
    4522              :                                re-construct the vectype from its mode.  */
    4523            0 :                             masktype = build_truth_vector_type_for_mode
    4524            0 :                                          (TYPE_VECTOR_SUBPARTS (masktype),
    4525            0 :                                           TYPE_MODE (masktype));
    4526              :                         }
    4527              :                       else
    4528          105 :                         masktype = truth_type_for (masktype);
    4529          105 :                       callee_nelements = TYPE_VECTOR_SUBPARTS (masktype);
    4530              :                     }
    4531          121 :                   if (!masktype)
    4532              :                     {
    4533            0 :                       if (dump_enabled_p ())
    4534            0 :                         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    4535              :                                          "can't use a fully-masked loop because"
    4536              :                                          " no suitable vector type was found for"
    4537              :                                          " the mask.\n");
    4538            0 :                       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    4539              :                     }
    4540              :                   else
    4541              :                     {
    4542          121 :                       auto o = vector_unroll_factor (nunits, callee_nelements);
    4543          121 :                       vect_record_loop_mask (loop_vinfo,
    4544              :                                              &LOOP_VINFO_MASKS (loop_vinfo),
    4545              :                                              ncopies * o, masktype, NULL_TREE);
    4546              :                     }
    4547              :                 }
    4548              :             }
    4549         1125 :           else if ((bestn->simdclone->args[i].arg_type
    4550              :                     == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
    4551         1018 :                    || (bestn->simdclone->args[i].arg_type
    4552              :                        == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
    4553         1007 :                    || (bestn_inbranch
    4554          399 :                        && ((bestn_inbranch->simdclone->args[i].arg_type
    4555              :                             == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
    4556          399 :                            || (bestn_inbranch->simdclone->args[i].arg_type
    4557              :                                == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))))
    4558              :             {
    4559          118 :               simd_clone_info.safe_grow_cleared (i * 3 + 1, true);
    4560          118 :               simd_clone_info.safe_push (arginfo[i].op);
    4561          202 :               tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
    4562          202 :                           ? size_type_node : TREE_TYPE (arginfo[i].op));
    4563          118 :               tree ls = build_int_cst (lst, arginfo[i].linear_step);
    4564          118 :               simd_clone_info.safe_push (ls);
    4565          118 :               tree sll = (arginfo[i].simd_lane_linear
    4566          118 :                           ? boolean_true_node : boolean_false_node);
    4567          118 :               simd_clone_info.safe_push (sll);
    4568              :             }
    4569              :         }
    4570              : 
    4571          501 :       SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
    4572          501 :       slp_node->data = new vect_simd_clone_data (std::move (_data));
    4573          501 :       DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
    4574              :       /* ???  We're confused by calls w/o LHS.  */
    4575          501 :       if (SLP_TREE_VECTYPE (slp_node))
    4576          495 :         vect_model_simple_cost (vinfo, ncopies, slp_node, cost_vec);
    4577          501 :       return true;
    4578              :     }
    4579              : 
    4580              :   /* Transform.  */
    4581              : 
    4582          382 :   if (dump_enabled_p ())
    4583          259 :     dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
    4584              : 
    4585              :   /* Handle def.  */
    4586          382 :   scalar_dest = gimple_call_lhs (stmt);
    4587          382 :   vec_dest = NULL_TREE;
    4588          382 :   rtype = NULL_TREE;
    4589          382 :   ratype = NULL_TREE;
    4590          382 :   if (scalar_dest)
    4591              :     {
    4592          376 :       vec_dest = vect_create_destination_var (scalar_dest, vectype);
    4593          376 :       rtype = TREE_TYPE (TREE_TYPE (fndecl));
    4594          376 :       if (TREE_CODE (rtype) == ARRAY_TYPE)
    4595              :         {
    4596           12 :           ratype = rtype;
    4597           12 :           rtype = TREE_TYPE (ratype);
    4598              :         }
    4599              :     }
    4600              : 
    4601          764 :   auto_vec<vec<tree> > vec_oprnds;
    4602          382 :   auto_vec<unsigned> vec_oprnds_i;
    4603          382 :   vec_oprnds_i.safe_grow_cleared (nargs, true);
    4604          382 :   vec_oprnds.reserve_exact (nargs);
    4605          382 :   vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
    4606         1284 :   for (j = 0; j < ncopies; ++j)
    4607              :     {
    4608          520 :       poly_uint64 callee_nelements;
    4609          520 :       poly_uint64 caller_nelements;
    4610              :       /* Build argument list for the vectorized call.  */
    4611          520 :       if (j == 0)
    4612          382 :         vargs.create (nargs);
    4613              :       else
    4614          138 :         vargs.truncate (0);
    4615              : 
    4616         1697 :       for (i = 0; i < nargs; i++)
    4617              :         {
    4618         1177 :           unsigned int k, l, m, o;
    4619         1177 :           tree atype;
    4620         1177 :           tree op = gimple_call_arg (stmt, i + masked_call_offset);
    4621         1177 :           switch (bestn->simdclone->args[i].arg_type)
    4622              :             {
    4623          883 :             case SIMD_CLONE_ARG_TYPE_VECTOR:
    4624          883 :               atype = bestn->simdclone->args[i].vector_type;
    4625          883 :               caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
    4626          883 :               callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
    4627          883 :               o = vector_unroll_factor (nunits, callee_nelements);
    4628         2001 :               for (m = j * o; m < (j + 1) * o; m++)
    4629              :                 {
    4630         1118 :                   if (known_lt (callee_nelements, caller_nelements))
    4631              :                     {
    4632          596 :                       poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
    4633          298 :                       if (!constant_multiple_p (caller_nelements,
    4634              :                                                 callee_nelements, &k))
    4635            0 :                         gcc_unreachable ();
    4636              : 
    4637          298 :                       gcc_assert ((k & (k - 1)) == 0);
    4638          298 :                       if (m == 0)
    4639              :                         {
    4640           65 :                           vec_oprnds_i[i] = 0;
    4641           65 :                           vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
    4642              :                         }
    4643              :                       else
    4644              :                         {
    4645          233 :                           vec_oprnd0 = arginfo[i].op;
    4646          233 :                           if ((m & (k - 1)) == 0)
    4647           78 :                             vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
    4648              :                         }
    4649          298 :                       arginfo[i].op = vec_oprnd0;
    4650          298 :                       vec_oprnd0
    4651          298 :                         = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
    4652          298 :                                   bitsize_int (prec),
    4653          298 :                                   bitsize_int ((m & (k - 1)) * prec));
    4654          298 :                       gassign *new_stmt
    4655          298 :                         = gimple_build_assign (make_ssa_name (atype),
    4656              :                                                vec_oprnd0);
    4657          298 :                       vect_finish_stmt_generation (vinfo, stmt_info,
    4658              :                                                    new_stmt, gsi);
    4659          298 :                       vargs.safe_push (gimple_assign_lhs (new_stmt));
    4660              :                     }
    4661              :                   else
    4662              :                     {
    4663          820 :                       if (!constant_multiple_p (callee_nelements,
    4664              :                                                 caller_nelements, &k))
    4665            0 :                         gcc_unreachable ();
    4666          820 :                       gcc_assert ((k & (k - 1)) == 0);
    4667          820 :                       vec<constructor_elt, va_gc> *ctor_elts;
    4668          820 :                       if (k != 1)
    4669           12 :                         vec_alloc (ctor_elts, k);
    4670              :                       else
    4671          808 :                         ctor_elts = NULL;
    4672          844 :                       for (l = 0; l < k; l++)
    4673              :                         {
    4674          832 :                           if (m == 0 && l == 0)
    4675              :                             {
    4676          479 :                               vec_oprnds_i[i] = 0;
    4677          479 :                               vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
    4678              :                             }
    4679              :                           else
    4680          353 :                             vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
    4681          832 :                           arginfo[i].op = vec_oprnd0;
    4682          832 :                           if (k == 1)
    4683              :                             break;
    4684           24 :                           CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
    4685              :                                                   vec_oprnd0);
    4686              :                         }
    4687          820 :                       if (k == 1)
    4688          808 :                         if (!useless_type_conversion_p (TREE_TYPE (vec_oprnd0),
    4689              :                                                        atype))
    4690              :                           {
    4691            0 :                             vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, atype,
    4692              :                                                  vec_oprnd0);
    4693            0 :                             gassign *new_stmt
    4694            0 :                               = gimple_build_assign (make_ssa_name (atype),
    4695              :                                                      vec_oprnd0);
    4696            0 :                             vect_finish_stmt_generation (vinfo, stmt_info,
    4697              :                                                          new_stmt, gsi);
    4698            0 :                             vargs.safe_push (gimple_get_lhs (new_stmt));
    4699              :                           }
    4700              :                         else
    4701          808 :                           vargs.safe_push (vec_oprnd0);
    4702              :                       else
    4703              :                         {
    4704           12 :                           vec_oprnd0 = build_constructor (atype, ctor_elts);
    4705           12 :                           gassign *new_stmt
    4706           12 :                             = gimple_build_assign (make_ssa_name (atype),
    4707              :                                                    vec_oprnd0);
    4708           12 :                           vect_finish_stmt_generation (vinfo, stmt_info,
    4709              :                                                        new_stmt, gsi);
    4710           12 :                           vargs.safe_push (gimple_assign_lhs (new_stmt));
    4711              :                         }
    4712              :                     }
    4713              :                 }
    4714              :               break;
    4715           71 :             case SIMD_CLONE_ARG_TYPE_MASK:
    4716           71 :               if (bestn->simdclone->mask_mode == VOIDmode)
    4717              :                 {
    4718           64 :                   atype = bestn->simdclone->args[i].vector_type;
    4719           64 :                   tree elt_type = TREE_TYPE (atype);
    4720           64 :                   tree one = fold_convert (elt_type, integer_one_node);
    4721           64 :                   tree zero = fold_convert (elt_type, integer_zero_node);
    4722           64 :                   callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
    4723           64 :                   caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
    4724           64 :                   o = vector_unroll_factor (nunits, callee_nelements);
    4725          132 :                   for (m = j * o; m < (j + 1) * o; m++)
    4726              :                     {
    4727           68 :                       if (maybe_lt (callee_nelements, caller_nelements))
    4728              :                         {
    4729              :                           /* The mask type has fewer elements than simdlen.  */
    4730              : 
    4731              :                           /* FORNOW */
    4732            0 :                           gcc_unreachable ();
    4733              :                         }
    4734           68 :                       else if (known_eq (callee_nelements, caller_nelements))
    4735              :                         {
    4736              :                           /* The SIMD clone function has the same number of
    4737              :                              elements as the current function.  */
    4738           68 :                           if (m == 0)
    4739           62 :                             vec_oprnds_i[i] = 0;
    4740           68 :                           vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
    4741           68 :                           if (loop_vinfo
    4742           68 :                               && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
    4743              :                             {
    4744            0 :                               vec_loop_masks *loop_masks
    4745              :                                 = &LOOP_VINFO_MASKS (loop_vinfo);
    4746            0 :                               tree loop_mask
    4747            0 :                                 = vect_get_loop_mask (loop_vinfo, gsi,
    4748              :                                                       loop_masks, ncopies_in,
    4749            0 :                                                       vectype, m);
    4750            0 :                               vec_oprnd0
    4751            0 :                                 = prepare_vec_mask (loop_vinfo,
    4752            0 :                                                     TREE_TYPE (loop_mask),
    4753              :                                                     loop_mask, vec_oprnd0,
    4754              :                                                     gsi);
    4755            0 :                               loop_vinfo->vec_cond_masked_set.add ({ vec_oprnd0,
    4756              :                                                                      loop_mask });
    4757              : 
    4758              :                             }
    4759           68 :                           vec_oprnd0
    4760           68 :                             = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
    4761              :                                       build_vector_from_val (atype, one),
    4762              :                                       build_vector_from_val (atype, zero));
    4763           68 :                           gassign *new_stmt
    4764           68 :                             = gimple_build_assign (make_ssa_name (atype),
    4765              :                                                    vec_oprnd0);
    4766           68 :                           vect_finish_stmt_generation (vinfo, stmt_info,
    4767              :                                                        new_stmt, gsi);
    4768           68 :                           vargs.safe_push (gimple_assign_lhs (new_stmt));
    4769              :                         }
    4770              :                       else
    4771              :                         {
    4772              :                           /* The mask type has more elements than simdlen.  */
    4773              : 
    4774              :                           /* FORNOW */
    4775            0 :                           gcc_unreachable ();
    4776              :                         }
    4777              :                     }
    4778              :                 }
    4779            7 :               else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
    4780              :                 {
    4781            7 :                   atype = bestn->simdclone->args[i].vector_type;
    4782            7 :                   poly_uint64 atype_subparts
    4783            7 :                     = exact_div (bestn->simdclone->simdlen,
    4784              :                                  bestn->simdclone->args[i].linear_step);
    4785            7 :                   o = bestn->simdclone->args[i].linear_step;
    4786           15 :                   for (m = j * o; m < (j + 1) * o; m++)
    4787              :                     {
    4788            8 :                       if (m == 0)
    4789            7 :                         vec_oprnds_i[i] = 0;
    4790            8 :                       if (maybe_lt (atype_subparts,
    4791            8 :                                     TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
    4792              :                         {
    4793              :                           /* The mask argument has fewer elements than the
    4794              :                              input vector.  */
    4795              :                           /* FORNOW */
    4796            0 :                           gcc_unreachable ();
    4797              :                         }
    4798            8 :                       else if (known_eq (atype_subparts,
    4799              :                                          TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
    4800              :                         {
    4801            8 :                           vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
    4802            8 :                           if (loop_vinfo
    4803            8 :                               && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
    4804              :                             {
    4805            3 :                               vec_loop_masks *loop_masks
    4806              :                                 = &LOOP_VINFO_MASKS (loop_vinfo);
    4807            3 :                               tree loop_mask
    4808            3 :                                 = vect_get_loop_mask (loop_vinfo, gsi,
    4809              :                                                       loop_masks, ncopies_in,
    4810              :                                                       vectype, m);
    4811            3 :                               vec_oprnd0
    4812            3 :                                 = prepare_vec_mask (loop_vinfo,
    4813            3 :                                                     TREE_TYPE (loop_mask),
    4814              :                                                     loop_mask, vec_oprnd0,
    4815              :                                                     gsi);
    4816              :                             }
    4817              :                           /* The vector mask argument matches the input
    4818              :                              in the number of lanes, but not necessarily
    4819              :                              in the mode.  */
    4820            8 :                           tree st = lang_hooks.types.type_for_mode
    4821            8 :                                       (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1);
    4822            8 :                           vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st,
    4823              :                                                vec_oprnd0);
    4824            8 :                           gassign *new_stmt
    4825            8 :                             = gimple_build_assign (make_ssa_name (st),
    4826              :                                                    vec_oprnd0);
    4827            8 :                           vect_finish_stmt_generation (vinfo, stmt_info,
    4828              :                                                        new_stmt, gsi);
    4829            8 :                           if (!types_compatible_p (atype, st))
    4830              :                             {
    4831            8 :                               new_stmt
    4832            8 :                                 = gimple_build_assign (make_ssa_name (atype),
    4833              :                                                        NOP_EXPR,
    4834              :                                                        gimple_assign_lhs
    4835              :                                                          (new_stmt));
    4836            8 :                               vect_finish_stmt_generation (vinfo, stmt_info,
    4837              :                                                            new_stmt, gsi);
    4838              :                             }
    4839            8 :                           vargs.safe_push (gimple_assign_lhs (new_stmt));
    4840              :                         }
    4841              :                       else
    4842              :                         {
    4843              :                           /* The mask argument has more elements than the
    4844              :                              input vector.  */
    4845              :                           /* FORNOW */
    4846            0 :                           gcc_unreachable ();
    4847              :                         }
    4848              :                     }
    4849              :                 }
    4850              :               else
    4851            0 :                 gcc_unreachable ();
    4852              :               break;
    4853          102 :             case SIMD_CLONE_ARG_TYPE_UNIFORM:
    4854          102 :               vargs.safe_push (op);
    4855          102 :               break;
    4856          121 :             case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
    4857          121 :             case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
    4858          121 :               if (j == 0)
    4859              :                 {
    4860          118 :                   gimple_seq stmts;
    4861          118 :                   arginfo[i].op
    4862          118 :                     = force_gimple_operand (unshare_expr (arginfo[i].op),
    4863              :                                             &stmts, true, NULL_TREE);
    4864          118 :                   if (stmts != NULL)
    4865              :                     {
    4866            0 :                       basic_block new_bb;
    4867            0 :                       edge pe = loop_preheader_edge (loop);
    4868            0 :                       new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
    4869            0 :                       gcc_assert (!new_bb);
    4870              :                     }
    4871          118 :                   if (arginfo[i].simd_lane_linear)
    4872              :                     {
    4873            6 :                       vargs.safe_push (arginfo[i].op);
    4874            6 :                       break;
    4875              :                     }
    4876          112 :                   tree phi_res = copy_ssa_name (op);
    4877          112 :                   gphi *new_phi = create_phi_node (phi_res, loop->header);
    4878          112 :                   add_phi_arg (new_phi, arginfo[i].op,
    4879              :                                loop_preheader_edge (loop), UNKNOWN_LOCATION);
    4880          112 :                   enum tree_code code
    4881          196 :                     = POINTER_TYPE_P (TREE_TYPE (op))
    4882          112 :                       ? POINTER_PLUS_EXPR : PLUS_EXPR;
    4883          196 :                   tree type = POINTER_TYPE_P (TREE_TYPE (op))
    4884          196 :                               ? sizetype : TREE_TYPE (op);
    4885          112 :                   poly_widest_int cst
    4886          112 :                     = wi::mul (bestn->simdclone->args[i].linear_step,
    4887          112 :                                ncopies * nunits);
    4888          112 :                   tree tcst = wide_int_to_tree (type, cst);
    4889          112 :                   tree phi_arg = copy_ssa_name (op);
    4890          112 :                   gassign *new_stmt
    4891          112 :                     = gimple_build_assign (phi_arg, code, phi_res, tcst);
    4892          112 :                   gimple_stmt_iterator si = gsi_after_labels (loop->header);
    4893          112 :                   gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
    4894          112 :                   add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
    4895              :                                UNKNOWN_LOCATION);
    4896          112 :                   arginfo[i].op = phi_res;
    4897          112 :                   vargs.safe_push (phi_res);
    4898          112 :                 }
    4899              :               else
    4900              :                 {
    4901            3 :                   enum tree_code code
    4902            6 :                     = POINTER_TYPE_P (TREE_TYPE (op))
    4903            3 :                       ? POINTER_PLUS_EXPR : PLUS_EXPR;
    4904            6 :                   tree type = POINTER_TYPE_P (TREE_TYPE (op))
    4905            6 :                               ? sizetype : TREE_TYPE (op);
    4906            3 :                   poly_widest_int cst
    4907            3 :                     = wi::mul (bestn->simdclone->args[i].linear_step,
    4908            3 :                                j * nunits);
    4909            3 :                   tree tcst = wide_int_to_tree (type, cst);
    4910            3 :                   new_temp = make_ssa_name (TREE_TYPE (op));
    4911            3 :                   gassign *new_stmt
    4912            6 :                     = gimple_build_assign (new_temp, code,
    4913            3 :                                            arginfo[i].op, tcst);
    4914            3 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    4915            3 :                   vargs.safe_push (new_temp);
    4916            3 :                 }
    4917              :               break;
    4918            0 :             case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
    4919            0 :             case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
    4920            0 :             case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
    4921            0 :             case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
    4922            0 :             case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
    4923            0 :             case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
    4924            0 :             default:
    4925            0 :               gcc_unreachable ();
    4926              :             }
    4927              :         }
    4928              : 
    4929          520 :       if (masked_call_offset == 0
    4930          449 :           && bestn->simdclone->inbranch
    4931           37 :           && bestn->simdclone->nargs > nargs)
    4932              :         {
    4933           37 :           unsigned long m, o;
    4934           37 :           size_t mask_i = bestn->simdclone->nargs - 1;
    4935           37 :           tree mask;
    4936           37 :           gcc_assert (bestn->simdclone->args[mask_i].arg_type ==
    4937              :                       SIMD_CLONE_ARG_TYPE_MASK);
    4938              : 
    4939           37 :           tree mask_argtype = bestn->simdclone->args[mask_i].vector_type;
    4940           37 :           tree mask_vectype;
    4941           37 :           if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
    4942              :             {
    4943            2 :               callee_nelements = exact_div (bestn->simdclone->simdlen,
    4944              :                                             bestn->simdclone->args[i].linear_step);
    4945            2 :               mask_vectype = get_related_vectype_for_scalar_type
    4946            2 :                   (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements);
    4947              :             }
    4948              :           else
    4949              :             {
    4950           35 :               mask_vectype = mask_argtype;
    4951           35 :               callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype);
    4952              :             }
    4953           37 :           o = vector_unroll_factor (nunits, callee_nelements);
    4954           74 :           for (m = j * o; m < (j + 1) * o; m++)
    4955              :             {
    4956           37 :               if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
    4957              :                 {
    4958            1 :                   vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
    4959            1 :                   mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
    4960              :                                              ncopies * o, mask_vectype, m);
    4961              :                 }
    4962           36 :               else if (bestn->simdclone->mask_mode == VOIDmode)
    4963              :                 {
    4964              :                   /* All lanes are active: pass what the VEC_COND_EXPR below
    4965              :                      would select.  An all-ones floating-point MASK_ARGTYPE
    4966              :                      is not a valid condition.  */
    4967           35 :                   vargs.safe_push (build_one_cst (mask_argtype));
    4968           35 :                   continue;
    4969              :                 }
    4970              :               else
    4971            1 :                 mask = vect_build_all_ones_mask (vinfo, stmt_info,
    4972              :                                                  mask_argtype);
    4973              : 
    4974            2 :               gassign *new_stmt;
    4975            2 :               if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
    4976              :                 {
    4977              :                   /* This means we are dealing with integer mask modes.
    4978              :                      First convert to an integer type with the same size as
    4979              :                      the current vector type.  */
    4980            2 :                   unsigned HOST_WIDE_INT intermediate_size
    4981            2 :                       = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (mask)));
    4982            2 :                   tree mid_int_type =
    4983            2 :                       build_nonstandard_integer_type (intermediate_size, 1);
    4984            2 :                   mask = build1 (VIEW_CONVERT_EXPR, mid_int_type, mask);
    4985            2 :                   new_stmt
    4986            2 :                       = gimple_build_assign (make_ssa_name (mid_int_type),
    4987              :                                              mask);
    4988            2 :                   gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    4989              :                   /* Then zero-extend to the mask mode.  */
    4990            2 :                   mask = fold_build1 (NOP_EXPR, mask_argtype,
    4991              :                                       gimple_get_lhs (new_stmt));
    4992              :                 }
    4993            0 :               else if (bestn->simdclone->mask_mode == VOIDmode)
    4994            0 :                 mask = build3 (VEC_COND_EXPR, mask_argtype, mask,
    4995              :                                build_one_cst (mask_argtype),
    4996              :                                build_zero_cst (mask_argtype));
    4997              :               else
    4998            0 :                 gcc_unreachable ();
    4999              : 
    5000            2 :               new_stmt = gimple_build_assign (make_ssa_name (mask_argtype),
    5001              :                                               mask);
    5002            2 :               vect_finish_stmt_generation (vinfo, stmt_info,
    5003              :                                            new_stmt, gsi);
    5004            2 :               mask = gimple_assign_lhs (new_stmt);
    5005            2 :               vargs.safe_push (mask);
    5006              :             }
    5007              :         }
    5008              : 
    5009          520 :       gcall *new_call = gimple_build_call_vec (fndecl, vargs);
    5010          520 :       if (vec_dest)
    5011              :         {
    5012          514 :           gcc_assert (ratype
    5013              :                       || known_eq (TYPE_VECTOR_SUBPARTS (rtype), nunits));
    5014          514 :           if (ratype)
    5015           20 :             new_temp = create_tmp_var (ratype);
    5016          494 :           else if (useless_type_conversion_p (vectype, rtype))
    5017          438 :             new_temp = make_ssa_name (vec_dest, new_call);
    5018              :           else
    5019           56 :             new_temp = make_ssa_name (rtype, new_call);
    5020          514 :           gimple_call_set_lhs (new_call, new_temp);
    5021              :         }
    5022          520 :       vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
    5023          520 :       gimple *new_stmt = new_call;
    5024              : 
    5025          520 :       if (vec_dest)
    5026              :         {
    5027          514 :           if (!multiple_p (TYPE_VECTOR_SUBPARTS (vectype), nunits))
    5028              :             {
    5029           20 :               unsigned int k, l;
    5030           40 :               poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
    5031           40 :               poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
    5032           20 :               k = vector_unroll_factor (nunits,
    5033              :                                         TYPE_VECTOR_SUBPARTS (vectype));
    5034           20 :               gcc_assert ((k & (k - 1)) == 0);
    5035           72 :               for (l = 0; l < k; l++)
    5036              :                 {
    5037           52 :                   tree t;
    5038           52 :                   if (ratype)
    5039              :                     {
    5040           44 :                       t = build_fold_addr_expr (new_temp);
    5041           44 :                       t = build2 (MEM_REF, vectype, t,
    5042           44 :                                   build_int_cst (TREE_TYPE (t), l * bytes));
    5043              :                     }
    5044              :                   else
    5045            8 :                     t = build3 (BIT_FIELD_REF, vectype, new_temp,
    5046            8 :                                 bitsize_int (prec), bitsize_int (l * prec));
    5047           52 :                   new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
    5048           52 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5049              : 
    5050           52 :                   SLP_TREE_VEC_DEFS (slp_node)
    5051           52 :                     .quick_push (gimple_assign_lhs (new_stmt));
    5052              :                 }
    5053              : 
    5054           20 :               if (ratype)
    5055           16 :                 vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
    5056           20 :               continue;
    5057           20 :             }
    5058          494 :           else if (!multiple_p (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
    5059              :             {
    5060           56 :               unsigned int k;
    5061           56 :               if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
    5062           56 :                                         TYPE_VECTOR_SUBPARTS (rtype), &k))
    5063            0 :                 gcc_unreachable ();
    5064           56 :               gcc_assert ((k & (k - 1)) == 0);
    5065           56 :               if (!ret_ctor_elts)
    5066           18 :                 vec_alloc (ret_ctor_elts, k);
    5067           56 :               if (ratype)
    5068              :                 {
    5069            4 :                   unsigned int m, o;
    5070            4 :                   o = vector_unroll_factor (nunits,
    5071              :                                             TYPE_VECTOR_SUBPARTS (rtype));
    5072           12 :                   for (m = 0; m < o; m++)
    5073              :                     {
    5074            8 :                       tree tem = build4 (ARRAY_REF, rtype, new_temp,
    5075            8 :                                          size_int (m), NULL_TREE, NULL_TREE);
    5076            8 :                       new_stmt = gimple_build_assign (make_ssa_name (rtype),
    5077              :                                                       tem);
    5078            8 :                       vect_finish_stmt_generation (vinfo, stmt_info,
    5079              :                                                    new_stmt, gsi);
    5080            8 :                       CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
    5081              :                                               gimple_assign_lhs (new_stmt));
    5082              :                     }
    5083            4 :                   vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
    5084              :                 }
    5085              :               else
    5086           52 :                 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
    5087           56 :               if (ret_ctor_elts->length () != k)
    5088           38 :                 continue;
    5089           18 :               vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
    5090           18 :               new_stmt
    5091           18 :                 = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
    5092           18 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5093           18 :               ret_ctor_elts = NULL;
    5094              : 
    5095           18 :               SLP_TREE_VEC_DEFS (slp_node)
    5096           18 :                 .quick_push (gimple_assign_lhs (new_stmt));
    5097           18 :               continue;
    5098           18 :             }
    5099          438 :           else if (ratype)
    5100              :             {
    5101            0 :               tree t = build_fold_addr_expr (new_temp);
    5102            0 :               t = build2 (MEM_REF, vectype, t,
    5103            0 :                           build_int_cst (TREE_TYPE (t), 0));
    5104            0 :               new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
    5105            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5106            0 :               vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
    5107              :             }
    5108          438 :           else if (!useless_type_conversion_p (vectype, rtype))
    5109              :             {
    5110            0 :               vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, vectype, new_temp);
    5111            0 :               new_stmt
    5112            0 :                 = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
    5113            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5114              :             }
    5115              :         }
    5116              : 
    5117          444 :       if (gimple_get_lhs (new_stmt))
    5118          438 :         SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
    5119              :     }
    5120              : 
    5121         1215 :   for (i = 0; i < nargs; ++i)
    5122              :     {
    5123          833 :       vec<tree> oprndsi = vec_oprnds[i];
    5124          833 :       oprndsi.release ();
    5125              :     }
    5126          382 :   vargs.release ();
    5127              : 
    5128              :   /* Mark the clone as no longer being a candidate for GC.  */
    5129          382 :   bestn->gc_candidate = false;
    5130              : 
    5131          382 :   return true;
    5132         1492 : }
    5133              : 
    5134              : 
    5135              : /* Function vect_gen_widened_results_half
    5136              : 
    5137              :    Create a vector stmt whose code, type, number of arguments, and result
    5138              :    variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
    5139              :    VEC_OPRND0 and VEC_OPRND1.  The new vector stmt is to be inserted at GSI.
    5140              :    In the case that CODE is a CALL_EXPR, this means that a call to DECL
    5141              :    needs to be created (DECL is a function-decl of a target-builtin).
    5142              :    STMT_INFO is the original scalar stmt that we are vectorizing.  */
    5143              : 
    5144              : static gimple *
    5145        31971 : vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
    5146              :                                tree vec_oprnd0, tree vec_oprnd1, int op_type,
    5147              :                                tree vec_dest, gimple_stmt_iterator *gsi,
    5148              :                                stmt_vec_info stmt_info)
    5149              : {
    5150        31971 :   gimple *new_stmt;
    5151        31971 :   tree new_temp;
    5152              : 
    5153              :   /* Generate half of the widened result:  */
    5154        31971 :   if (op_type != binary_op)
    5155        30835 :     vec_oprnd1 = NULL;
    5156        31971 :   new_stmt = vect_gimple_build (vec_dest, ch, vec_oprnd0, vec_oprnd1);
    5157        31971 :   new_temp = make_ssa_name (vec_dest, new_stmt);
    5158        31971 :   gimple_set_lhs (new_stmt, new_temp);
    5159        31971 :   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5160              : 
    5161        31971 :   return new_stmt;
    5162              : }
    5163              : 
    5164              : 
    5165              : /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
    5166              :    For multi-step conversions store the resulting vectors and call the function
    5167              :    recursively. When NARROW_SRC_P is true, there's still a conversion after
    5168              :    narrowing, don't store the vectors in the SLP_NODE or in vector info of
    5169              :    the scalar statement(or in STMT_VINFO_RELATED_STMT chain).  */
    5170              : 
    5171              : static void
    5172        12440 : vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
    5173              :                                        int multi_step_cvt,
    5174              :                                        stmt_vec_info stmt_info,
    5175              :                                        vec<tree> &vec_dsts,
    5176              :                                        gimple_stmt_iterator *gsi,
    5177              :                                        slp_tree slp_node, code_helper code,
    5178              :                                        bool narrow_src_p)
    5179              : {
    5180        12440 :   unsigned int i;
    5181        12440 :   tree vop0, vop1, new_tmp, vec_dest;
    5182              : 
    5183        12440 :   vec_dest = vec_dsts.pop ();
    5184              : 
    5185        29444 :   for (i = 0; i < vec_oprnds->length (); i += 2)
    5186              :     {
    5187              :       /* Create demotion operation.  */
    5188        17004 :       vop0 = (*vec_oprnds)[i];
    5189        17004 :       vop1 = (*vec_oprnds)[i + 1];
    5190        17004 :       gimple *new_stmt = vect_gimple_build (vec_dest, code, vop0, vop1);
    5191        17004 :       new_tmp = make_ssa_name (vec_dest, new_stmt);
    5192        17004 :       gimple_set_lhs (new_stmt, new_tmp);
    5193        17004 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5194        17004 :       if (multi_step_cvt || narrow_src_p)
    5195              :         /* Store the resulting vector for next recursive call,
    5196              :            or return the resulting vector_tmp for NARROW FLOAT_EXPR.  */
    5197         7010 :         (*vec_oprnds)[i/2] = new_tmp;
    5198              :       else
    5199              :         {
    5200              :           /* This is the last step of the conversion sequence. Store the
    5201              :              vectors in SLP_NODE.  */
    5202         9994 :           slp_node->push_vec_def (new_stmt);
    5203              :         }
    5204              :     }
    5205              : 
    5206              :   /* For multi-step demotion operations we first generate demotion operations
    5207              :      from the source type to the intermediate types, and then combine the
    5208              :      results (stored in VEC_OPRNDS) in demotion operation to the destination
    5209              :      type.  */
    5210        12440 :   if (multi_step_cvt)
    5211              :     {
    5212              :       /* At each level of recursion we have half of the operands we had at the
    5213              :          previous level.  */
    5214         3119 :       vec_oprnds->truncate ((i+1)/2);
    5215         3119 :       vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
    5216              :                                              multi_step_cvt - 1,
    5217              :                                              stmt_info, vec_dsts, gsi,
    5218         3119 :                                              slp_node, VEC_PACK_TRUNC_EXPR,
    5219              :                                              narrow_src_p);
    5220              :     }
    5221              : 
    5222        12440 :   vec_dsts.quick_push (vec_dest);
    5223        12440 : }
    5224              : 
    5225              : 
    5226              : /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
    5227              :    and VEC_OPRNDS1, for a binary operation associated with scalar statement
    5228              :    STMT_INFO.  For multi-step conversions store the resulting vectors and
    5229              :    call the function recursively.  When NUM_VECTORS is not -1U then only
    5230              :    NUM_VECTORS will be produced.  */
    5231              : 
    5232              : static void
    5233        11663 : vect_create_vectorized_promotion_stmts (vec_info *vinfo,
    5234              :                                         vec<tree> *vec_oprnds0,
    5235              :                                         vec<tree> *vec_oprnds1,
    5236              :                                         stmt_vec_info stmt_info, tree vec_dest,
    5237              :                                         gimple_stmt_iterator *gsi,
    5238              :                                         code_helper ch1,
    5239              :                                         code_helper ch2, int op_type,
    5240              :                                         unsigned num_vectors)
    5241              : {
    5242        11663 :   int i;
    5243        11663 :   tree vop0, vop1, new_tmp1, new_tmp2;
    5244        11663 :   gimple *new_stmt1, *new_stmt2;
    5245        11663 :   vec<tree> vec_tmp = vNULL;
    5246              : 
    5247        11663 :   vec_tmp.create (num_vectors != -1U
    5248         1830 :                   ? num_vectors : 2 * vec_oprnds0->length ());
    5249        27654 :   FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
    5250              :     {
    5251        15991 :       if (op_type == binary_op)
    5252          568 :         vop1 = (*vec_oprnds1)[i];
    5253              :       else
    5254              :         vop1 = NULL_TREE;
    5255              : 
    5256              :       /* Generate the two halves of promotion operation.  */
    5257        15991 :       new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
    5258              :                                                  op_type, vec_dest, gsi,
    5259              :                                                  stmt_info);
    5260        15991 :       new_tmp1 = gimple_get_lhs (new_stmt1);
    5261        15991 :       vec_tmp.quick_push (new_tmp1);
    5262              : 
    5263        15991 :       if (vec_tmp.space (1))
    5264              :         {
    5265        15980 :           new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
    5266              :                                                      op_type, vec_dest, gsi,
    5267              :                                                      stmt_info);
    5268        15980 :           new_tmp2 = gimple_get_lhs (new_stmt2);
    5269        15980 :           vec_tmp.quick_push (new_tmp2);
    5270              :         }
    5271              : 
    5272        15991 :       if (!vec_tmp.space (1))
    5273              :         break;
    5274              :     }
    5275              : 
    5276        11663 :   vec_oprnds0->release ();
    5277        11663 :   *vec_oprnds0 = vec_tmp;
    5278        11663 : }
    5279              : 
    5280              : /* Create vectorized promotion stmts for widening stmts using only half the
    5281              :    potential vector size for input.  */
    5282              : static void
    5283           44 : vect_create_half_widening_stmts (vec_info *vinfo,
    5284              :                                         vec<tree> *vec_oprnds0,
    5285              :                                         vec<tree> *vec_oprnds1,
    5286              :                                         stmt_vec_info stmt_info, tree vec_dest,
    5287              :                                         gimple_stmt_iterator *gsi,
    5288              :                                         code_helper code1,
    5289              :                                         int op_type)
    5290              : {
    5291           44 :   int i;
    5292           44 :   tree vop0, vop1;
    5293           44 :   gimple *new_stmt1;
    5294           44 :   gimple *new_stmt2;
    5295           44 :   gimple *new_stmt3;
    5296           44 :   vec<tree> vec_tmp = vNULL;
    5297              : 
    5298           44 :   vec_tmp.create (vec_oprnds0->length ());
    5299          132 :   FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
    5300              :     {
    5301           44 :       tree new_tmp1, new_tmp2, new_tmp3, out_type;
    5302              : 
    5303           44 :       gcc_assert (op_type == binary_op);
    5304           44 :       vop1 = (*vec_oprnds1)[i];
    5305              : 
    5306              :       /* Widen the first vector input.  */
    5307           44 :       out_type = TREE_TYPE (vec_dest);
    5308           44 :       new_tmp1 = make_ssa_name (out_type);
    5309           44 :       new_stmt1 = gimple_build_assign (new_tmp1, NOP_EXPR, vop0);
    5310           44 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt1, gsi);
    5311           44 :       if (VECTOR_TYPE_P (TREE_TYPE (vop1)))
    5312              :         {
    5313              :           /* Widen the second vector input.  */
    5314           44 :           new_tmp2 = make_ssa_name (out_type);
    5315           44 :           new_stmt2 = gimple_build_assign (new_tmp2, NOP_EXPR, vop1);
    5316           44 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt2, gsi);
    5317              :           /* Perform the operation.  With both vector inputs widened.  */
    5318           44 :           new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, new_tmp2);
    5319              :         }
    5320              :       else
    5321              :         {
    5322              :           /* Perform the operation.  With the single vector input widened.  */
    5323            0 :           new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, vop1);
    5324              :         }
    5325              : 
    5326           44 :       new_tmp3 = make_ssa_name (vec_dest, new_stmt3);
    5327           44 :       gimple_assign_set_lhs (new_stmt3, new_tmp3);
    5328           44 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt3, gsi);
    5329              : 
    5330              :       /* Store the results for the next step.  */
    5331           44 :       vec_tmp.quick_push (new_tmp3);
    5332              :     }
    5333              : 
    5334           44 :   vec_oprnds0->release ();
    5335           44 :   *vec_oprnds0 = vec_tmp;
    5336           44 : }
    5337              : 
    5338              : 
    5339              : /* Check if STMT_INFO performs a conversion operation that can be vectorized.
    5340              :    If COST_VEC is passed, calculate costs but don't change anything,
    5341              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    5342              :    it, and insert it at GSI.
    5343              :    Return true if STMT_INFO is vectorizable in this way.  */
    5344              : 
    5345              : static bool
    5346      2689066 : vectorizable_conversion (vec_info *vinfo,
    5347              :                          stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    5348              :                          slp_tree slp_node,
    5349              :                          stmt_vector_for_cost *cost_vec)
    5350              : {
    5351      2689066 :   tree vec_dest, cvt_op = NULL_TREE;
    5352      2689066 :   tree scalar_dest;
    5353      2689066 :   tree_code tc1;
    5354      2689066 :   code_helper code, code1, code2;
    5355      2689066 :   code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
    5356      2689066 :   tree new_temp;
    5357      2689066 :   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
    5358      2689066 :   poly_uint64 nunits_in;
    5359      2689066 :   poly_uint64 nunits_out;
    5360      2689066 :   tree vectype_out, vectype_in;
    5361      2689066 :   int i;
    5362      2689066 :   tree lhs_type, rhs_type;
    5363              :   /* For conversions between floating point and integer, there're 2 NARROW
    5364              :      cases. NARROW_SRC is for FLOAT_EXPR, means
    5365              :      integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
    5366              :      This is safe when the range of the source integer can fit into the lower
    5367              :      precision. NARROW_DST is for FIX_TRUNC_EXPR, means
    5368              :      floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
    5369              :      For other conversions, when there's narrowing, NARROW_DST is used as
    5370              :      default.  */
    5371      2689066 :   enum { NARROW_SRC, NARROW_DST, NONE, WIDEN } modifier;
    5372      2689066 :   vec<tree> vec_oprnds0 = vNULL;
    5373      2689066 :   vec<tree> vec_oprnds1 = vNULL;
    5374      2689066 :   tree vop0;
    5375      2689066 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    5376      2689066 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    5377      2689066 :   int multi_step_cvt = 0;
    5378      2689066 :   vec<tree> interm_types = vNULL;
    5379      2689066 :   tree intermediate_type, cvt_type = NULL_TREE;
    5380      2689066 :   int op_type;
    5381      2689066 :   unsigned short fltsz;
    5382              : 
    5383              :   /* Is STMT a vectorizable conversion?   */
    5384              : 
    5385      2689066 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    5386              :     return false;
    5387              : 
    5388      2689066 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    5389       251571 :       && cost_vec)
    5390              :     return false;
    5391              : 
    5392      2437495 :   gimple* stmt = stmt_info->stmt;
    5393      2437495 :   if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
    5394              :     return false;
    5395              : 
    5396      2375585 :   if (gimple_get_lhs (stmt) == NULL_TREE
    5397      2375585 :       || TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
    5398              :     return false;
    5399              : 
    5400      1538133 :   if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
    5401              :     return false;
    5402              : 
    5403      1538133 :   if (is_gimple_assign (stmt))
    5404              :     {
    5405      1524618 :       code = gimple_assign_rhs_code (stmt);
    5406      1524618 :       op_type = TREE_CODE_LENGTH ((tree_code) code);
    5407              :     }
    5408        13515 :   else if (gimple_call_internal_p (stmt))
    5409              :     {
    5410         7827 :       code = gimple_call_internal_fn (stmt);
    5411         7827 :       op_type = gimple_call_num_args (stmt);
    5412              :     }
    5413              :   else
    5414              :     return false;
    5415              : 
    5416      1532445 :   bool widen_arith = (code == WIDEN_MULT_EXPR
    5417      1529819 :                  || code == WIDEN_LSHIFT_EXPR
    5418      3062264 :                  || widening_fn_p (code));
    5419              : 
    5420      1529819 :   if (!widen_arith
    5421      1529819 :       && !CONVERT_EXPR_CODE_P (code)
    5422              :       && code != FIX_TRUNC_EXPR
    5423              :       && code != FLOAT_EXPR)
    5424              :     return false;
    5425              : 
    5426              :   /* Check types of lhs and rhs.  */
    5427       202041 :   scalar_dest = gimple_get_lhs (stmt);
    5428       202041 :   lhs_type = TREE_TYPE (scalar_dest);
    5429       202041 :   vectype_out = SLP_TREE_VECTYPE (slp_node);
    5430              : 
    5431              :   /* Check the operands of the operation.  */
    5432       202041 :   slp_tree slp_op0, slp_op1 = NULL;
    5433       202041 :   if (!vect_is_simple_use (vinfo, slp_node,
    5434              :                            0, &slp_op0, &dt[0], &vectype_in))
    5435              :     {
    5436            0 :       if (dump_enabled_p ())
    5437            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5438              :                          "use not simple.\n");
    5439              :       return false;
    5440              :     }
    5441              : 
    5442       202041 :   rhs_type = TREE_TYPE (gimple_arg (stmt, 0));
    5443       202041 :   if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
    5444       384107 :       && !((INTEGRAL_TYPE_P (lhs_type)
    5445       166983 :             && INTEGRAL_TYPE_P (rhs_type))
    5446              :            || (SCALAR_FLOAT_TYPE_P (lhs_type)
    5447        10165 :                && SCALAR_FLOAT_TYPE_P (rhs_type))))
    5448              :     return false;
    5449              : 
    5450       197123 :   if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
    5451       176559 :       && INTEGRAL_TYPE_P (lhs_type)
    5452       341516 :       && !type_has_mode_precision_p (lhs_type))
    5453              :     {
    5454          581 :       if (dump_enabled_p ())
    5455           54 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5456              :                          "type conversion to bit-precision unsupported\n");
    5457              :       return false;
    5458              :     }
    5459              : 
    5460              :   /* _BitInt values are not sign-/zero-extended to mode precision.  */
    5461       196542 :   if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
    5462       175978 :       && TREE_CODE (rhs_type) == BITINT_TYPE
    5463       196715 :       && !type_has_mode_precision_p (rhs_type))
    5464              :     {
    5465           16 :       if (dump_enabled_p ())
    5466            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5467              :                          "type conversion from _BitInt unsupported\n");
    5468              :       return false;
    5469              :     }
    5470              : 
    5471       196526 :   if (op_type == binary_op)
    5472              :     {
    5473         2626 :       gcc_assert (code == WIDEN_MULT_EXPR
    5474              :                   || code == WIDEN_LSHIFT_EXPR
    5475              :                   || widening_fn_p (code));
    5476              : 
    5477         2626 :       tree vectype1_in;
    5478         2626 :       if (!vect_is_simple_use (vinfo, slp_node, 1,
    5479              :                                &slp_op1, &dt[1], &vectype1_in))
    5480              :         {
    5481            0 :           if (dump_enabled_p ())
    5482            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5483              :                              "use not simple.\n");
    5484            0 :           return false;
    5485              :         }
    5486              :       /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
    5487              :          OP1.  */
    5488         2626 :       if (!vectype_in)
    5489          171 :         vectype_in = vectype1_in;
    5490              :     }
    5491              : 
    5492              :   /* If slp_op0 is an external or constant def, infer the vector type
    5493              :      from the scalar type.  */
    5494       196526 :   if (!cost_vec)
    5495        23709 :     gcc_assert (vectype_in);
    5496       196526 :   if (!vectype_in)
    5497        25289 :     vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
    5498       196526 :   if (!vectype_in)
    5499              :     {
    5500          280 :       if (dump_enabled_p ())
    5501            2 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5502              :                          "no vectype for scalar type %T\n", rhs_type);
    5503              : 
    5504              :       return false;
    5505              :     }
    5506              : 
    5507       392492 :   if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
    5508       196246 :       != VECTOR_BOOLEAN_TYPE_P (vectype_in))
    5509              :     {
    5510          309 :       if (dump_enabled_p ())
    5511           36 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5512              :                          "can't convert between boolean and non "
    5513              :                          "boolean vectors %T\n", rhs_type);
    5514              : 
    5515              :       return false;
    5516              :     }
    5517              : 
    5518       195937 :   nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
    5519       195937 :   nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
    5520       195937 :   if (known_eq (nunits_out, nunits_in))
    5521        93980 :     if (widen_arith)
    5522              :       modifier = WIDEN;
    5523              :     else
    5524       195937 :       modifier = NONE;
    5525       101957 :   else if (multiple_p (nunits_out, nunits_in))
    5526              :     modifier = NARROW_DST;
    5527              :   else
    5528              :     {
    5529        56827 :       gcc_checking_assert (multiple_p (nunits_in, nunits_out));
    5530              :       modifier = WIDEN;
    5531              :     }
    5532              : 
    5533       195937 :   bool found_mode = false;
    5534       195937 :   scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
    5535       195937 :   scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
    5536       195937 :   opt_scalar_mode rhs_mode_iter;
    5537       195937 :   auto_vec<std::pair<tree, tree_code>, 2> converts;
    5538       195937 :   bool evenodd_ok = false;
    5539              : 
    5540              :   /* Supportable by target?  */
    5541       195937 :   switch (modifier)
    5542              :     {
    5543        93619 :     case NONE:
    5544        93619 :       if (code != FIX_TRUNC_EXPR
    5545              :           && code != FLOAT_EXPR
    5546              :           && !CONVERT_EXPR_CODE_P (code))
    5547              :         return false;
    5548        93619 :       gcc_assert (code.is_tree_code ());
    5549        93619 :       if (supportable_indirect_convert_operation (code,
    5550              :                                                   vectype_out, vectype_in,
    5551              :                                                   converts, slp_op0))
    5552              :         {
    5553        86700 :           gcc_assert (converts.length () <= 2);
    5554        86700 :           if (converts.length () == 1)
    5555              :             {
    5556        86626 :               code1 = converts[0].second;
    5557        86626 :               if (CONVERT_EXPR_CODE_P (code)
    5558       162701 :                   && tree_nop_conversion_p (TREE_TYPE (vectype_out),
    5559        76075 :                                             TREE_TYPE (vectype_in)))
    5560              :                 /* NOP conversions are handled by vectorizable_assignment.  */
    5561              :                 return false;
    5562              :             }
    5563              :           else
    5564              :             {
    5565           74 :               cvt_type = NULL_TREE;
    5566           74 :               multi_step_cvt = converts.length () - 1;
    5567           74 :               codecvt1 = converts[0].second;
    5568           74 :               code1 = converts[1].second;
    5569           74 :               interm_types.safe_push (converts[0].first);
    5570              :             }
    5571              :           break;
    5572              :         }
    5573              : 
    5574              :       /* FALLTHRU */
    5575         6919 :     unsupported:
    5576        19613 :       if (dump_enabled_p ())
    5577          618 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5578              :                          "conversion not supported by target.\n");
    5579              :       return false;
    5580              : 
    5581        57188 :     case WIDEN:
    5582        57188 :       if (known_eq (nunits_in, nunits_out))
    5583              :         {
    5584          722 :           if (!(code.is_tree_code ()
    5585          361 :                 && supportable_half_widening_operation ((tree_code) code,
    5586              :                                                         vectype_out, vectype_in,
    5587              :                                                         &tc1)))
    5588          119 :             goto unsupported;
    5589          242 :           code1 = tc1;
    5590          242 :           gcc_assert (!(multi_step_cvt && op_type == binary_op));
    5591              :           break;
    5592              :         }
    5593              :       /* Elements in a vector can only be reordered if used in a reduction
    5594              :          operation only.  */
    5595        56827 :       if (code == WIDEN_MULT_EXPR
    5596         2265 :           && loop_vinfo
    5597         2124 :           && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
    5598              :           /* For a SLP reduction we cannot swizzle lanes, detecting a
    5599              :              reduction chain isn't possible here.  */
    5600        58929 :           && SLP_TREE_LANES (slp_node) == 1)
    5601              :         {
    5602              :           /* ???  There is no way to look for SLP uses, so work on
    5603              :              the stmt and what the stmt-based cycle detection gives us.  */
    5604         4008 :           tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
    5605         2004 :           stmt_vec_info use_stmt_info
    5606         2004 :             = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
    5607         2004 :           if (use_stmt_info
    5608         1859 :               && STMT_VINFO_REDUC_DEF (use_stmt_info))
    5609        56827 :             evenodd_ok = true;
    5610              :         }
    5611        56827 :       if (supportable_widening_operation (code, vectype_out, vectype_in,
    5612              :                                           evenodd_ok, &code1,
    5613              :                                           &code2, &multi_step_cvt,
    5614              :                                           &interm_types))
    5615              :         {
    5616              :           /* Binary widening operation can only be supported directly by the
    5617              :              architecture.  */
    5618        50398 :           gcc_assert (!(multi_step_cvt && op_type == binary_op));
    5619              :           break;
    5620              :         }
    5621              : 
    5622         6429 :       if (code != FLOAT_EXPR
    5623         6811 :           || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
    5624         6238 :         goto unsupported;
    5625              : 
    5626          191 :       fltsz = GET_MODE_SIZE (lhs_mode);
    5627          278 :       FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
    5628              :         {
    5629          278 :           rhs_mode = rhs_mode_iter.require ();
    5630          556 :           if (GET_MODE_SIZE (rhs_mode) > fltsz)
    5631              :             break;
    5632              : 
    5633          278 :           cvt_type
    5634          278 :             = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
    5635          278 :           cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
    5636          278 :           if (cvt_type == NULL_TREE)
    5637            0 :             goto unsupported;
    5638              : 
    5639          556 :           if (GET_MODE_SIZE (rhs_mode) == fltsz)
    5640              :             {
    5641           86 :               gcc_assert (code.is_tree_code ());
    5642           86 :               if (!supportable_convert_operation ((tree_code) code, vectype_out,
    5643              :                                                   cvt_type))
    5644           22 :                 goto unsupported;
    5645           64 :               codecvt1 = code;
    5646              :             }
    5647          192 :           else if (!supportable_widening_operation (code, vectype_out,
    5648              :                                                     cvt_type, evenodd_ok,
    5649              :                                                     &codecvt1,
    5650              :                                                     &codecvt2, &multi_step_cvt,
    5651              :                                                     &interm_types))
    5652           87 :             continue;
    5653              :           else
    5654          105 :             gcc_assert (multi_step_cvt == 0);
    5655              : 
    5656          169 :           if (supportable_widening_operation (NOP_EXPR, cvt_type,
    5657              :                                               vectype_in, evenodd_ok, &code1,
    5658              :                                               &code2, &multi_step_cvt,
    5659              :                                               &interm_types))
    5660              :             {
    5661              :               found_mode = true;
    5662              :               break;
    5663              :             }
    5664              :         }
    5665              : 
    5666          169 :       if (!found_mode)
    5667            0 :         goto unsupported;
    5668              : 
    5669          338 :       if (GET_MODE_SIZE (rhs_mode) == fltsz)
    5670           64 :         codecvt2 = ERROR_MARK;
    5671              :       else
    5672              :         {
    5673          105 :           multi_step_cvt++;
    5674          105 :           interm_types.safe_push (cvt_type);
    5675          105 :           cvt_type = NULL_TREE;
    5676              :         }
    5677              :       break;
    5678              : 
    5679        45130 :     case NARROW_DST:
    5680        45130 :       gcc_assert (op_type == unary_op);
    5681        45130 :       if (supportable_narrowing_operation (code, vectype_out, vectype_in,
    5682              :                                            &code1, &multi_step_cvt,
    5683              :                                            &interm_types))
    5684              :         break;
    5685              : 
    5686        19347 :       if (GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
    5687          986 :         goto unsupported;
    5688              : 
    5689         5463 :       if (code == FIX_TRUNC_EXPR)
    5690              :         {
    5691          107 :           cvt_type
    5692          107 :             = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
    5693          107 :           cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
    5694          107 :           if (cvt_type == NULL_TREE)
    5695            0 :             goto unsupported;
    5696          107 :           if (supportable_convert_operation ((tree_code) code, cvt_type,
    5697              :                                              vectype_in))
    5698          105 :             codecvt1 = code;
    5699              :           else
    5700            2 :             goto unsupported;
    5701          105 :           if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
    5702              :                                                &code1, &multi_step_cvt,
    5703              :                                                &interm_types))
    5704              :             break;
    5705              :         }
    5706              :       /* If slp_op0 can be represented with low precision integer,
    5707              :          truncate it to cvt_type and the do FLOAT_EXPR.  */
    5708         5356 :       else if (code == FLOAT_EXPR)
    5709              :         {
    5710          792 :           if (cost_vec)
    5711              :             {
    5712          784 :               wide_int op_min_value, op_max_value;
    5713          784 :               tree def;
    5714              : 
    5715              :               /* ???  Merge ranges in case of more than one lane.  */
    5716          784 :               if (SLP_TREE_LANES (slp_op0) != 1
    5717          133 :                   || !(def = vect_get_slp_scalar_def (slp_op0, 0))
    5718          917 :                   || !vect_get_range_info (def, &op_min_value, &op_max_value))
    5719          755 :                 goto unsupported;
    5720              : 
    5721           29 :               if ((wi::min_precision (op_max_value, SIGNED)
    5722           29 :                    > GET_MODE_BITSIZE (lhs_mode))
    5723           29 :                   || (wi::min_precision (op_min_value, SIGNED)
    5724           27 :                       > GET_MODE_BITSIZE (lhs_mode)))
    5725            2 :                 goto unsupported;
    5726          784 :             }
    5727              : 
    5728           35 :           cvt_type
    5729           35 :             = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0);
    5730           35 :           cvt_type = get_same_sized_vectype (cvt_type, vectype_out);
    5731           35 :           if (cvt_type == NULL_TREE)
    5732            0 :             goto unsupported;
    5733           35 :           if (!supportable_narrowing_operation (NOP_EXPR, cvt_type, vectype_in,
    5734              :                                                 &code1, &multi_step_cvt,
    5735              :                                                 &interm_types))
    5736            2 :             goto unsupported;
    5737           33 :           if (supportable_convert_operation ((tree_code) code, vectype_out,
    5738              :                                              cvt_type))
    5739              :             {
    5740           33 :               codecvt1 = code;
    5741           33 :               modifier = NARROW_SRC;
    5742           33 :               break;
    5743              :             }
    5744              :         }
    5745              : 
    5746         4568 :       goto unsupported;
    5747              : 
    5748              :     default:
    5749              :       gcc_unreachable ();
    5750              :     }
    5751              : 
    5752       111260 :   if (modifier == WIDEN
    5753       111260 :       && loop_vinfo
    5754        49240 :       && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    5755       132492 :       && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
    5756        21214 :           || widening_evenodd_fn_p (code1)))
    5757              :     {
    5758           18 :       if (dump_enabled_p ())
    5759            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5760              :                          "can't use a fully-masked loop because"
    5761              :                          " widening operation on even/odd elements"
    5762              :                          " mixes up lanes.\n");
    5763           18 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    5764              :     }
    5765              : 
    5766       111260 :   if (cost_vec)         /* transformation not required.  */
    5767              :     {
    5768        87551 :       if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
    5769        87551 :           || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
    5770              :         {
    5771            0 :           if (dump_enabled_p ())
    5772            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5773              :                              "incompatible vector types for invariants\n");
    5774              :           return false;
    5775              :         }
    5776        87551 :       DUMP_VECT_SCOPE ("vectorizable_conversion");
    5777        87551 :       unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
    5778        87551 :       if (modifier == NONE)
    5779              :         {
    5780        17125 :           SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
    5781        17125 :           vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
    5782              :                                   slp_node, cost_vec);
    5783              :         }
    5784        70426 :       else if (modifier == NARROW_SRC || modifier == NARROW_DST)
    5785              :         {
    5786        29494 :           SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
    5787              :           /* The final packing step produces one vector result per copy.  */
    5788        29494 :           vect_model_promotion_demotion_cost (slp_node, nvectors,
    5789              :                                               multi_step_cvt, cost_vec,
    5790              :                                               widen_arith);
    5791              :         }
    5792              :       else
    5793              :         {
    5794        40932 :           SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
    5795              :           /* The initial unpacking step produces two vector results
    5796              :              per copy.  MULTI_STEP_CVT is 0 for a single conversion,
    5797              :              so >> MULTI_STEP_CVT divides by 2^(number of steps - 1).  */
    5798        40932 :           vect_model_promotion_demotion_cost (slp_node,
    5799              :                                               nvectors >> multi_step_cvt,
    5800              :                                               multi_step_cvt, cost_vec,
    5801              :                                               widen_arith);
    5802              :         }
    5803        87551 :       interm_types.release ();
    5804        87551 :       return true;
    5805              :     }
    5806              : 
    5807              :   /* Transform.  */
    5808        23709 :   if (dump_enabled_p ())
    5809         4488 :     dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n");
    5810              : 
    5811              :   /* In case of multi-step conversion, we first generate conversion operations
    5812              :      to the intermediate types, and then from that types to the final one.
    5813              :      We create vector destinations for the intermediate type (TYPES) received
    5814              :      from supportable_*_operation, and store them in the correct order
    5815              :      for future use in vect_create_vectorized_*_stmts ().  */
    5816        47418 :   auto_vec<tree> vec_dsts (multi_step_cvt + 1);
    5817        23709 :   bool widen_or_narrow_float_p
    5818        23709 :     = cvt_type && (modifier == WIDEN || modifier == NARROW_SRC);
    5819        23709 :   vec_dest = vect_create_destination_var (scalar_dest,
    5820              :                                           widen_or_narrow_float_p
    5821              :                                           ? cvt_type : vectype_out);
    5822        23709 :   vec_dsts.quick_push (vec_dest);
    5823              : 
    5824        23709 :   if (multi_step_cvt)
    5825              :     {
    5826         9455 :       for (i = interm_types.length () - 1;
    5827         9455 :            interm_types.iterate (i, &intermediate_type); i--)
    5828              :         {
    5829         4970 :           vec_dest = vect_create_destination_var (scalar_dest,
    5830              :                                                   intermediate_type);
    5831         4970 :           vec_dsts.quick_push (vec_dest);
    5832              :         }
    5833              :     }
    5834              : 
    5835        23709 :   if (cvt_type)
    5836           80 :     vec_dest = vect_create_destination_var (scalar_dest,
    5837              :                                             widen_or_narrow_float_p
    5838              :                                             ? vectype_out : cvt_type);
    5839              : 
    5840        23709 :   unsigned num_vectors = vect_get_num_copies (vinfo, slp_node);
    5841        23709 :   switch (modifier)
    5842              :     {
    5843         4511 :     case NONE:
    5844         4511 :       vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
    5845              :       /* vec_dest is intermediate type operand when multi_step_cvt.  */
    5846         4511 :       if (multi_step_cvt)
    5847              :         {
    5848           21 :           cvt_op = vec_dest;
    5849           21 :           vec_dest = vec_dsts[0];
    5850              :         }
    5851              : 
    5852         9415 :       FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
    5853              :         {
    5854              :           /* Arguments are ready, create the new vector stmt.  */
    5855         4904 :           gimple* new_stmt;
    5856         4904 :           if (multi_step_cvt)
    5857              :             {
    5858           21 :               gcc_assert (multi_step_cvt == 1);
    5859           21 :               new_stmt = vect_gimple_build (cvt_op, codecvt1, vop0);
    5860           21 :               new_temp = make_ssa_name (cvt_op, new_stmt);
    5861           21 :               gimple_assign_set_lhs (new_stmt, new_temp);
    5862           21 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5863           21 :               vop0 = new_temp;
    5864              :             }
    5865         4904 :           if (code1 == COND_EXPR)
    5866              :             {
    5867            1 :               gcc_assert (!multi_step_cvt);
    5868            1 :               new_stmt
    5869            2 :                 = gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
    5870              :                                        build_minus_one_cst
    5871            1 :                                          (TREE_TYPE (vec_dest)),
    5872            1 :                                        build_zero_cst (TREE_TYPE (vec_dest)));
    5873            1 :               new_temp = make_ssa_name (vec_dest, new_stmt);
    5874            1 :               gimple_set_lhs (new_stmt, new_temp);
    5875            1 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5876            1 :               tree new_temp2 = make_ssa_name (vectype_out);
    5877            1 :               new_stmt = gimple_build_assign (new_temp2,
    5878              :                                               build1 (VIEW_CONVERT_EXPR,
    5879              :                                                       vectype_out, new_temp));
    5880            1 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5881            1 :               slp_node->push_vec_def (new_temp2);
    5882              :             }
    5883              :           else
    5884              :             {
    5885         4903 :               new_stmt = vect_gimple_build (vec_dest, code1, vop0);
    5886         4903 :               new_temp = make_ssa_name (vec_dest, new_stmt);
    5887         4903 :               gimple_set_lhs (new_stmt, new_temp);
    5888         4903 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5889              : 
    5890         4903 :               slp_node->push_vec_def (new_stmt);
    5891              :             }
    5892              :         }
    5893              :       break;
    5894              : 
    5895         9877 :     case WIDEN:
    5896              :       /* In case the vectorization factor (VF) is bigger than the number
    5897              :          of elements that we can fit in a vectype (nunits), we have to
    5898              :          generate more than one vector stmt - i.e - we need to "unroll"
    5899              :          the vector stmt by a factor VF/nunits.  */
    5900        19754 :       vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
    5901         9877 :                          code != WIDEN_LSHIFT_EXPR && slp_op1, &vec_oprnds1);
    5902         9877 :       if (code == WIDEN_LSHIFT_EXPR)
    5903              :         {
    5904            0 :           int oprnds_size = vec_oprnds0.length ();
    5905            0 :           vec_oprnds1.create (oprnds_size);
    5906            0 :           for (i = 0; i < oprnds_size; ++i)
    5907            0 :             vec_oprnds1.quick_push (vect_get_slp_scalar_def (slp_op1, 0));
    5908              :         }
    5909              :       /* Arguments are ready.  Create the new vector stmts.  */
    5910        21584 :       for (i = multi_step_cvt; i >= 0; i--)
    5911              :         {
    5912        11707 :           tree this_dest = vec_dsts[i];
    5913        11707 :           code_helper c1 = code1, c2 = code2;
    5914        11707 :           if (i == 0 && codecvt2 != ERROR_MARK)
    5915              :             {
    5916           48 :               c1 = codecvt1;
    5917           48 :               c2 = codecvt2;
    5918              :             }
    5919        11707 :           if (known_eq (nunits_out, nunits_in))
    5920           44 :             vect_create_half_widening_stmts (vinfo, &vec_oprnds0, &vec_oprnds1,
    5921              :                                              stmt_info, this_dest, gsi, c1,
    5922              :                                              op_type);
    5923              :           else
    5924              :             /* ???  For constant/external inputs we can end up with
    5925              :                excess lanes.  When the number of inputs already match
    5926              :                the number of required outputs request half of the
    5927              :                lanes (gcc.dg/vect/O3-vect-pr32243.c).  Low coverage
    5928              :                makes this likely incomplete.  */
    5929        13493 :             vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
    5930              :                                                     &vec_oprnds1, stmt_info,
    5931              :                                                     this_dest, gsi,
    5932              :                                                     c1, c2, op_type,
    5933              :                                                     i == 0 ? num_vectors : -1u);
    5934              :         }
    5935              : 
    5936        61343 :       FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
    5937              :         {
    5938        27757 :           gimple *new_stmt;
    5939        27757 :           if (cvt_type)
    5940              :             {
    5941          128 :               new_temp = make_ssa_name (vec_dest);
    5942          128 :               new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
    5943          128 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5944              :             }
    5945              :           else
    5946        27629 :             new_stmt = SSA_NAME_DEF_STMT (vop0);
    5947              : 
    5948        27757 :           slp_node->push_vec_def (new_stmt);
    5949              :         }
    5950              :       break;
    5951              : 
    5952         9321 :     case NARROW_SRC:
    5953         9321 :     case NARROW_DST:
    5954              :       /* In case the vectorization factor (VF) is bigger than the number
    5955              :          of elements that we can fit in a vectype (nunits), we have to
    5956              :          generate more than one vector stmt - i.e - we need to "unroll"
    5957              :          the vector stmt by a factor VF/nunits.  */
    5958         9321 :       vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
    5959              :       /* Arguments are ready.  Create the new vector stmts.  */
    5960         9321 :       if (cvt_type && modifier == NARROW_DST)
    5961         9474 :         FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
    5962              :           {
    5963          124 :             new_temp = make_ssa_name (vec_dest);
    5964          124 :             gimple *new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
    5965          124 :             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5966          124 :             vec_oprnds0[i] = new_temp;
    5967              :           }
    5968              : 
    5969         9321 :       vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
    5970              :                                              multi_step_cvt,
    5971              :                                              stmt_info, vec_dsts, gsi,
    5972              :                                              slp_node, code1,
    5973              :                                              modifier == NARROW_SRC);
    5974              :       /* After demoting slp_op0 to cvt_type, convert it to dest.  */
    5975         9321 :       if (cvt_type && code == FLOAT_EXPR)
    5976              :         {
    5977           32 :           for (unsigned int i = 0; i != vec_oprnds0.length() / 2;  i++)
    5978              :             {
    5979              :               /* Arguments are ready, create the new vector stmt.  */
    5980            8 :               gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
    5981            8 :               gimple *new_stmt
    5982            8 :                 = vect_gimple_build (vec_dest, codecvt1, vec_oprnds0[i]);
    5983            8 :               new_temp = make_ssa_name (vec_dest, new_stmt);
    5984            8 :               gimple_set_lhs (new_stmt, new_temp);
    5985            8 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    5986              : 
    5987              :               /* This is the last step of the conversion sequence. Store the
    5988              :                  vectors in SLP_NODE or in vector info of the scalar statement
    5989              :                  (or in STMT_VINFO_RELATED_STMT chain).  */
    5990            8 :               slp_node->push_vec_def (new_stmt);
    5991              :             }
    5992              :         }
    5993              :       break;
    5994              :     }
    5995              : 
    5996        23709 :   vec_oprnds0.release ();
    5997        23709 :   vec_oprnds1.release ();
    5998        23709 :   interm_types.release ();
    5999              : 
    6000        23709 :   return true;
    6001       195937 : }
    6002              : 
    6003              : /* Return true if we can assume from the scalar form of STMT_INFO that
    6004              :    neither the scalar nor the vector forms will generate code.  STMT_INFO
    6005              :    is known not to involve a data reference.  */
    6006              : 
    6007              : bool
    6008      3359507 : vect_nop_conversion_p (stmt_vec_info stmt_info)
    6009              : {
    6010      3359507 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
    6011      3049605 :   if (!stmt || STMT_VINFO_DATA_REF (stmt_info))
    6012              :     return false;
    6013              : 
    6014      1022919 :   tree lhs = gimple_assign_lhs (stmt);
    6015      1022919 :   tree_code code = gimple_assign_rhs_code (stmt);
    6016      1022919 :   tree rhs = gimple_assign_rhs1 (stmt);
    6017              : 
    6018      1022919 :   if (code == SSA_NAME || code == VIEW_CONVERT_EXPR)
    6019              :     return true;
    6020              : 
    6021      1019554 :   if (CONVERT_EXPR_CODE_P (code))
    6022       255436 :     return tree_nop_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
    6023              : 
    6024              :   return false;
    6025              : }
    6026              : 
    6027              : /* Function vectorizable_assignment.
    6028              : 
    6029              :    Check if STMT_INFO performs an assignment (copy) that can be vectorized.
    6030              :    If COST_VEC is passed, calculate costs but don't change anything,
    6031              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    6032              :    it, and insert it at GSI.
    6033              :    Return true if STMT_INFO is vectorizable in this way.  */
    6034              : 
    6035              : static bool
    6036      2170961 : vectorizable_assignment (vec_info *vinfo,
    6037              :                          stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    6038              :                          slp_tree slp_node,
    6039              :                          stmt_vector_for_cost *cost_vec)
    6040              : {
    6041      2170961 :   tree vec_dest;
    6042      2170961 :   tree scalar_dest;
    6043      2170961 :   tree op;
    6044      2170961 :   tree new_temp;
    6045      2170961 :   enum vect_def_type dt[1] = {vect_unknown_def_type};
    6046      2170961 :   int i;
    6047      2170961 :   vec<tree> vec_oprnds = vNULL;
    6048      2170961 :   tree vop;
    6049      2170961 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    6050      2170961 :   enum tree_code code;
    6051      2170961 :   tree vectype_in;
    6052              : 
    6053      2170961 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    6054              :     return false;
    6055              : 
    6056      2170961 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    6057       251571 :       && cost_vec)
    6058              :     return false;
    6059              : 
    6060              :   /* Is vectorizable assignment?  */
    6061      1919390 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
    6062      1842614 :   if (!stmt)
    6063              :     return false;
    6064              : 
    6065      1842614 :   scalar_dest = gimple_assign_lhs (stmt);
    6066      1842614 :   if (TREE_CODE (scalar_dest) != SSA_NAME)
    6067              :     return false;
    6068              : 
    6069      1006513 :   if (STMT_VINFO_DATA_REF (stmt_info))
    6070              :     return false;
    6071              : 
    6072       423991 :   code = gimple_assign_rhs_code (stmt);
    6073       423991 :   if (!(gimple_assign_single_p (stmt)
    6074       422337 :         || code == PAREN_EXPR
    6075       421065 :         || CONVERT_EXPR_CODE_P (code)))
    6076              :     return false;
    6077              : 
    6078       106134 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    6079       106134 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    6080              : 
    6081       106134 :   slp_tree slp_op;
    6082       106134 :   if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
    6083              :                            &dt[0], &vectype_in))
    6084              :     {
    6085            0 :       if (dump_enabled_p ())
    6086            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6087              :                          "use not simple.\n");
    6088              :       return false;
    6089              :     }
    6090       106134 :   if (!vectype_in)
    6091        20104 :     vectype_in = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node);
    6092              : 
    6093              :   /* We can handle VIEW_CONVERT conversions that do not change the number
    6094              :      of elements or the vector size or other conversions when the component
    6095              :      types are nop-convertible.  */
    6096       106134 :   if (!vectype_in
    6097       110794 :       || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
    6098        93542 :       || (code == VIEW_CONVERT_EXPR
    6099         3020 :           && maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
    6100         3020 :                        GET_MODE_SIZE (TYPE_MODE (vectype_in))))
    6101       199676 :       || (CONVERT_EXPR_CODE_P (code)
    6102        90648 :           && !tree_nop_conversion_p (TREE_TYPE (vectype),
    6103        90648 :                                      TREE_TYPE (vectype_in))))
    6104              :     return false;
    6105              : 
    6106       266193 :   if (VECTOR_BOOLEAN_TYPE_P (vectype) != VECTOR_BOOLEAN_TYPE_P (vectype_in))
    6107              :     {
    6108           27 :       if (dump_enabled_p ())
    6109            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6110              :                          "can't convert between boolean and non "
    6111            0 :                          "boolean vectors %T\n", TREE_TYPE (op));
    6112              : 
    6113              :       return false;
    6114              :     }
    6115              : 
    6116              :   /* We do not handle bit-precision changes.  */
    6117        88855 :   if ((CONVERT_EXPR_CODE_P (code)
    6118         2894 :        || code == VIEW_CONVERT_EXPR)
    6119        87471 :       && ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
    6120        86183 :            && !type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
    6121        86925 :           || (INTEGRAL_TYPE_P (TREE_TYPE (op))
    6122        78867 :               && !type_has_mode_precision_p (TREE_TYPE (op))))
    6123              :       /* But a conversion that does not change the bit-pattern is ok.  */
    6124        90003 :       && !(INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
    6125         1148 :            && INTEGRAL_TYPE_P (TREE_TYPE (op))
    6126         1148 :            && (((TYPE_PRECISION (TREE_TYPE (scalar_dest))
    6127         1148 :                > TYPE_PRECISION (TREE_TYPE (op)))
    6128          602 :              && TYPE_UNSIGNED (TREE_TYPE (op)))
    6129          608 :                || (TYPE_PRECISION (TREE_TYPE (scalar_dest))
    6130          608 :                    == TYPE_PRECISION (TREE_TYPE (op))))))
    6131              :     {
    6132          377 :       if (dump_enabled_p ())
    6133           42 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6134              :                          "type conversion to/from bit-precision "
    6135              :                          "unsupported.\n");
    6136              :       return false;
    6137              :     }
    6138              : 
    6139        88478 :   if (cost_vec) /* transformation not required.  */
    6140              :     {
    6141        71915 :       if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
    6142              :         {
    6143            0 :           if (dump_enabled_p ())
    6144            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6145              :                              "incompatible vector types for invariants\n");
    6146              :           return false;
    6147              :         }
    6148        71915 :       SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
    6149        71915 :       DUMP_VECT_SCOPE ("vectorizable_assignment");
    6150        71915 :       if (!vect_nop_conversion_p (stmt_info))
    6151         1118 :         vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
    6152        71915 :       return true;
    6153              :     }
    6154              : 
    6155              :   /* Transform.  */
    6156        16563 :   if (dump_enabled_p ())
    6157         3819 :     dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
    6158              : 
    6159              :   /* Handle def.  */
    6160        16563 :   vec_dest = vect_create_destination_var (scalar_dest, vectype);
    6161              : 
    6162              :   /* Handle use.  */
    6163        16563 :   vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
    6164              : 
    6165              :   /* Arguments are ready. create the new vector stmt.  */
    6166        53700 :   FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
    6167              :     {
    6168        20574 :       gassign *new_stmt;
    6169        20574 :       if (code == PAREN_EXPR)
    6170          510 :         new_stmt = gimple_build_assign (vec_dest, PAREN_EXPR, vop);
    6171              :       else
    6172              :         {
    6173        20064 :           if (CONVERT_EXPR_CODE_P (code)
    6174          208 :               || code == VIEW_CONVERT_EXPR)
    6175        20004 :             vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
    6176        20064 :           new_stmt = gimple_build_assign (vec_dest, vop);
    6177              :         }
    6178        20574 :       new_temp = make_ssa_name (vec_dest, new_stmt);
    6179        20574 :       gimple_assign_set_lhs (new_stmt, new_temp);
    6180        20574 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    6181        20574 :       slp_node->push_vec_def (new_stmt);
    6182              :     }
    6183              : 
    6184        16563 :   vec_oprnds.release ();
    6185        16563 :   return true;
    6186              : }
    6187              : 
    6188              : 
    6189              : /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
    6190              :    either as shift by a scalar or by a vector.  */
    6191              : 
    6192              : bool
    6193       317200 : vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
    6194              : {
    6195       317200 :   optab optab;
    6196       317200 :   tree vectype;
    6197              : 
    6198       317200 :   vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
    6199       317200 :   if (!vectype)
    6200              :     return false;
    6201              : 
    6202       317200 :   optab = optab_for_tree_code (code, vectype, optab_scalar);
    6203       317200 :   if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
    6204              :     return true;
    6205              : 
    6206       278508 :   optab = optab_for_tree_code (code, vectype, optab_vector);
    6207       278508 :   if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
    6208              :     return true;
    6209              : 
    6210              :   return false;
    6211              : }
    6212              : 
    6213              : 
    6214              : /* Function vectorizable_shift.
    6215              : 
    6216              :    Check if STMT_INFO performs a shift operation that can be vectorized.
    6217              :    If COST_VEC is passed, calculate costs but don't change anything,
    6218              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    6219              :    it, and insert it at GSI.
    6220              :    Return true if STMT_INFO is vectorizable in this way.  */
    6221              : 
    6222              : static bool
    6223       789659 : vectorizable_shift (vec_info *vinfo,
    6224              :                     stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    6225              :                     slp_tree slp_node,
    6226              :                     stmt_vector_for_cost *cost_vec)
    6227              : {
    6228       789659 :   tree vec_dest;
    6229       789659 :   tree scalar_dest;
    6230       789659 :   tree op0, op1 = NULL;
    6231       789659 :   tree vec_oprnd1 = NULL_TREE;
    6232       789659 :   tree vectype;
    6233       789659 :   enum tree_code code;
    6234       789659 :   machine_mode vec_mode;
    6235       789659 :   tree new_temp;
    6236       789659 :   optab optab;
    6237       789659 :   int icode;
    6238       789659 :   machine_mode optab_op2_mode;
    6239       789659 :   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
    6240       789659 :   poly_uint64 nunits_in;
    6241       789659 :   poly_uint64 nunits_out;
    6242       789659 :   tree vectype_out;
    6243       789659 :   tree op1_vectype;
    6244       789659 :   int i;
    6245       789659 :   vec<tree> vec_oprnds0 = vNULL;
    6246       789659 :   vec<tree> vec_oprnds1 = vNULL;
    6247       789659 :   tree vop0, vop1;
    6248       789659 :   unsigned int k;
    6249       789659 :   bool scalar_shift_arg = true;
    6250       789659 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    6251       789659 :   bool incompatible_op1_vectype_p = false;
    6252              : 
    6253       789659 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    6254              :     return false;
    6255              : 
    6256       789659 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    6257       251571 :       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
    6258       250179 :       && cost_vec)
    6259              :     return false;
    6260              : 
    6261              :   /* Is STMT a vectorizable binary/unary operation?   */
    6262       539480 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
    6263       463804 :   if (!stmt)
    6264              :     return false;
    6265              : 
    6266       463804 :   if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
    6267              :     return false;
    6268              : 
    6269       463346 :   code = gimple_assign_rhs_code (stmt);
    6270              : 
    6271       463346 :   if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
    6272              :       || code == RROTATE_EXPR))
    6273              :     return false;
    6274              : 
    6275        65101 :   scalar_dest = gimple_assign_lhs (stmt);
    6276        65101 :   vectype_out = SLP_TREE_VECTYPE (slp_node);
    6277        65101 :   if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
    6278              :     {
    6279            0 :       if (dump_enabled_p ())
    6280            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6281              :                          "bit-precision shifts not supported.\n");
    6282              :       return false;
    6283              :     }
    6284              : 
    6285        65101 :   slp_tree slp_op0;
    6286        65101 :   if (!vect_is_simple_use (vinfo, slp_node,
    6287              :                            0, &op0, &slp_op0, &dt[0], &vectype))
    6288              :     {
    6289            0 :       if (dump_enabled_p ())
    6290            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6291              :                          "use not simple.\n");
    6292              :       return false;
    6293              :     }
    6294              :   /* If op0 is an external or constant def, infer the vector type
    6295              :      from the scalar type.  */
    6296        65101 :   if (!vectype)
    6297        14590 :     vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
    6298        65101 :   if (!cost_vec)
    6299         9232 :     gcc_assert (vectype);
    6300        65101 :   if (!vectype)
    6301              :     {
    6302            0 :       if (dump_enabled_p ())
    6303            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6304              :                          "no vectype for scalar type\n");
    6305              :       return false;
    6306              :     }
    6307              : 
    6308        65101 :   nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
    6309        65101 :   nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
    6310        65101 :   if (maybe_ne (nunits_out, nunits_in))
    6311              :     return false;
    6312              : 
    6313        65101 :   stmt_vec_info op1_def_stmt_info;
    6314        65101 :   slp_tree slp_op1;
    6315        65101 :   if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
    6316              :                            &dt[1], &op1_vectype, &op1_def_stmt_info))
    6317              :     {
    6318            0 :       if (dump_enabled_p ())
    6319            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6320              :                          "use not simple.\n");
    6321              :       return false;
    6322              :     }
    6323              : 
    6324              :   /* Determine whether the shift amount is a vector, or scalar.  If the
    6325              :      shift/rotate amount is a vector, use the vector/vector shift optabs.  */
    6326              : 
    6327        65101 :   if ((dt[1] == vect_internal_def
    6328        65101 :        || dt[1] == vect_induction_def
    6329        48449 :        || dt[1] == vect_nested_cycle)
    6330        16670 :       && SLP_TREE_LANES (slp_node) == 1)
    6331              :     scalar_shift_arg = false;
    6332        48490 :   else if (dt[1] == vect_constant_def
    6333              :            || dt[1] == vect_external_def
    6334        48490 :            || dt[1] == vect_internal_def)
    6335              :     {
    6336              :       /* For internal SLP defs we have to make sure we see scalar stmts
    6337              :          for all vector elements.
    6338              :          ???  For different vectors we could resort to a different
    6339              :          scalar shift operand but code-generation below simply always
    6340              :          takes the first.  */
    6341        48484 :       if (dt[1] == vect_internal_def
    6342        48484 :           && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node),
    6343              :                        SLP_TREE_LANES (slp_node)))
    6344              :         scalar_shift_arg = false;
    6345              :       /* If the shift amount is computed by a pattern stmt we cannot
    6346              :          use the scalar amount directly thus give up and use a vector
    6347              :          shift.  */
    6348        48474 :       else if (op1_def_stmt_info && is_pattern_stmt_p (op1_def_stmt_info))
    6349              :         scalar_shift_arg = false;
    6350              :       else
    6351              :         {
    6352        48467 :           tree l0 = vect_get_slp_scalar_def (slp_op1, 0);
    6353        48467 :           if (!l0)
    6354              :             scalar_shift_arg = false;
    6355              :           else
    6356        76283 :             for (k = 1; k < SLP_TREE_LANES (slp_node); ++k)
    6357        27846 :               if (!operand_equal_p (vect_get_slp_scalar_def (slp_op1, k), l0))
    6358              :                 {
    6359              :                   scalar_shift_arg = false;
    6360              :                   break;
    6361              :                 }
    6362              :         }
    6363              :     }
    6364              :   else
    6365              :     {
    6366            6 :       if (dump_enabled_p ())
    6367            6 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6368              :                          "operand mode requires invariant argument.\n");
    6369              :       return false;
    6370              :     }
    6371              : 
    6372              :   /* Vector shifted by vector.  */
    6373        65125 :   bool was_scalar_shift_arg = scalar_shift_arg;
    6374        48467 :   if (!scalar_shift_arg)
    6375              :     {
    6376        16658 :       optab = optab_for_tree_code (code, vectype, optab_vector);
    6377        16658 :       if (dump_enabled_p ())
    6378         1251 :         dump_printf_loc (MSG_NOTE, vect_location,
    6379              :                          "vector/vector shift/rotate found.\n");
    6380              : 
    6381        16658 :       if (!op1_vectype)
    6382           15 :         op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1),
    6383              :                                                    slp_op1);
    6384        16658 :       incompatible_op1_vectype_p
    6385        33316 :         = (op1_vectype == NULL_TREE
    6386        16658 :            || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype),
    6387        16658 :                         TYPE_VECTOR_SUBPARTS (vectype))
    6388        33314 :            || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
    6389        16658 :       if (incompatible_op1_vectype_p
    6390            7 :           && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
    6391            1 :               || slp_op1->refcnt != 1))
    6392              :         {
    6393            6 :           if (dump_enabled_p ())
    6394            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6395              :                              "unusable type for last operand in"
    6396              :                              " vector/vector shift/rotate.\n");
    6397              :           return false;
    6398              :         }
    6399              :     }
    6400              :   /* See if the machine has a vector shifted by scalar insn and if not
    6401              :      then see if it has a vector shifted by vector insn.  */
    6402              :   else
    6403              :     {
    6404        48437 :       optab = optab_for_tree_code (code, vectype, optab_scalar);
    6405        48437 :       if (optab
    6406        48437 :           && can_implement_p (optab, TYPE_MODE (vectype)))
    6407              :         {
    6408        48167 :           if (dump_enabled_p ())
    6409         5683 :             dump_printf_loc (MSG_NOTE, vect_location,
    6410              :                              "vector/scalar shift/rotate found.\n");
    6411              :         }
    6412              :       else
    6413              :         {
    6414          270 :           optab = optab_for_tree_code (code, vectype, optab_vector);
    6415          270 :           if (optab
    6416          270 :               && can_implement_p (optab, TYPE_MODE (vectype)))
    6417              :             {
    6418            0 :               scalar_shift_arg = false;
    6419              : 
    6420            0 :               if (dump_enabled_p ())
    6421            0 :                 dump_printf_loc (MSG_NOTE, vect_location,
    6422              :                                  "vector/vector shift/rotate found.\n");
    6423              : 
    6424            0 :               if (!op1_vectype)
    6425            0 :                 op1_vectype = get_vectype_for_scalar_type (vinfo,
    6426            0 :                                                            TREE_TYPE (op1),
    6427              :                                                            slp_op1);
    6428              : 
    6429              :               /* Unlike the other binary operators, shifts/rotates have
    6430              :                  the rhs being int, instead of the same type as the lhs,
    6431              :                  so make sure the scalar is the right type if we are
    6432              :                  dealing with vectors of long long/long/short/char.  */
    6433            0 :               incompatible_op1_vectype_p
    6434            0 :                 = (!op1_vectype
    6435            0 :                    || !tree_nop_conversion_p (TREE_TYPE (vectype),
    6436            0 :                                               TREE_TYPE (op1)));
    6437              :               if (incompatible_op1_vectype_p
    6438            0 :                   && dt[1] == vect_internal_def)
    6439              :                 {
    6440            0 :                   if (dump_enabled_p ())
    6441            0 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6442              :                                      "unusable type for last operand in"
    6443              :                                      " vector/vector shift/rotate.\n");
    6444              :                   return false;
    6445              :                 }
    6446              :             }
    6447              :         }
    6448              :     }
    6449              : 
    6450              :   /* Supportable by target?  */
    6451        65089 :   if (!optab)
    6452              :     {
    6453            0 :       if (dump_enabled_p ())
    6454            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6455              :                          "no shift optab for %s and %T.\n",
    6456              :                          get_tree_code_name (code), vectype);
    6457              :       return false;
    6458              :     }
    6459        65089 :   vec_mode = TYPE_MODE (vectype);
    6460        65089 :   icode = (int) optab_handler (optab, vec_mode);
    6461        65089 :   if (icode == CODE_FOR_nothing)
    6462              :     {
    6463         6560 :       if (dump_enabled_p ())
    6464          972 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6465              :                          "shift op not supported by target.\n");
    6466              :       return false;
    6467              :     }
    6468              :   /* vector lowering cannot optimize vector shifts using word arithmetic.  */
    6469        58529 :   if (vect_emulated_vector_p (vectype))
    6470              :     return false;
    6471              : 
    6472        58529 :   if (cost_vec) /* transformation not required.  */
    6473              :     {
    6474        49297 :       if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
    6475        49297 :           || ((!scalar_shift_arg || dt[1] == vect_internal_def)
    6476         8201 :               && (!incompatible_op1_vectype_p
    6477            1 :                   || dt[1] == vect_constant_def)
    6478         8201 :               && !vect_maybe_update_slp_op_vectype
    6479         8201 :                          (slp_op1,
    6480              :                           incompatible_op1_vectype_p ? vectype : op1_vectype)))
    6481              :         {
    6482            0 :           if (dump_enabled_p ())
    6483            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6484              :                              "incompatible vector types for invariants\n");
    6485              :           return false;
    6486              :         }
    6487              :       /* Now adjust the constant shift amount in place.  */
    6488        49297 :       if (incompatible_op1_vectype_p
    6489            1 :           && dt[1] == vect_constant_def)
    6490              :         {
    6491            1 :           unsigned group_size = SLP_TREE_LANES (slp_op1);
    6492            2 :           gcc_assert (SLP_TREE_SCALAR_OPS (slp_op1).length () == group_size);
    6493            5 :           for (unsigned i = 0; i < group_size; ++i)
    6494              :             {
    6495            4 :               SLP_TREE_SCALAR_OPS (slp_op1)[i]
    6496            4 :                 = fold_convert (TREE_TYPE (vectype),
    6497              :                                 SLP_TREE_SCALAR_OPS (slp_op1)[i]);
    6498            4 :               gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
    6499              :                            == INTEGER_CST));
    6500              :             }
    6501              :         }
    6502        49297 :       SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
    6503        49297 :       DUMP_VECT_SCOPE ("vectorizable_shift");
    6504        49297 :       vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
    6505        49297 :       return true;
    6506              :     }
    6507              : 
    6508              :   /* Transform.  */
    6509              : 
    6510         9232 :   if (dump_enabled_p ())
    6511         2278 :     dump_printf_loc (MSG_NOTE, vect_location,
    6512              :                      "transform binary/unary operation.\n");
    6513              : 
    6514              :   /* Handle def.  */
    6515         9232 :   vec_dest = vect_create_destination_var (scalar_dest, vectype);
    6516              : 
    6517         9232 :   unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
    6518         9232 :   if (scalar_shift_arg && dt[1] != vect_internal_def)
    6519              :     {
    6520              :       /* Vector shl and shr insn patterns can be defined with scalar
    6521              :          operand 2 (shift operand).  In this case, use constant or loop
    6522              :          invariant op1 directly, without extending it to vector mode
    6523              :          first.  */
    6524         7047 :       optab_op2_mode = insn_data[icode].operand[2].mode;
    6525         7047 :       if (!VECTOR_MODE_P (optab_op2_mode))
    6526              :         {
    6527         7047 :           if (dump_enabled_p ())
    6528         2161 :             dump_printf_loc (MSG_NOTE, vect_location,
    6529              :                              "operand 1 using scalar mode.\n");
    6530         7047 :           vec_oprnd1 = op1;
    6531         7047 :           vec_oprnds1.create (nvectors);
    6532         7047 :           vec_oprnds1.quick_push (vec_oprnd1);
    6533              :           /* Store vec_oprnd1 for every vector stmt to be created.
    6534              :              We check during the analysis that all the shift arguments
    6535              :              are the same.
    6536              :              TODO: Allow different constants for different vector
    6537              :              stmts generated for an SLP instance.  */
    6538        16118 :           for (k = 0; k < nvectors - 1; k++)
    6539         2024 :             vec_oprnds1.quick_push (vec_oprnd1);
    6540              :         }
    6541              :     }
    6542         2185 :   else if (!scalar_shift_arg && incompatible_op1_vectype_p)
    6543              :     {
    6544            0 :       if (was_scalar_shift_arg)
    6545              :         {
    6546              :           /* If the argument was the same in all lanes create the
    6547              :              correctly typed vector shift amount directly.  Note
    6548              :              we made SLP scheduling think we use the original scalars,
    6549              :              so place the compensation code next to the shift which
    6550              :              is conservative.  See PR119640 where it otherwise breaks.  */
    6551            0 :           op1 = fold_convert (TREE_TYPE (vectype), op1);
    6552            0 :           op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
    6553              :                                   gsi);
    6554            0 :           vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
    6555              :                                          gsi);
    6556            0 :           vec_oprnds1.create (nvectors);
    6557            0 :           for (k = 0; k < nvectors; k++)
    6558            0 :             vec_oprnds1.quick_push (vec_oprnd1);
    6559              :         }
    6560            0 :       else if (dt[1] == vect_constant_def)
    6561              :         /* The constant shift amount has been adjusted in place.  */
    6562              :         ;
    6563              :       else
    6564            0 :         gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
    6565              :     }
    6566              : 
    6567              :   /* vec_oprnd1 is available if operand 1 should be of a scalar-type
    6568              :      (a special case for certain kind of vector shifts); otherwise,
    6569              :      operand 1 should be of a vector type (the usual case).  */
    6570         9232 :   vect_get_vec_defs (vinfo, slp_node,
    6571              :                      true, &vec_oprnds0, !vec_oprnd1, &vec_oprnds1);
    6572              : 
    6573              :   /* Arguments are ready.  Create the new vector stmt.  */
    6574        33254 :   FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
    6575              :     {
    6576              :       /* For internal defs where we need to use a scalar shift arg
    6577              :          extract the first lane.  */
    6578        14790 :       if (scalar_shift_arg && dt[1] == vect_internal_def)
    6579              :         {
    6580           12 :           vop1 = vec_oprnds1[0];
    6581           12 :           new_temp = make_ssa_name (TREE_TYPE (TREE_TYPE (vop1)));
    6582           12 :           gassign *new_stmt
    6583           12 :             = gimple_build_assign (new_temp,
    6584           12 :                                    build3 (BIT_FIELD_REF, TREE_TYPE (new_temp),
    6585              :                                            vop1,
    6586           12 :                                            TYPE_SIZE (TREE_TYPE (new_temp)),
    6587              :                                            bitsize_zero_node));
    6588           12 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    6589           12 :           vop1 = new_temp;
    6590           12 :         }
    6591              :       else
    6592        14778 :         vop1 = vec_oprnds1[i];
    6593        14790 :       gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
    6594        14790 :       new_temp = make_ssa_name (vec_dest, new_stmt);
    6595        14790 :       gimple_assign_set_lhs (new_stmt, new_temp);
    6596        14790 :       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    6597        14790 :       slp_node->push_vec_def (new_stmt);
    6598              :     }
    6599              : 
    6600         9232 :   vec_oprnds0.release ();
    6601         9232 :   vec_oprnds1.release ();
    6602              : 
    6603         9232 :   return true;
    6604              : }
    6605              : 
    6606              : /* Function vectorizable_operation.
    6607              : 
    6608              :    Check if STMT_INFO performs a binary, unary or ternary operation that can
    6609              :    be vectorized.
    6610              :    If COST_VEC is passed, calculate costs but don't change anything,
    6611              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    6612              :    it, and insert it at GSI.
    6613              :    Return true if STMT_INFO is vectorizable in this way.  */
    6614              : 
    6615              : static bool
    6616      2697036 : vectorizable_operation (vec_info *vinfo,
    6617              :                         stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    6618              :                         slp_tree slp_node,
    6619              :                         stmt_vector_for_cost *cost_vec)
    6620              : {
    6621      2697036 :   tree vec_dest;
    6622      2697036 :   tree scalar_dest;
    6623      2697036 :   tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
    6624      2697036 :   tree vectype;
    6625      2697036 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    6626      2697036 :   enum tree_code code, orig_code;
    6627      2697036 :   machine_mode vec_mode;
    6628      2697036 :   tree new_temp;
    6629      2697036 :   int op_type;
    6630      2697036 :   optab optab;
    6631      2697036 :   bool target_support_p;
    6632      2697036 :   enum vect_def_type dt[3]
    6633              :     = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
    6634      2697036 :   poly_uint64 nunits_in;
    6635      2697036 :   poly_uint64 nunits_out;
    6636      2697036 :   tree vectype_out;
    6637      2697036 :   int i;
    6638      2697036 :   vec<tree> vec_oprnds0 = vNULL;
    6639      2697036 :   vec<tree> vec_oprnds1 = vNULL;
    6640      2697036 :   vec<tree> vec_oprnds2 = vNULL;
    6641      2697036 :   tree vop0, vop1, vop2;
    6642      2697036 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    6643              : 
    6644      2697036 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    6645              :     return false;
    6646              : 
    6647      2697036 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    6648       251571 :       && cost_vec)
    6649              :     return false;
    6650              : 
    6651              :   /* Is STMT a vectorizable binary/unary operation?   */
    6652      2445465 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
    6653      2368689 :   if (!stmt)
    6654              :     return false;
    6655              : 
    6656              :   /* Loads and stores are handled in vectorizable_{load,store}.  */
    6657      2368689 :   if (STMT_VINFO_DATA_REF (stmt_info))
    6658              :     return false;
    6659              : 
    6660       950066 :   orig_code = code = gimple_assign_rhs_code (stmt);
    6661              : 
    6662              :   /* Shifts are handled in vectorizable_shift.  */
    6663       950066 :   if (code == LSHIFT_EXPR
    6664              :       || code == RSHIFT_EXPR
    6665              :       || code == LROTATE_EXPR
    6666       950066 :       || code == RROTATE_EXPR)
    6667              :    return false;
    6668              : 
    6669              :   /* Comparisons are handled in vectorizable_comparison.  */
    6670       894197 :   if (TREE_CODE_CLASS (code) == tcc_comparison)
    6671              :     return false;
    6672              : 
    6673              :   /* Conditions are handled in vectorizable_condition.  */
    6674       693983 :   if (code == COND_EXPR)
    6675              :     return false;
    6676              : 
    6677              :   /* For pointer addition and subtraction, we should use the normal
    6678              :      plus and minus for the vector operation.  */
    6679       666757 :   if (code == POINTER_PLUS_EXPR)
    6680              :     code = PLUS_EXPR;
    6681       644409 :   if (code == POINTER_DIFF_EXPR)
    6682         3382 :     code = MINUS_EXPR;
    6683              : 
    6684              :   /* Support only unary or binary operations.  */
    6685       666757 :   op_type = TREE_CODE_LENGTH (code);
    6686       666757 :   if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
    6687              :     {
    6688           15 :       if (dump_enabled_p ())
    6689            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6690              :                          "num. args = %d (not unary/binary/ternary op).\n",
    6691              :                          op_type);
    6692              :       return false;
    6693              :     }
    6694              : 
    6695       666742 :   scalar_dest = gimple_assign_lhs (stmt);
    6696       666742 :   vectype_out = SLP_TREE_VECTYPE (slp_node);
    6697              : 
    6698              :   /* Most operations cannot handle bit-precision types without extra
    6699              :      truncations.  */
    6700       666742 :   bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
    6701       655153 :   if (!mask_op_p
    6702       655153 :       && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
    6703              :       /* Exception are bitwise binary operations.  */
    6704              :       && code != BIT_IOR_EXPR
    6705         2048 :       && code != BIT_XOR_EXPR
    6706         1179 :       && code != BIT_AND_EXPR)
    6707              :     {
    6708          847 :       if (dump_enabled_p ())
    6709           54 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6710              :                          "bit-precision arithmetic not supported.\n");
    6711              :       return false;
    6712              :     }
    6713              : 
    6714       665895 :   slp_tree slp_op0;
    6715       665895 :   if (!vect_is_simple_use (vinfo, slp_node,
    6716              :                            0, &op0, &slp_op0, &dt[0], &vectype))
    6717              :     {
    6718            0 :       if (dump_enabled_p ())
    6719            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6720              :                          "use not simple.\n");
    6721              :       return false;
    6722              :     }
    6723       665895 :   bool is_invariant = (dt[0] == vect_external_def
    6724       665895 :                        || dt[0] == vect_constant_def);
    6725              :   /* If op0 is an external or constant def, infer the vector type
    6726              :      from the scalar type.  */
    6727       665895 :   if (!vectype)
    6728              :     {
    6729              :       /* For boolean type we cannot determine vectype by
    6730              :          invariant value (don't know whether it is a vector
    6731              :          of booleans or vector of integers).  We use output
    6732              :          vectype because operations on boolean don't change
    6733              :          type.  */
    6734        79058 :       if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
    6735              :         {
    6736         2168 :           if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
    6737              :             {
    6738          337 :               if (dump_enabled_p ())
    6739            0 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6740              :                                  "not supported operation on bool value.\n");
    6741              :               return false;
    6742              :             }
    6743         1831 :           vectype = vectype_out;
    6744              :         }
    6745              :       else
    6746        76890 :         vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
    6747              :                                                slp_node);
    6748              :     }
    6749       665558 :   if (!cost_vec)
    6750       119230 :     gcc_assert (vectype);
    6751       665558 :   if (!vectype)
    6752              :     {
    6753          312 :       if (dump_enabled_p ())
    6754            2 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6755              :                          "no vectype for scalar type %T\n",
    6756            2 :                          TREE_TYPE (op0));
    6757              : 
    6758              :       return false;
    6759              :     }
    6760              : 
    6761       665246 :   nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
    6762       665246 :   nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
    6763       665246 :   if (maybe_ne (nunits_out, nunits_in)
    6764       665246 :       || !tree_nop_conversion_p (TREE_TYPE (vectype_out), TREE_TYPE (vectype)))
    6765              :     return false;
    6766              : 
    6767       644087 :   tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
    6768       644087 :   slp_tree slp_op1 = NULL, slp_op2 = NULL;
    6769       644087 :   if (op_type == binary_op || op_type == ternary_op)
    6770              :     {
    6771       561391 :       if (!vect_is_simple_use (vinfo, slp_node,
    6772              :                                1, &op1, &slp_op1, &dt[1], &vectype2))
    6773              :         {
    6774            0 :           if (dump_enabled_p ())
    6775            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6776              :                              "use not simple.\n");
    6777              :           return false;
    6778              :         }
    6779       561391 :       is_invariant &= (dt[1] == vect_external_def
    6780       561391 :                        || dt[1] == vect_constant_def);
    6781       561391 :       if (vectype2
    6782       903169 :           && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype2))
    6783       341778 :               || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
    6784       341778 :                                          TREE_TYPE (vectype2))))
    6785              :         return false;
    6786              :     }
    6787       644087 :   if (op_type == ternary_op)
    6788              :     {
    6789            0 :       if (!vect_is_simple_use (vinfo, slp_node,
    6790              :                                2, &op2, &slp_op2, &dt[2], &vectype3))
    6791              :         {
    6792            0 :           if (dump_enabled_p ())
    6793            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6794              :                              "use not simple.\n");
    6795              :           return false;
    6796              :         }
    6797            0 :       is_invariant &= (dt[2] == vect_external_def
    6798            0 :                        || dt[2] == vect_constant_def);
    6799            0 :       if (vectype3
    6800            0 :           && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype3))
    6801            0 :               || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
    6802            0 :                                          TREE_TYPE (vectype3))))
    6803              :         return false;
    6804              :     }
    6805              : 
    6806              :   /* Multiple types in SLP are handled by creating the appropriate number of
    6807              :      vectorized stmts for each SLP node.  */
    6808       644087 :   auto vec_num = vect_get_num_copies (vinfo, slp_node);
    6809              : 
    6810              :   /* Reject attempts to combine mask types with nonmask types, e.g. if
    6811              :      we have an AND between a (nonmask) boolean loaded from memory and
    6812              :      a (mask) boolean result of a comparison.
    6813              : 
    6814              :      TODO: We could easily fix these cases up using pattern statements.  */
    6815       644087 :   if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
    6816       977642 :       || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
    6817      1288174 :       || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
    6818              :     {
    6819            0 :       if (dump_enabled_p ())
    6820            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6821              :                          "mixed mask and nonmask vector types\n");
    6822              :       return false;
    6823              :     }
    6824              : 
    6825              :   /* Supportable by target?  */
    6826              : 
    6827       644087 :   vec_mode = TYPE_MODE (vectype);
    6828       644087 :   optab = optab_for_tree_code (code, vectype, optab_default);
    6829       644087 :   if (!optab)
    6830              :     {
    6831        71207 :       if (dump_enabled_p ())
    6832         6157 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6833              :                          "no optab for %s and %T.\n",
    6834              :                          get_tree_code_name (code), vectype);
    6835              :       return false;
    6836              :     }
    6837       572880 :   target_support_p = can_implement_p (optab, vec_mode);
    6838              : 
    6839       572880 :   bool using_emulated_vectors_p = vect_emulated_vector_p (vectype);
    6840       572880 :   if (!target_support_p || using_emulated_vectors_p)
    6841              :     {
    6842        30682 :       if (dump_enabled_p ())
    6843         1302 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6844              :                          "op not supported by target.\n");
    6845              :       /* When vec_mode is not a vector mode and we verified ops we
    6846              :          do not have to lower like AND are natively supported let
    6847              :          those through even when the mode isn't word_mode.  For
    6848              :          ops we have to lower the lowering code assumes we are
    6849              :          dealing with word_mode.  */
    6850        61364 :       if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype))
    6851        30532 :           || !GET_MODE_SIZE (vec_mode).is_constant ()
    6852        30532 :           || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
    6853        25327 :                || !target_support_p)
    6854        65591 :               && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
    6855              :           /* Check only during analysis.  */
    6856        43021 :           || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
    6857              :         {
    6858        30080 :           if (dump_enabled_p ())
    6859         1300 :             dump_printf (MSG_NOTE, "using word mode not possible.\n");
    6860              :           return false;
    6861              :         }
    6862          602 :       if (dump_enabled_p ())
    6863            2 :         dump_printf_loc (MSG_NOTE, vect_location,
    6864              :                          "proceeding using word mode.\n");
    6865              :       using_emulated_vectors_p = true;
    6866              :     }
    6867              : 
    6868       542800 :   int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
    6869       542800 :   vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
    6870       438789 :   vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
    6871       542800 :   internal_fn cond_fn = get_conditional_internal_fn (code);
    6872       542800 :   internal_fn cond_len_fn = get_conditional_len_internal_fn (code);
    6873              : 
    6874              :   /* If operating on inactive elements could generate spurious traps,
    6875              :      we need to restrict the operation to active lanes.  Note that this
    6876              :      specifically doesn't apply to unhoisted invariants, since they
    6877              :      operate on the same value for every lane.
    6878              : 
    6879              :      Similarly, if this operation is part of a reduction, a fully-masked
    6880              :      loop should only change the active lanes of the reduction chain,
    6881              :      keeping the inactive lanes as-is.  */
    6882       513947 :   bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
    6883       987428 :                             || reduc_idx >= 0);
    6884              : 
    6885       542800 :   if (cost_vec) /* transformation not required.  */
    6886              :     {
    6887       423570 :       if (loop_vinfo
    6888       332367 :           && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    6889        92891 :           && mask_out_inactive)
    6890              :         {
    6891        19983 :           if (cond_len_fn != IFN_LAST
    6892        19983 :               && direct_internal_fn_supported_p (cond_len_fn, vectype,
    6893              :                                                  OPTIMIZE_FOR_SPEED))
    6894            0 :             vect_record_loop_len (loop_vinfo, lens, vec_num, vectype,
    6895              :                                   1);
    6896        19983 :           else if (cond_fn != IFN_LAST
    6897        19983 :                    && direct_internal_fn_supported_p (cond_fn, vectype,
    6898              :                                                       OPTIMIZE_FOR_SPEED))
    6899         8182 :             vect_record_loop_mask (loop_vinfo, masks, vec_num,
    6900              :                                    vectype, NULL);
    6901              :           else
    6902              :             {
    6903        11801 :               if (dump_enabled_p ())
    6904          613 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6905              :                                  "can't use a fully-masked loop because no"
    6906              :                                  " conditional operation is available.\n");
    6907        11801 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    6908              :             }
    6909              :         }
    6910              : 
    6911              :       /* Put types on constant and invariant SLP children.  */
    6912       423570 :       if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
    6913       423502 :           || !vect_maybe_update_slp_op_vectype (slp_op1, vectype)
    6914       846978 :           || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))
    6915              :         {
    6916          162 :           if (dump_enabled_p ())
    6917            4 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6918              :                              "incompatible vector types for invariants\n");
    6919              :           return false;
    6920              :         }
    6921              : 
    6922       423408 :       SLP_TREE_TYPE (slp_node) = op_vec_info_type;
    6923       423408 :       DUMP_VECT_SCOPE ("vectorizable_operation");
    6924       423408 :       vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
    6925       423408 :       if (using_emulated_vectors_p)
    6926              :         {
    6927              :           /* The above vect_model_simple_cost call handles constants
    6928              :              in the prologue and (mis-)costs one of the stmts as
    6929              :              vector stmt.  See below for the actual lowering that will
    6930              :              be applied.  */
    6931          600 :           unsigned n = vect_get_num_copies (vinfo, slp_node);
    6932          600 :           switch (code)
    6933              :             {
    6934          217 :             case PLUS_EXPR:
    6935          217 :               n *= 5;
    6936          217 :               break;
    6937          346 :             case MINUS_EXPR:
    6938          346 :               n *= 6;
    6939          346 :               break;
    6940            0 :             case NEGATE_EXPR:
    6941            0 :               n *= 4;
    6942            0 :               break;
    6943              :             default:
    6944              :               /* Bit operations do not have extra cost and are accounted
    6945              :                  as vector stmt by vect_model_simple_cost.  */
    6946              :               n = 0;
    6947              :               break;
    6948              :             }
    6949          563 :           if (n != 0)
    6950              :             {
    6951              :               /* We also need to materialize two large constants.  */
    6952          563 :               record_stmt_cost (cost_vec, 2, scalar_stmt, slp_node,
    6953              :                                 0, vect_prologue);
    6954          563 :               record_stmt_cost (cost_vec, n, scalar_stmt, slp_node,
    6955              :                                 0, vect_body);
    6956              :             }
    6957              :         }
    6958       423408 :       return true;
    6959              :     }
    6960              : 
    6961              :   /* Transform.  */
    6962              : 
    6963       119230 :   if (dump_enabled_p ())
    6964        17516 :     dump_printf_loc (MSG_NOTE, vect_location,
    6965              :                      "transform binary/unary operation.\n");
    6966              : 
    6967       119230 :   bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
    6968       106422 :   bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
    6969              : 
    6970              :   /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
    6971              :      vectors with unsigned elements, but the result is signed.  So, we
    6972              :      need to compute the MINUS_EXPR into vectype temporary and
    6973              :      VIEW_CONVERT_EXPR it into the final vectype_out result.  */
    6974       119230 :   tree vec_cvt_dest = NULL_TREE;
    6975       119230 :   if (orig_code == POINTER_DIFF_EXPR)
    6976              :     {
    6977          194 :       vec_dest = vect_create_destination_var (scalar_dest, vectype);
    6978          194 :       vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
    6979              :     }
    6980              :   /* For reduction operations with undefined overflow behavior make sure to
    6981              :      pun them to unsigned since we change the order of evaluation.
    6982              :      ???  Avoid for in-order reductions?  */
    6983       119036 :   else if (arith_code_with_undefined_signed_overflow (orig_code)
    6984       100162 :            && ANY_INTEGRAL_TYPE_P (vectype)
    6985        48804 :            && TYPE_OVERFLOW_UNDEFINED (vectype)
    6986       145129 :            && SLP_TREE_REDUC_IDX (slp_node) != -1)
    6987              :     {
    6988         2489 :       gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR
    6989              :                   || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR);
    6990         2489 :       vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
    6991         2489 :       vectype = unsigned_type_for (vectype);
    6992         2489 :       vec_dest = vect_create_destination_var (scalar_dest, vectype);
    6993              :     }
    6994              :   /* Handle def.  */
    6995              :   else
    6996       116547 :     vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
    6997              : 
    6998       119230 :   vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
    6999              :                      slp_op1, &vec_oprnds1, slp_op2, &vec_oprnds2);
    7000              :   /* Arguments are ready.  Create the new vector stmt.  */
    7001       381987 :   FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
    7002              :     {
    7003       143527 :       gimple *new_stmt = NULL;
    7004       287054 :       vop1 = ((op_type == binary_op || op_type == ternary_op)
    7005       143527 :               ? vec_oprnds1[i] : NULL_TREE);
    7006       143527 :       vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
    7007              : 
    7008       143527 :       if (vec_cvt_dest
    7009       143527 :           && !useless_type_conversion_p (vectype, TREE_TYPE (vop0)))
    7010              :         {
    7011         2946 :           new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0);
    7012         2946 :           new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
    7013              :                                           new_temp);
    7014         2946 :           new_temp = make_ssa_name (vec_dest, new_stmt);
    7015         2946 :           gimple_assign_set_lhs (new_stmt, new_temp);
    7016         2946 :           vect_finish_stmt_generation (vinfo, stmt_info,
    7017              :                                        new_stmt, gsi);
    7018         2946 :           vop0 = new_temp;
    7019              :         }
    7020       143527 :       if (vop1
    7021       140845 :           && vec_cvt_dest
    7022       146682 :           && !useless_type_conversion_p (vectype, TREE_TYPE (vop1)))
    7023              :         {
    7024         2946 :           new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1);
    7025         2946 :           new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
    7026              :                                           new_temp);
    7027         2946 :           new_temp = make_ssa_name (vec_dest, new_stmt);
    7028         2946 :           gimple_assign_set_lhs (new_stmt, new_temp);
    7029         2946 :           vect_finish_stmt_generation (vinfo, stmt_info,
    7030              :                                        new_stmt, gsi);
    7031         2946 :           vop1 = new_temp;
    7032              :         }
    7033       143527 :       if (vop2
    7034            0 :           && vec_cvt_dest
    7035       143527 :           && !useless_type_conversion_p (vectype, TREE_TYPE (vop2)))
    7036              :         {
    7037            0 :           new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2);
    7038            0 :           new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
    7039              :                                           new_temp);
    7040            0 :           new_temp = make_ssa_name (vec_dest, new_stmt);
    7041            0 :           gimple_assign_set_lhs (new_stmt, new_temp);
    7042            0 :           vect_finish_stmt_generation (vinfo, stmt_info,
    7043              :                                        new_stmt, gsi);
    7044            0 :           vop2 = new_temp;
    7045              :         }
    7046              : 
    7047       143527 :       if (using_emulated_vectors_p)
    7048              :         {
    7049              :           /* Lower the operation.  This follows vector lowering.  */
    7050            2 :           tree word_type = build_nonstandard_integer_type
    7051            2 :                              (GET_MODE_BITSIZE (vec_mode).to_constant (), 1);
    7052            2 :           tree wvop0 = make_ssa_name (word_type);
    7053            2 :           new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR,
    7054              :                                           build1 (VIEW_CONVERT_EXPR,
    7055              :                                                   word_type, vop0));
    7056            2 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7057            2 :           tree wvop1 = NULL_TREE;
    7058            2 :           if (vop1)
    7059              :             {
    7060            2 :               wvop1 = make_ssa_name (word_type);
    7061            2 :               new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR,
    7062              :                                               build1 (VIEW_CONVERT_EXPR,
    7063              :                                                       word_type, vop1));
    7064            2 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7065              :             }
    7066              : 
    7067            2 :           tree result_low;
    7068            2 :           if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
    7069              :             {
    7070            1 :               unsigned int width = vector_element_bits (vectype);
    7071            1 :               tree inner_type = TREE_TYPE (vectype);
    7072            1 :               HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type));
    7073            1 :               tree low_bits
    7074            1 :                 = build_replicated_int_cst (word_type, width, max >> 1);
    7075            1 :               tree high_bits
    7076            2 :                 = build_replicated_int_cst (word_type,
    7077            1 :                                             width, max & ~(max >> 1));
    7078            1 :               tree signs;
    7079            1 :               if (code == PLUS_EXPR || code == MINUS_EXPR)
    7080              :                 {
    7081            1 :                   signs = make_ssa_name (word_type);
    7082            1 :                   new_stmt = gimple_build_assign (signs,
    7083              :                                                   BIT_XOR_EXPR, wvop0, wvop1);
    7084            1 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7085            1 :                   tree b_low = make_ssa_name (word_type);
    7086            1 :                   new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR,
    7087              :                                                   wvop1, low_bits);
    7088            1 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7089            1 :                   tree a_low = make_ssa_name (word_type);
    7090            1 :                   if (code == PLUS_EXPR)
    7091            1 :                     new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
    7092              :                                                     wvop0, low_bits);
    7093              :                   else
    7094            0 :                     new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR,
    7095              :                                                     wvop0, high_bits);
    7096            1 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7097            1 :                   if (code == MINUS_EXPR)
    7098              :                     {
    7099            0 :                       new_stmt = gimple_build_assign (NULL_TREE,
    7100              :                                                       BIT_NOT_EXPR, signs);
    7101            0 :                       signs = make_ssa_name (word_type);
    7102            0 :                       gimple_assign_set_lhs (new_stmt, signs);
    7103            0 :                       vect_finish_stmt_generation (vinfo, stmt_info,
    7104              :                                                    new_stmt, gsi);
    7105              :                     }
    7106            1 :                   new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
    7107              :                                                   signs, high_bits);
    7108            1 :                   signs = make_ssa_name (word_type);
    7109            1 :                   gimple_assign_set_lhs (new_stmt, signs);
    7110            1 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7111            1 :                   result_low = make_ssa_name (word_type);
    7112            1 :                   new_stmt = gimple_build_assign (result_low, code,
    7113              :                                                   a_low, b_low);
    7114            1 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7115              :                 }
    7116              :               else /* if (code == NEGATE_EXPR) */
    7117              :                 {
    7118            0 :                   tree a_low = make_ssa_name (word_type);
    7119            0 :                   new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
    7120              :                                                   wvop0, low_bits);
    7121            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7122            0 :                   signs = make_ssa_name (word_type);
    7123            0 :                   new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0);
    7124            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7125            0 :                   new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
    7126              :                                                   signs, high_bits);
    7127            0 :                   signs = make_ssa_name (word_type);
    7128            0 :                   gimple_assign_set_lhs (new_stmt, signs);
    7129            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7130            0 :                   result_low = make_ssa_name (word_type);
    7131            0 :                   new_stmt = gimple_build_assign (result_low,
    7132              :                                                   MINUS_EXPR, high_bits, a_low);
    7133            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7134              :                 }
    7135            1 :               new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR,
    7136              :                                               result_low, signs);
    7137            1 :               result_low = make_ssa_name (word_type);
    7138            1 :               gimple_assign_set_lhs (new_stmt, result_low);
    7139            1 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7140              :             }
    7141              :           else
    7142              :             {
    7143            1 :               new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1);
    7144            1 :               result_low = make_ssa_name (word_type);
    7145            1 :               gimple_assign_set_lhs (new_stmt, result_low);
    7146            1 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7147              : 
    7148              :             }
    7149            2 :           new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR,
    7150              :                                           build1 (VIEW_CONVERT_EXPR,
    7151              :                                                   vectype, result_low));
    7152            2 :           new_temp = make_ssa_name (vectype);
    7153            2 :           gimple_assign_set_lhs (new_stmt, new_temp);
    7154            2 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7155              :         }
    7156       143525 :       else if ((masked_loop_p || len_loop_p) && mask_out_inactive)
    7157              :         {
    7158           16 :           tree mask;
    7159           16 :           if (masked_loop_p)
    7160           16 :             mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    7161              :                                        vec_num, vectype, i);
    7162              :           else
    7163              :             /* Dummy mask.  */
    7164            0 :             mask = build_minus_one_cst (truth_type_for (vectype));
    7165           16 :           auto_vec<tree> vops (6);
    7166           16 :           vops.quick_push (mask);
    7167           16 :           vops.quick_push (vop0);
    7168           16 :           if (vop1)
    7169           16 :             vops.quick_push (vop1);
    7170           16 :           if (vop2)
    7171            0 :             vops.quick_push (vop2);
    7172           16 :           if (reduc_idx >= 0)
    7173              :             {
    7174              :               /* Perform the operation on active elements only and take
    7175              :                  inactive elements from the reduction chain input.  */
    7176            8 :               gcc_assert (!vop2);
    7177            8 :               vops.quick_push (reduc_idx == 1 ? vop1 : vop0);
    7178              :             }
    7179              :           else
    7180              :             {
    7181            8 :               auto else_value = targetm.preferred_else_value
    7182            8 :                 (cond_fn, vectype, vops.length () - 1, &vops[1]);
    7183            8 :               vops.quick_push (else_value);
    7184              :             }
    7185           16 :           if (len_loop_p)
    7186              :             {
    7187            0 :               tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
    7188            0 :                                             vec_num, vectype, i, 1, true);
    7189            0 :               signed char biasval
    7190            0 :                 = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    7191            0 :               tree bias = build_int_cst (intQI_type_node, biasval);
    7192            0 :               vops.quick_push (len);
    7193            0 :               vops.quick_push (bias);
    7194              :             }
    7195           16 :           gcall *call
    7196           16 :             = gimple_build_call_internal_vec (masked_loop_p ? cond_fn
    7197              :                                                             : cond_len_fn,
    7198              :                                               vops);
    7199           16 :           new_temp = make_ssa_name (vec_dest, call);
    7200           16 :           gimple_call_set_lhs (call, new_temp);
    7201           16 :           gimple_call_set_nothrow (call, true);
    7202           16 :           vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    7203           16 :           new_stmt = call;
    7204           16 :         }
    7205              :       else
    7206              :         {
    7207       143509 :           tree mask = NULL_TREE;
    7208              :           /* When combining two masks check if either of them is elsewhere
    7209              :              combined with a loop mask, if that's the case we can mark that the
    7210              :              new combined mask doesn't need to be combined with a loop mask.  */
    7211       143509 :           if (masked_loop_p
    7212       143509 :               && code == BIT_AND_EXPR
    7213       143509 :               && VECTOR_BOOLEAN_TYPE_P (vectype))
    7214              :             {
    7215            8 :               if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
    7216              :                 {
    7217            0 :                   mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    7218              :                                              vec_num, vectype, i);
    7219              : 
    7220            0 :                   vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
    7221              :                                            vop0, gsi);
    7222              :                 }
    7223              : 
    7224            8 :               if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
    7225              :                 {
    7226            0 :                   mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    7227              :                                              vec_num, vectype, i);
    7228              : 
    7229            0 :                   vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
    7230              :                                            vop1, gsi);
    7231              :                 }
    7232              :             }
    7233              : 
    7234       143509 :           new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
    7235       143509 :           new_temp = make_ssa_name (vec_dest, new_stmt);
    7236       143509 :           gimple_assign_set_lhs (new_stmt, new_temp);
    7237       143509 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    7238       143509 :           if (using_emulated_vectors_p)
    7239              :             suppress_warning (new_stmt, OPT_Wvector_operation_performance);
    7240              : 
    7241              :           /* Enter the combined value into the vector cond hash so we don't
    7242              :              AND it with a loop mask again.  */
    7243       143509 :           if (mask)
    7244            0 :             loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
    7245              :         }
    7246              : 
    7247       143527 :       if (vec_cvt_dest)
    7248              :         {
    7249         3155 :           new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
    7250         3155 :           new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
    7251              :                                           new_temp);
    7252         3155 :           new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
    7253         3155 :           gimple_assign_set_lhs (new_stmt, new_temp);
    7254         3155 :           vect_finish_stmt_generation (vinfo, stmt_info,
    7255              :                                        new_stmt, gsi);
    7256              :         }
    7257              : 
    7258       143527 :       slp_node->push_vec_def (new_stmt);
    7259              :     }
    7260              : 
    7261       119230 :   vec_oprnds0.release ();
    7262       119230 :   vec_oprnds1.release ();
    7263       119230 :   vec_oprnds2.release ();
    7264              : 
    7265       119230 :   return true;
    7266              : }
    7267              : 
    7268              : /* A helper function to ensure data reference DR_INFO's base alignment.  */
    7269              : 
    7270              : static void
    7271       708178 : ensure_base_align (dr_vec_info *dr_info)
    7272              : {
    7273              :   /* Alignment is only analyzed for the first element of a DR group,
    7274              :      use that to look at base alignment we need to enforce.  */
    7275       708178 :   if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
    7276       583582 :     dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
    7277              : 
    7278       708178 :   gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
    7279              : 
    7280       708178 :   if (dr_info->base_misaligned)
    7281              :     {
    7282       113568 :       tree base_decl = dr_info->base_decl;
    7283              : 
    7284              :       // We should only be able to increase the alignment of a base object if
    7285              :       // we know what its new alignment should be at compile time.
    7286       113568 :       unsigned HOST_WIDE_INT align_base_to =
    7287       113568 :         DR_TARGET_ALIGNMENT (dr_info).to_constant () * BITS_PER_UNIT;
    7288              : 
    7289       113568 :       if (decl_in_symtab_p (base_decl))
    7290         3261 :         symtab_node::get (base_decl)->increase_alignment (align_base_to);
    7291       110307 :       else if (DECL_ALIGN (base_decl) < align_base_to)
    7292              :         {
    7293        91975 :           SET_DECL_ALIGN (base_decl, align_base_to);
    7294        91975 :           DECL_USER_ALIGN (base_decl) = 1;
    7295              :         }
    7296       113568 :       dr_info->base_misaligned = false;
    7297              :     }
    7298       708178 : }
    7299              : 
    7300              : 
    7301              : /* Function get_group_alias_ptr_type.
    7302              : 
    7303              :    Return the alias type for the group starting at FIRST_STMT_INFO.  */
    7304              : 
    7305              : static tree
    7306      1659002 : get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
    7307              : {
    7308      1659002 :   struct data_reference *first_dr, *next_dr;
    7309              : 
    7310      1659002 :   first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
    7311      1659002 :   stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
    7312      3927520 :   while (next_stmt_info)
    7313              :     {
    7314      2456965 :       next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
    7315      4913930 :       if (get_alias_set (DR_REF (first_dr))
    7316      2456965 :           != get_alias_set (DR_REF (next_dr)))
    7317              :         {
    7318       188447 :           if (dump_enabled_p ())
    7319           30 :             dump_printf_loc (MSG_NOTE, vect_location,
    7320              :                              "conflicting alias set types.\n");
    7321       188447 :           return ptr_type_node;
    7322              :         }
    7323      2268518 :       next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
    7324              :     }
    7325      1470555 :   return reference_alias_ptr_type (DR_REF (first_dr));
    7326              : }
    7327              : 
    7328              : 
    7329              : /* Function scan_operand_equal_p.
    7330              : 
    7331              :    Helper function for check_scan_store.  Compare two references
    7332              :    with .GOMP_SIMD_LANE bases.  */
    7333              : 
    7334              : static bool
    7335         1284 : scan_operand_equal_p (tree ref1, tree ref2)
    7336              : {
    7337         1284 :   tree ref[2] = { ref1, ref2 };
    7338         1284 :   poly_int64 bitsize[2], bitpos[2];
    7339              :   tree offset[2], base[2];
    7340         3852 :   for (int i = 0; i < 2; ++i)
    7341              :     {
    7342         2568 :       machine_mode mode;
    7343         2568 :       int unsignedp, reversep, volatilep = 0;
    7344         2568 :       base[i] = get_inner_reference (ref[i], &bitsize[i], &bitpos[i],
    7345              :                                      &offset[i], &mode, &unsignedp,
    7346              :                                      &reversep, &volatilep);
    7347         2568 :       if (reversep || volatilep || maybe_ne (bitpos[i], 0))
    7348            0 :         return false;
    7349         2568 :       if (TREE_CODE (base[i]) == MEM_REF
    7350           42 :           && offset[i] == NULL_TREE
    7351         2610 :           && TREE_CODE (TREE_OPERAND (base[i], 0)) == SSA_NAME)
    7352              :         {
    7353           42 :           gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base[i], 0));
    7354           42 :           if (is_gimple_assign (def_stmt)
    7355           42 :               && gimple_assign_rhs_code (def_stmt) == POINTER_PLUS_EXPR
    7356           42 :               && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == ADDR_EXPR
    7357           84 :               && TREE_CODE (gimple_assign_rhs2 (def_stmt)) == SSA_NAME)
    7358              :             {
    7359           42 :               if (maybe_ne (mem_ref_offset (base[i]), 0))
    7360              :                 return false;
    7361           42 :               base[i] = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
    7362           42 :               offset[i] = gimple_assign_rhs2 (def_stmt);
    7363              :             }
    7364              :         }
    7365              :     }
    7366              : 
    7367         1284 :   if (!operand_equal_p (base[0], base[1], 0))
    7368              :     return false;
    7369          934 :   if (maybe_ne (bitsize[0], bitsize[1]))
    7370              :     return false;
    7371          934 :   if (offset[0] != offset[1])
    7372              :     {
    7373          916 :       if (!offset[0] || !offset[1])
    7374              :         return false;
    7375          916 :       if (!operand_equal_p (offset[0], offset[1], 0))
    7376              :         {
    7377              :           tree step[2];
    7378            0 :           for (int i = 0; i < 2; ++i)
    7379              :             {
    7380            0 :               step[i] = integer_one_node;
    7381            0 :               if (TREE_CODE (offset[i]) == SSA_NAME)
    7382              :                 {
    7383            0 :                   gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
    7384            0 :                   if (is_gimple_assign (def_stmt)
    7385            0 :                       && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
    7386            0 :                       && (TREE_CODE (gimple_assign_rhs2 (def_stmt))
    7387              :                           == INTEGER_CST))
    7388              :                     {
    7389            0 :                       step[i] = gimple_assign_rhs2 (def_stmt);
    7390            0 :                       offset[i] = gimple_assign_rhs1 (def_stmt);
    7391              :                     }
    7392              :                 }
    7393            0 :               else if (TREE_CODE (offset[i]) == MULT_EXPR)
    7394              :                 {
    7395            0 :                   step[i] = TREE_OPERAND (offset[i], 1);
    7396            0 :                   offset[i] = TREE_OPERAND (offset[i], 0);
    7397              :                 }
    7398            0 :               tree rhs1 = NULL_TREE;
    7399            0 :               if (TREE_CODE (offset[i]) == SSA_NAME)
    7400              :                 {
    7401            0 :                   gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
    7402            0 :                   if (gimple_assign_cast_p (def_stmt))
    7403            0 :                     rhs1 = gimple_assign_rhs1 (def_stmt);
    7404              :                 }
    7405            0 :               else if (CONVERT_EXPR_P (offset[i]))
    7406            0 :                 rhs1 = TREE_OPERAND (offset[i], 0);
    7407            0 :               if (rhs1
    7408            0 :                   && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    7409            0 :                   && INTEGRAL_TYPE_P (TREE_TYPE (offset[i]))
    7410            0 :                   && (TYPE_PRECISION (TREE_TYPE (offset[i]))
    7411            0 :                       >= TYPE_PRECISION (TREE_TYPE (rhs1))))
    7412            0 :                 offset[i] = rhs1;
    7413              :             }
    7414            0 :           if (!operand_equal_p (offset[0], offset[1], 0)
    7415            0 :               || !operand_equal_p (step[0], step[1], 0))
    7416            0 :             return false;
    7417              :         }
    7418              :     }
    7419              :   return true;
    7420              : }
    7421              : 
    7422              : 
    7423              : enum scan_store_kind {
    7424              :   /* Normal permutation.  */
    7425              :   scan_store_kind_perm,
    7426              : 
    7427              :   /* Whole vector left shift permutation with zero init.  */
    7428              :   scan_store_kind_lshift_zero,
    7429              : 
    7430              :   /* Whole vector left shift permutation and VEC_COND_EXPR.  */
    7431              :   scan_store_kind_lshift_cond
    7432              : };
    7433              : 
    7434              : /* Function check_scan_store.
    7435              : 
    7436              :    Verify if we can perform the needed permutations or whole vector shifts.
    7437              :    Return -1 on failure, otherwise exact log2 of vectype's nunits.
    7438              :    USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
    7439              :    to do at each step.  */
    7440              : 
    7441              : static int
    7442         1024 : scan_store_can_perm_p (tree vectype, tree init,
    7443              :                        vec<enum scan_store_kind> *use_whole_vector = NULL)
    7444              : {
    7445         1024 :   enum machine_mode vec_mode = TYPE_MODE (vectype);
    7446         1024 :   unsigned HOST_WIDE_INT nunits;
    7447         1024 :   if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
    7448              :     return -1;
    7449         1024 :   int units_log2 = exact_log2 (nunits);
    7450         1024 :   if (units_log2 <= 0)
    7451              :     return -1;
    7452              : 
    7453              :   int i;
    7454              :   enum scan_store_kind whole_vector_shift_kind = scan_store_kind_perm;
    7455         4784 :   for (i = 0; i <= units_log2; ++i)
    7456              :     {
    7457         3760 :       unsigned HOST_WIDE_INT j, k;
    7458         3760 :       enum scan_store_kind kind = scan_store_kind_perm;
    7459         3760 :       vec_perm_builder sel (nunits, nunits, 1);
    7460         3760 :       sel.quick_grow (nunits);
    7461         3760 :       if (i == units_log2)
    7462              :         {
    7463         9728 :           for (j = 0; j < nunits; ++j)
    7464         8704 :             sel[j] = nunits - 1;
    7465              :         }
    7466              :       else
    7467              :         {
    7468        10416 :           for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
    7469         7680 :             sel[j] = j;
    7470        26416 :           for (k = 0; j < nunits; ++j, ++k)
    7471        23680 :             sel[j] = nunits + k;
    7472              :         }
    7473         6496 :       vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
    7474         3760 :       if (!can_vec_perm_const_p (vec_mode, vec_mode, indices))
    7475              :         {
    7476            0 :           if (i == units_log2)
    7477              :             return -1;
    7478              : 
    7479            0 :           if (whole_vector_shift_kind == scan_store_kind_perm)
    7480              :             {
    7481            0 :               if (!can_implement_p (vec_shl_optab, vec_mode))
    7482              :                 return -1;
    7483            0 :               whole_vector_shift_kind = scan_store_kind_lshift_zero;
    7484              :               /* Whole vector shifts shift in zeros, so if init is all zero
    7485              :                  constant, there is no need to do anything further.  */
    7486            0 :               if ((TREE_CODE (init) != INTEGER_CST
    7487            0 :                    && TREE_CODE (init) != REAL_CST)
    7488            0 :                   || !initializer_zerop (init))
    7489              :                 {
    7490            0 :                   tree masktype = truth_type_for (vectype);
    7491            0 :                   if (!expand_vec_cond_expr_p (vectype, masktype))
    7492              :                     return -1;
    7493              :                   whole_vector_shift_kind = scan_store_kind_lshift_cond;
    7494              :                 }
    7495              :             }
    7496            0 :           kind = whole_vector_shift_kind;
    7497              :         }
    7498         3760 :       if (use_whole_vector)
    7499              :         {
    7500         1880 :           if (kind != scan_store_kind_perm && use_whole_vector->is_empty ())
    7501            0 :             use_whole_vector->safe_grow_cleared (i, true);
    7502         5640 :           if (kind != scan_store_kind_perm || !use_whole_vector->is_empty ())
    7503            0 :             use_whole_vector->safe_push (kind);
    7504              :         }
    7505         3760 :     }
    7506              : 
    7507              :   return units_log2;
    7508              : }
    7509              : 
    7510              : 
    7511              : /* Function check_scan_store.
    7512              : 
    7513              :    Check magic stores for #pragma omp scan {in,ex}clusive reductions.  */
    7514              : 
    7515              : static bool
    7516         1076 : check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
    7517              :                   enum vect_def_type rhs_dt, slp_tree slp_node,
    7518              :                   slp_tree mask_node,
    7519              :                   vect_memory_access_type memory_access_type)
    7520              : {
    7521         1076 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    7522         1076 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    7523         1076 :   tree ref_type;
    7524              : 
    7525         1076 :   gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
    7526         1076 :   if (SLP_TREE_LANES (slp_node) > 1
    7527         1076 :       || mask_node
    7528         1076 :       || memory_access_type != VMAT_CONTIGUOUS
    7529         1076 :       || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
    7530         1076 :       || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
    7531         1076 :       || loop_vinfo == NULL
    7532         1076 :       || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
    7533         1076 :       || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    7534         1076 :       || STMT_VINFO_GROUPED_ACCESS (stmt_info)
    7535         1076 :       || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
    7536         1076 :       || !integer_zerop (DR_INIT (dr_info->dr))
    7537         1076 :       || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
    7538         2152 :       || !alias_sets_conflict_p (get_alias_set (vectype),
    7539         1076 :                                  get_alias_set (TREE_TYPE (ref_type))))
    7540              :     {
    7541            0 :       if (dump_enabled_p ())
    7542            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7543              :                          "unsupported OpenMP scan store.\n");
    7544              :       return false;
    7545              :     }
    7546              : 
    7547              :   /* We need to pattern match code built by OpenMP lowering and simplified
    7548              :      by following optimizations into something we can handle.
    7549              :      #pragma omp simd reduction(inscan,+:r)
    7550              :      for (...)
    7551              :        {
    7552              :          r += something ();
    7553              :          #pragma omp scan inclusive (r)
    7554              :          use (r);
    7555              :        }
    7556              :      shall have body with:
    7557              :        // Initialization for input phase, store the reduction initializer:
    7558              :        _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
    7559              :        _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
    7560              :        D.2042[_21] = 0;
    7561              :        // Actual input phase:
    7562              :        ...
    7563              :        r.0_5 = D.2042[_20];
    7564              :        _6 = _4 + r.0_5;
    7565              :        D.2042[_20] = _6;
    7566              :        // Initialization for scan phase:
    7567              :        _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
    7568              :        _26 = D.2043[_25];
    7569              :        _27 = D.2042[_25];
    7570              :        _28 = _26 + _27;
    7571              :        D.2043[_25] = _28;
    7572              :        D.2042[_25] = _28;
    7573              :        // Actual scan phase:
    7574              :        ...
    7575              :        r.1_8 = D.2042[_20];
    7576              :        ...
    7577              :      The "omp simd array" variable D.2042 holds the privatized copy used
    7578              :      inside of the loop and D.2043 is another one that holds copies of
    7579              :      the current original list item.  The separate GOMP_SIMD_LANE ifn
    7580              :      kinds are there in order to allow optimizing the initializer store
    7581              :      and combiner sequence, e.g. if it is originally some C++ish user
    7582              :      defined reduction, but allow the vectorizer to pattern recognize it
    7583              :      and turn into the appropriate vectorized scan.
    7584              : 
    7585              :      For exclusive scan, this is slightly different:
    7586              :      #pragma omp simd reduction(inscan,+:r)
    7587              :      for (...)
    7588              :        {
    7589              :          use (r);
    7590              :          #pragma omp scan exclusive (r)
    7591              :          r += something ();
    7592              :        }
    7593              :      shall have body with:
    7594              :        // Initialization for input phase, store the reduction initializer:
    7595              :        _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
    7596              :        _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
    7597              :        D.2042[_21] = 0;
    7598              :        // Actual input phase:
    7599              :        ...
    7600              :        r.0_5 = D.2042[_20];
    7601              :        _6 = _4 + r.0_5;
    7602              :        D.2042[_20] = _6;
    7603              :        // Initialization for scan phase:
    7604              :        _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
    7605              :        _26 = D.2043[_25];
    7606              :        D.2044[_25] = _26;
    7607              :        _27 = D.2042[_25];
    7608              :        _28 = _26 + _27;
    7609              :        D.2043[_25] = _28;
    7610              :        // Actual scan phase:
    7611              :        ...
    7612              :        r.1_8 = D.2044[_20];
    7613              :        ...  */
    7614              : 
    7615         1076 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 2)
    7616              :     {
    7617              :       /* Match the D.2042[_21] = 0; store above.  Just require that
    7618              :          it is a constant or external definition store.  */
    7619          564 :       if (rhs_dt != vect_constant_def && rhs_dt != vect_external_def)
    7620              :         {
    7621            0 :          fail_init:
    7622            0 :           if (dump_enabled_p ())
    7623            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7624              :                              "unsupported OpenMP scan initializer store.\n");
    7625              :           return false;
    7626              :         }
    7627              : 
    7628          564 :       if (! loop_vinfo->scan_map)
    7629          322 :         loop_vinfo->scan_map = new hash_map<tree, tree>;
    7630          564 :       tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
    7631          564 :       tree &cached = loop_vinfo->scan_map->get_or_insert (var);
    7632          564 :       if (cached)
    7633            0 :         goto fail_init;
    7634          564 :       cached = gimple_assign_rhs1 (STMT_VINFO_STMT (stmt_info));
    7635              : 
    7636              :       /* These stores can be vectorized normally.  */
    7637          564 :       return true;
    7638              :     }
    7639              : 
    7640          512 :   if (rhs_dt != vect_internal_def)
    7641              :     {
    7642            0 :      fail:
    7643            0 :       if (dump_enabled_p ())
    7644            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7645              :                          "unsupported OpenMP scan combiner pattern.\n");
    7646              :       return false;
    7647              :     }
    7648              : 
    7649          512 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
    7650          512 :   tree rhs = gimple_assign_rhs1 (stmt);
    7651          512 :   if (TREE_CODE (rhs) != SSA_NAME)
    7652            0 :     goto fail;
    7653              : 
    7654          512 :   gimple *other_store_stmt = NULL;
    7655          512 :   tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
    7656          512 :   bool inscan_var_store
    7657          512 :     = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
    7658              : 
    7659          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
    7660              :     {
    7661          252 :       if (!inscan_var_store)
    7662              :         {
    7663          126 :           use_operand_p use_p;
    7664          126 :           imm_use_iterator iter;
    7665          378 :           FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
    7666              :             {
    7667          252 :               gimple *use_stmt = USE_STMT (use_p);
    7668          252 :               if (use_stmt == stmt || is_gimple_debug (use_stmt))
    7669          126 :                 continue;
    7670          126 :               if (gimple_bb (use_stmt) != gimple_bb (stmt)
    7671          126 :                   || !is_gimple_assign (use_stmt)
    7672          126 :                   || gimple_assign_rhs_class (use_stmt) != GIMPLE_BINARY_RHS
    7673          126 :                   || other_store_stmt
    7674          252 :                   || TREE_CODE (gimple_assign_lhs (use_stmt)) != SSA_NAME)
    7675            0 :                 goto fail;
    7676          126 :               other_store_stmt = use_stmt;
    7677            0 :             }
    7678          126 :           if (other_store_stmt == NULL)
    7679            0 :             goto fail;
    7680          126 :           rhs = gimple_assign_lhs (other_store_stmt);
    7681          126 :           if (!single_imm_use (rhs, &use_p, &other_store_stmt))
    7682            0 :             goto fail;
    7683              :         }
    7684              :     }
    7685          260 :   else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3)
    7686              :     {
    7687          260 :       use_operand_p use_p;
    7688          260 :       imm_use_iterator iter;
    7689          780 :       FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
    7690              :         {
    7691          520 :           gimple *use_stmt = USE_STMT (use_p);
    7692          520 :           if (use_stmt == stmt || is_gimple_debug (use_stmt))
    7693          260 :             continue;
    7694          260 :           if (other_store_stmt)
    7695            0 :             goto fail;
    7696          260 :           other_store_stmt = use_stmt;
    7697          260 :         }
    7698              :     }
    7699              :   else
    7700            0 :     goto fail;
    7701              : 
    7702          512 :   gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
    7703          512 :   if (gimple_bb (def_stmt) != gimple_bb (stmt)
    7704          512 :       || !is_gimple_assign (def_stmt)
    7705         1024 :       || gimple_assign_rhs_class (def_stmt) != GIMPLE_BINARY_RHS)
    7706            0 :     goto fail;
    7707              : 
    7708          512 :   enum tree_code code = gimple_assign_rhs_code (def_stmt);
    7709              :   /* For pointer addition, we should use the normal plus for the vector
    7710              :      operation.  */
    7711          512 :   switch (code)
    7712              :     {
    7713            0 :     case POINTER_PLUS_EXPR:
    7714            0 :       code = PLUS_EXPR;
    7715            0 :       break;
    7716            0 :     case MULT_HIGHPART_EXPR:
    7717            0 :       goto fail;
    7718              :     default:
    7719              :       break;
    7720              :     }
    7721          512 :   if (TREE_CODE_LENGTH (code) != binary_op || !commutative_tree_code (code))
    7722            0 :     goto fail;
    7723              : 
    7724          512 :   tree rhs1 = gimple_assign_rhs1 (def_stmt);
    7725          512 :   tree rhs2 = gimple_assign_rhs2 (def_stmt);
    7726          512 :   if (TREE_CODE (rhs1) != SSA_NAME || TREE_CODE (rhs2) != SSA_NAME)
    7727            0 :     goto fail;
    7728              : 
    7729          512 :   gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
    7730          512 :   gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
    7731          512 :   if (gimple_bb (load1_stmt) != gimple_bb (stmt)
    7732          512 :       || !gimple_assign_load_p (load1_stmt)
    7733          512 :       || gimple_bb (load2_stmt) != gimple_bb (stmt)
    7734         1024 :       || !gimple_assign_load_p (load2_stmt))
    7735            0 :     goto fail;
    7736              : 
    7737          512 :   stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
    7738          512 :   stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
    7739          512 :   if (load1_stmt_info == NULL
    7740          512 :       || load2_stmt_info == NULL
    7741          512 :       || (STMT_VINFO_SIMD_LANE_ACCESS_P (load1_stmt_info)
    7742          512 :           != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info))
    7743          512 :       || (STMT_VINFO_SIMD_LANE_ACCESS_P (load2_stmt_info)
    7744          512 :           != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
    7745            0 :     goto fail;
    7746              : 
    7747          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && inscan_var_store)
    7748              :     {
    7749          126 :       dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
    7750          126 :       if (TREE_CODE (DR_BASE_ADDRESS (load1_dr_info->dr)) != ADDR_EXPR
    7751          126 :           || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0)))
    7752            0 :         goto fail;
    7753          126 :       tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
    7754          126 :       tree lrhs;
    7755          126 :       if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
    7756              :         lrhs = rhs1;
    7757              :       else
    7758           16 :         lrhs = rhs2;
    7759          126 :       use_operand_p use_p;
    7760          126 :       imm_use_iterator iter;
    7761          378 :       FOR_EACH_IMM_USE_FAST (use_p, iter, lrhs)
    7762              :         {
    7763          252 :           gimple *use_stmt = USE_STMT (use_p);
    7764          252 :           if (use_stmt == def_stmt || is_gimple_debug (use_stmt))
    7765          126 :             continue;
    7766          126 :           if (other_store_stmt)
    7767            0 :             goto fail;
    7768          126 :           other_store_stmt = use_stmt;
    7769          126 :         }
    7770              :     }
    7771              : 
    7772          512 :   if (other_store_stmt == NULL)
    7773            0 :     goto fail;
    7774          512 :   if (gimple_bb (other_store_stmt) != gimple_bb (stmt)
    7775          512 :       || !gimple_store_p (other_store_stmt))
    7776            0 :     goto fail;
    7777              : 
    7778          512 :   stmt_vec_info other_store_stmt_info
    7779          512 :     = loop_vinfo->lookup_stmt (other_store_stmt);
    7780          512 :   if (other_store_stmt_info == NULL
    7781          512 :       || (STMT_VINFO_SIMD_LANE_ACCESS_P (other_store_stmt_info)
    7782          512 :           != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
    7783            0 :     goto fail;
    7784              : 
    7785          512 :   gimple *stmt1 = stmt;
    7786          512 :   gimple *stmt2 = other_store_stmt;
    7787          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
    7788              :     std::swap (stmt1, stmt2);
    7789          512 :   if (scan_operand_equal_p (gimple_assign_lhs (stmt1),
    7790              :                             gimple_assign_rhs1 (load2_stmt)))
    7791              :     {
    7792          162 :       std::swap (rhs1, rhs2);
    7793          162 :       std::swap (load1_stmt, load2_stmt);
    7794          162 :       std::swap (load1_stmt_info, load2_stmt_info);
    7795              :     }
    7796          512 :   if (!scan_operand_equal_p (gimple_assign_lhs (stmt1),
    7797              :                              gimple_assign_rhs1 (load1_stmt)))
    7798            0 :     goto fail;
    7799              : 
    7800          512 :   tree var3 = NULL_TREE;
    7801          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3
    7802          512 :       && !scan_operand_equal_p (gimple_assign_lhs (stmt2),
    7803              :                                 gimple_assign_rhs1 (load2_stmt)))
    7804            0 :     goto fail;
    7805          512 :   else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
    7806              :     {
    7807          252 :       dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
    7808          252 :       if (TREE_CODE (DR_BASE_ADDRESS (load2_dr_info->dr)) != ADDR_EXPR
    7809          252 :           || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0)))
    7810            0 :         goto fail;
    7811          252 :       var3 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
    7812          252 :       if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var3))
    7813          252 :           || lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var3))
    7814          504 :           || lookup_attribute ("omp simd inscan exclusive",
    7815          252 :                                DECL_ATTRIBUTES (var3)))
    7816            0 :         goto fail;
    7817              :     }
    7818              : 
    7819          512 :   dr_vec_info *other_dr_info = STMT_VINFO_DR_INFO (other_store_stmt_info);
    7820          512 :   if (TREE_CODE (DR_BASE_ADDRESS (other_dr_info->dr)) != ADDR_EXPR
    7821          512 :       || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0)))
    7822            0 :     goto fail;
    7823              : 
    7824          512 :   tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
    7825          512 :   tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0);
    7826          512 :   if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var1))
    7827          512 :       || !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var2))
    7828         1024 :       || (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
    7829          512 :          == (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var2))))
    7830            0 :     goto fail;
    7831              : 
    7832          512 :   if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
    7833          256 :     std::swap (var1, var2);
    7834              : 
    7835          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
    7836              :     {
    7837          252 :       if (!lookup_attribute ("omp simd inscan exclusive",
    7838          252 :                              DECL_ATTRIBUTES (var1)))
    7839            0 :         goto fail;
    7840          252 :       var1 = var3;
    7841              :     }
    7842              : 
    7843          512 :   if (loop_vinfo->scan_map == NULL)
    7844            0 :     goto fail;
    7845          512 :   tree *init = loop_vinfo->scan_map->get (var1);
    7846          512 :   if (init == NULL)
    7847            0 :     goto fail;
    7848              : 
    7849              :   /* The IL is as expected, now check if we can actually vectorize it.
    7850              :      Inclusive scan:
    7851              :        _26 = D.2043[_25];
    7852              :        _27 = D.2042[_25];
    7853              :        _28 = _26 + _27;
    7854              :        D.2043[_25] = _28;
    7855              :        D.2042[_25] = _28;
    7856              :      should be vectorized as (where _40 is the vectorized rhs
    7857              :      from the D.2042[_21] = 0; store):
    7858              :        _30 = MEM <vector(8) int> [(int *)&D.2043];
    7859              :        _31 = MEM <vector(8) int> [(int *)&D.2042];
    7860              :        _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
    7861              :        _33 = _31 + _32;
    7862              :        // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
    7863              :        _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
    7864              :        _35 = _33 + _34;
    7865              :        // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
    7866              :        //         _31[1]+.._31[4], ... _31[4]+.._31[7] };
    7867              :        _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
    7868              :        _37 = _35 + _36;
    7869              :        // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
    7870              :        //         _31[0]+.._31[4], ... _31[0]+.._31[7] };
    7871              :        _38 = _30 + _37;
    7872              :        _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
    7873              :        MEM <vector(8) int> [(int *)&D.2043] = _39;
    7874              :        MEM <vector(8) int> [(int *)&D.2042] = _38;
    7875              :      Exclusive scan:
    7876              :        _26 = D.2043[_25];
    7877              :        D.2044[_25] = _26;
    7878              :        _27 = D.2042[_25];
    7879              :        _28 = _26 + _27;
    7880              :        D.2043[_25] = _28;
    7881              :      should be vectorized as (where _40 is the vectorized rhs
    7882              :      from the D.2042[_21] = 0; store):
    7883              :        _30 = MEM <vector(8) int> [(int *)&D.2043];
    7884              :        _31 = MEM <vector(8) int> [(int *)&D.2042];
    7885              :        _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
    7886              :        _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
    7887              :        _34 = _32 + _33;
    7888              :        // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
    7889              :        //         _31[3]+_31[4], ... _31[5]+.._31[6] };
    7890              :        _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
    7891              :        _36 = _34 + _35;
    7892              :        // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
    7893              :        //         _31[1]+.._31[4], ... _31[3]+.._31[6] };
    7894              :        _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
    7895              :        _38 = _36 + _37;
    7896              :        // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
    7897              :        //         _31[0]+.._31[4], ... _31[0]+.._31[6] };
    7898              :        _39 = _30 + _38;
    7899              :        _50 = _31 + _39;
    7900              :        _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
    7901              :        MEM <vector(8) int> [(int *)&D.2044] = _39;
    7902              :        MEM <vector(8) int> [(int *)&D.2042] = _51;  */
    7903          512 :   enum machine_mode vec_mode = TYPE_MODE (vectype);
    7904          512 :   optab optab = optab_for_tree_code (code, vectype, optab_default);
    7905          512 :   if (!optab || !can_implement_p (optab, vec_mode))
    7906            0 :     goto fail;
    7907              : 
    7908          512 :   int units_log2 = scan_store_can_perm_p (vectype, *init);
    7909          512 :   if (units_log2 == -1)
    7910            0 :     goto fail;
    7911              : 
    7912              :   return true;
    7913              : }
    7914              : 
    7915              : 
    7916              : /* Function vectorizable_scan_store.
    7917              : 
    7918              :    Helper of vectorizable_score, arguments like on vectorizable_store.
    7919              :    Handle only the transformation, checking is done in check_scan_store.  */
    7920              : 
    7921              : static bool
    7922          512 : vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
    7923              :                          slp_tree slp_node, gimple_stmt_iterator *gsi)
    7924              : {
    7925          512 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    7926          512 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    7927          512 :   tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
    7928          512 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    7929              : 
    7930          512 :   if (dump_enabled_p ())
    7931          492 :     dump_printf_loc (MSG_NOTE, vect_location,
    7932              :                      "transform scan store.\n");
    7933              : 
    7934          512 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
    7935          512 :   tree rhs = gimple_assign_rhs1 (stmt);
    7936          512 :   gcc_assert (TREE_CODE (rhs) == SSA_NAME);
    7937              : 
    7938          512 :   tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
    7939          512 :   bool inscan_var_store
    7940          512 :     = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
    7941              : 
    7942          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
    7943              :     {
    7944          126 :       use_operand_p use_p;
    7945          126 :       imm_use_iterator iter;
    7946          126 :       FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
    7947              :         {
    7948          126 :           gimple *use_stmt = USE_STMT (use_p);
    7949          126 :           if (use_stmt == stmt || is_gimple_debug (use_stmt))
    7950            0 :             continue;
    7951          126 :           rhs = gimple_assign_lhs (use_stmt);
    7952          126 :           break;
    7953          126 :         }
    7954              :     }
    7955              : 
    7956          512 :   gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
    7957          512 :   enum tree_code code = gimple_assign_rhs_code (def_stmt);
    7958          512 :   if (code == POINTER_PLUS_EXPR)
    7959            0 :     code = PLUS_EXPR;
    7960          512 :   gcc_assert (TREE_CODE_LENGTH (code) == binary_op
    7961              :               && commutative_tree_code (code));
    7962          512 :   tree rhs1 = gimple_assign_rhs1 (def_stmt);
    7963          512 :   tree rhs2 = gimple_assign_rhs2 (def_stmt);
    7964          512 :   gcc_assert (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == SSA_NAME);
    7965          512 :   gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
    7966          512 :   gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
    7967          512 :   stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
    7968          512 :   stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
    7969          512 :   dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
    7970          512 :   dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
    7971          512 :   tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
    7972          512 :   tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
    7973              : 
    7974          512 :   if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
    7975              :     {
    7976          436 :       std::swap (rhs1, rhs2);
    7977          436 :       std::swap (var1, var2);
    7978          436 :       std::swap (load1_dr_info, load2_dr_info);
    7979              :     }
    7980              : 
    7981          512 :   tree *init = loop_vinfo->scan_map->get (var1);
    7982          512 :   gcc_assert (init);
    7983              : 
    7984          512 :   unsigned HOST_WIDE_INT nunits;
    7985          512 :   if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
    7986              :     gcc_unreachable ();
    7987          512 :   auto_vec<enum scan_store_kind, 16> use_whole_vector;
    7988          512 :   int units_log2 = scan_store_can_perm_p (vectype, *init, &use_whole_vector);
    7989          512 :   gcc_assert (units_log2 > 0);
    7990          512 :   auto_vec<tree, 16> perms;
    7991          512 :   perms.quick_grow (units_log2 + 1);
    7992          512 :   tree zero_vec = NULL_TREE, masktype = NULL_TREE;
    7993         2904 :   for (int i = 0; i <= units_log2; ++i)
    7994              :     {
    7995         1880 :       unsigned HOST_WIDE_INT j, k;
    7996         1880 :       vec_perm_builder sel (nunits, nunits, 1);
    7997         1880 :       sel.quick_grow (nunits);
    7998         1880 :       if (i == units_log2)
    7999         4864 :         for (j = 0; j < nunits; ++j)
    8000         4352 :           sel[j] = nunits - 1;
    8001              :       else
    8002              :         {
    8003         5208 :           for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
    8004         3840 :             sel[j] = j;
    8005        13208 :           for (k = 0; j < nunits; ++j, ++k)
    8006        11840 :             sel[j] = nunits + k;
    8007              :         }
    8008         3248 :       vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
    8009         1880 :       if (!use_whole_vector.is_empty ()
    8010            0 :           && use_whole_vector[i] != scan_store_kind_perm)
    8011              :         {
    8012            0 :           if (zero_vec == NULL_TREE)
    8013            0 :             zero_vec = build_zero_cst (vectype);
    8014            0 :           if (masktype == NULL_TREE
    8015            0 :               && use_whole_vector[i] == scan_store_kind_lshift_cond)
    8016            0 :             masktype = truth_type_for (vectype);
    8017            0 :           perms[i] = vect_gen_perm_mask_any (vectype, indices);
    8018              :         }
    8019              :       else
    8020         1880 :         perms[i] = vect_gen_perm_mask_checked (vectype, indices);
    8021         1880 :     }
    8022              : 
    8023          512 :   tree vec_oprnd1 = NULL_TREE;
    8024          512 :   tree vec_oprnd2 = NULL_TREE;
    8025          512 :   tree vec_oprnd3 = NULL_TREE;
    8026          512 :   tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
    8027          512 :   tree dataref_offset = build_int_cst (ref_type, 0);
    8028          512 :   tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS);
    8029          512 :   tree ldataref_ptr = NULL_TREE;
    8030          512 :   tree orig = NULL_TREE;
    8031          512 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
    8032          126 :     ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
    8033              :   /* The initialization is invariant.  */
    8034          512 :   vec_oprnd1 = vect_init_vector (vinfo, stmt_info, *init, vectype, NULL);
    8035          512 :   auto_vec<tree> vec_oprnds2;
    8036          512 :   auto_vec<tree> vec_oprnds3;
    8037          512 :   if (ldataref_ptr == NULL)
    8038              :     {
    8039              :       /* We want to lookup the vector operands of the reduction, not those
    8040              :          of the store - for SLP we have to use the proper SLP node for the
    8041              :          lookup, which should be the single child of the scan store.  */
    8042          386 :       vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
    8043              :                          rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
    8044              :       /* ???  For SLP we do not key the def on 'rhs1' or 'rhs2' but get
    8045              :          them in SLP child order.  So we have to swap here with logic
    8046              :          similar to above.  */
    8047          386 :       stmt_vec_info load
    8048          386 :         = SLP_TREE_SCALAR_STMTS (SLP_TREE_CHILDREN
    8049          386 :                                    (SLP_TREE_CHILDREN (slp_node)[0])[0])[0];
    8050          386 :       dr_vec_info *dr_info = STMT_VINFO_DR_INFO (load);
    8051          386 :       tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
    8052          386 :       if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)))
    8053          820 :         for (unsigned i = 0; i < vec_oprnds2.length (); ++i)
    8054          494 :           std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
    8055              :     }
    8056              :   else
    8057          126 :     vect_get_vec_defs (vinfo, slp_node,
    8058              :                        rhs2, &vec_oprnds3);
    8059         1248 :   for (unsigned j = 0; j < vec_oprnds3.length (); j++)
    8060              :     {
    8061          736 :       if (ldataref_ptr == NULL)
    8062          554 :         vec_oprnd2 = vec_oprnds2[j];
    8063          736 :       vec_oprnd3 = vec_oprnds3[j];
    8064          736 :       if (j == 0)
    8065              :         orig = vec_oprnd3;
    8066          224 :       else if (!inscan_var_store)
    8067          112 :         dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
    8068              : 
    8069          736 :       if (ldataref_ptr)
    8070              :         {
    8071          182 :           vec_oprnd2 = make_ssa_name (vectype);
    8072          182 :           tree data_ref = fold_build2 (MEM_REF, vectype,
    8073              :                                        unshare_expr (ldataref_ptr),
    8074              :                                        dataref_offset);
    8075          182 :           vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
    8076          182 :           gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
    8077          182 :           vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8078              :         }
    8079              : 
    8080          736 :       tree v = vec_oprnd2;
    8081         3068 :       for (int i = 0; i < units_log2; ++i)
    8082              :         {
    8083         2332 :           tree new_temp = make_ssa_name (vectype);
    8084         2332 :           gimple *g = gimple_build_assign (new_temp, VEC_PERM_EXPR,
    8085              :                                            (zero_vec
    8086            0 :                                             && (use_whole_vector[i]
    8087            0 :                                                 != scan_store_kind_perm))
    8088              :                                            ? zero_vec : vec_oprnd1, v,
    8089         2332 :                                            perms[i]);
    8090         2332 :           vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8091              : 
    8092         2332 :           if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
    8093              :             {
    8094              :               /* Whole vector shift shifted in zero bits, but if *init
    8095              :                  is not initializer_zerop, we need to replace those elements
    8096              :                  with elements from vec_oprnd1.  */
    8097            0 :               tree_vector_builder vb (masktype, nunits, 1);
    8098            0 :               for (unsigned HOST_WIDE_INT k = 0; k < nunits; ++k)
    8099            0 :                 vb.quick_push (k < (HOST_WIDE_INT_1U << i)
    8100              :                                ? boolean_false_node : boolean_true_node);
    8101              : 
    8102            0 :               tree new_temp2 = make_ssa_name (vectype);
    8103            0 :               g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
    8104              :                                        new_temp, vec_oprnd1);
    8105            0 :               vect_finish_stmt_generation (vinfo, stmt_info,
    8106              :                                                            g, gsi);
    8107            0 :               new_temp = new_temp2;
    8108            0 :             }
    8109              : 
    8110              :           /* For exclusive scan, perform the perms[i] permutation once
    8111              :              more.  */
    8112         2332 :           if (i == 0
    8113         1100 :               && STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4
    8114          728 :               && v == vec_oprnd2)
    8115              :             {
    8116          364 :               v = new_temp;
    8117          364 :               --i;
    8118          364 :               continue;
    8119              :             }
    8120              : 
    8121         1968 :           tree new_temp2 = make_ssa_name (vectype);
    8122         1968 :           g = gimple_build_assign (new_temp2, code, v, new_temp);
    8123         1968 :           vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8124              : 
    8125         1968 :           v = new_temp2;
    8126              :         }
    8127              : 
    8128          736 :       tree new_temp = make_ssa_name (vectype);
    8129          736 :       gimple *g = gimple_build_assign (new_temp, code, orig, v);
    8130          736 :       vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8131              : 
    8132          736 :       tree last_perm_arg = new_temp;
    8133              :       /* For exclusive scan, new_temp computed above is the exclusive scan
    8134              :          prefix sum.  Turn it into inclusive prefix sum for the broadcast
    8135              :          of the last element into orig.  */
    8136          736 :       if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
    8137              :         {
    8138          364 :           last_perm_arg = make_ssa_name (vectype);
    8139          364 :           g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
    8140          364 :           vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8141              :         }
    8142              : 
    8143          736 :       orig = make_ssa_name (vectype);
    8144         2208 :       g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
    8145          736 :                                last_perm_arg, perms[units_log2]);
    8146          736 :       vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8147              : 
    8148          736 :       if (!inscan_var_store)
    8149              :         {
    8150          368 :           tree data_ref = fold_build2 (MEM_REF, vectype,
    8151              :                                        unshare_expr (dataref_ptr),
    8152              :                                        dataref_offset);
    8153          368 :           vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
    8154          368 :           g = gimple_build_assign (data_ref, new_temp);
    8155          368 :           vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8156              :         }
    8157              :     }
    8158              : 
    8159          512 :   if (inscan_var_store)
    8160          624 :     for (unsigned j = 0; j < vec_oprnds3.length (); j++)
    8161              :       {
    8162          368 :         if (j != 0)
    8163          112 :           dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
    8164              : 
    8165          368 :         tree data_ref = fold_build2 (MEM_REF, vectype,
    8166              :                                      unshare_expr (dataref_ptr),
    8167              :                                      dataref_offset);
    8168          368 :         vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
    8169          368 :         gimple *g = gimple_build_assign (data_ref, orig);
    8170          368 :         vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
    8171              :       }
    8172          512 :   return true;
    8173          512 : }
    8174              : 
    8175              : 
    8176              : /* Function vectorizable_store.
    8177              : 
    8178              :    Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
    8179              :    that can be vectorized.
    8180              :    If COST_VEC is passed, calculate costs but don't change anything,
    8181              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    8182              :    it, and insert it at GSI.
    8183              :    Return true if STMT_INFO is vectorizable in this way.  */
    8184              : 
    8185              : static bool
    8186      2167633 : vectorizable_store (vec_info *vinfo,
    8187              :                     stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    8188              :                     slp_tree slp_node,
    8189              :                     stmt_vector_for_cost *cost_vec)
    8190              : {
    8191      2167633 :   tree data_ref;
    8192      2167633 :   tree vec_oprnd = NULL_TREE;
    8193      2167633 :   tree elem_type;
    8194      2167633 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    8195      2167633 :   class loop *loop = NULL;
    8196      2167633 :   machine_mode vec_mode;
    8197      2167633 :   tree dummy;
    8198      2167633 :   enum vect_def_type rhs_dt = vect_unknown_def_type;
    8199      2167633 :   enum vect_def_type mask_dt = vect_unknown_def_type;
    8200      2167633 :   tree dataref_ptr = NULL_TREE;
    8201      2167633 :   tree dataref_offset = NULL_TREE;
    8202      2167633 :   int j;
    8203      2167633 :   stmt_vec_info first_stmt_info;
    8204      2167633 :   bool grouped_store;
    8205      2167633 :   unsigned int group_size, i;
    8206      2167633 :   unsigned int vec_num;
    8207      2167633 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    8208      2167633 :   tree aggr_type;
    8209      2167633 :   poly_uint64 vf;
    8210      2167633 :   vec_load_store_type vls_type;
    8211      2167633 :   tree ref_type;
    8212              : 
    8213      2167633 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    8214              :     return false;
    8215              : 
    8216      2167633 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    8217       251571 :       && cost_vec)
    8218              :     return false;
    8219              : 
    8220              :   /* Is vectorizable store? */
    8221              : 
    8222      1916062 :   tree mask_vectype = NULL_TREE;
    8223      1916062 :   slp_tree mask_node = NULL;
    8224      1916062 :   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
    8225              :     {
    8226      1840187 :       tree scalar_dest = gimple_assign_lhs (assign);
    8227      1840187 :       if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
    8228      1840187 :           && is_pattern_stmt_p (stmt_info))
    8229         1677 :         scalar_dest = TREE_OPERAND (scalar_dest, 0);
    8230      1840187 :       if (TREE_CODE (scalar_dest) != ARRAY_REF
    8231      1840187 :           && TREE_CODE (scalar_dest) != BIT_FIELD_REF
    8232              :           && TREE_CODE (scalar_dest) != INDIRECT_REF
    8233              :           && TREE_CODE (scalar_dest) != COMPONENT_REF
    8234              :           && TREE_CODE (scalar_dest) != IMAGPART_EXPR
    8235              :           && TREE_CODE (scalar_dest) != REALPART_EXPR
    8236              :           && TREE_CODE (scalar_dest) != MEM_REF)
    8237              :         return false;
    8238              :     }
    8239              :   else
    8240              :     {
    8241        75875 :       gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
    8242        13965 :       if (!call || !gimple_call_internal_p (call))
    8243              :         return false;
    8244              : 
    8245         8269 :       internal_fn ifn = gimple_call_internal_fn (call);
    8246         8269 :       if (!internal_store_fn_p (ifn))
    8247              :         return false;
    8248              : 
    8249         1883 :       int mask_index = internal_fn_mask_index (ifn);
    8250         1883 :       if (mask_index >= 0)
    8251         1883 :         mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
    8252         1883 :       if (mask_index >= 0
    8253         1883 :           && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
    8254              :                                       &mask_node, &mask_dt,
    8255              :                                       &mask_vectype))
    8256              :         return false;
    8257              :     }
    8258              : 
    8259      1387956 :   tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE;
    8260      1387956 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    8261              : 
    8262      1387956 :   if (loop_vinfo)
    8263              :     {
    8264       235905 :       loop = LOOP_VINFO_LOOP (loop_vinfo);
    8265       235905 :       vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    8266              :     }
    8267              :   else
    8268              :     vf = 1;
    8269      1387956 :   vec_num = vect_get_num_copies (vinfo, slp_node);
    8270              : 
    8271              :   /* FORNOW.  This restriction should be relaxed.  */
    8272      1387956 :   if (loop
    8273      1388243 :       && nested_in_vect_loop_p (loop, stmt_info)
    8274      1388251 :       && vec_num > 1)
    8275              :     {
    8276            8 :       if (dump_enabled_p ())
    8277            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8278              :                          "multiple types in nested loop.\n");
    8279              :       return false;
    8280              :     }
    8281              : 
    8282      1387948 :   slp_tree op_node;
    8283      1387948 :   if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
    8284              :                              &op_node, &rhs_dt, &rhs_vectype, &vls_type))
    8285              :     return false;
    8286              : 
    8287      1387924 :   elem_type = TREE_TYPE (vectype);
    8288      1387924 :   vec_mode = TYPE_MODE (vectype);
    8289              : 
    8290      1387924 :   if (!STMT_VINFO_DATA_REF (stmt_info))
    8291              :     return false;
    8292              : 
    8293      1387924 :   vect_load_store_data _ls_data{};
    8294      1387924 :   vect_load_store_data &ls = slp_node->get_data (_ls_data);
    8295      1387924 :   if (cost_vec
    8296      1387924 :       && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
    8297              :                                vls_type, &_ls_data))
    8298              :     return false;
    8299              :   /* Temporary aliases to analysis data, should not be modified through
    8300              :      these.  */
    8301      1387408 :   const vect_memory_access_type memory_access_type = ls.memory_access_type;
    8302      1387408 :   const dr_alignment_support alignment_support_scheme
    8303              :     = ls.alignment_support_scheme;
    8304      1387408 :   const int misalignment = ls.misalignment;
    8305      1387408 :   const poly_int64 poffset = ls.poffset;
    8306              : 
    8307      1387408 :   if (slp_node->ldst_lanes
    8308            0 :       && memory_access_type != VMAT_LOAD_STORE_LANES)
    8309              :     {
    8310            0 :       if (dump_enabled_p ())
    8311            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8312              :                          "discovered store-lane but cannot use it.\n");
    8313              :       return false;
    8314              :     }
    8315              : 
    8316      1387408 :   if (mask_node)
    8317              :     {
    8318         1793 :       if (memory_access_type == VMAT_CONTIGUOUS)
    8319              :         {
    8320          616 :           if (!VECTOR_MODE_P (vec_mode)
    8321         3070 :               || !can_vec_mask_load_store_p (vec_mode,
    8322         1535 :                                              TYPE_MODE (mask_vectype), false))
    8323              :             return false;
    8324              :         }
    8325          258 :       else if (memory_access_type != VMAT_LOAD_STORE_LANES
    8326          258 :                && (!mat_gather_scatter_p (memory_access_type)
    8327          242 :                    || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY
    8328          170 :                        && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
    8329              :         {
    8330           16 :           if (dump_enabled_p ())
    8331           16 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8332              :                              "unsupported access type for masked store.\n");
    8333              :           return false;
    8334              :         }
    8335          242 :       else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
    8336              :         {
    8337           72 :           if (dump_enabled_p ())
    8338           24 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8339              :                              "unsupported masked emulated scatter.\n");
    8340              :           return false;
    8341              :         }
    8342              :     }
    8343              :   else
    8344              :     {
    8345              :       /* FORNOW. In some cases can vectorize even if data-type not supported
    8346              :          (e.g. - array initialization with 0).  */
    8347      1385615 :       if (!can_implement_p (mov_optab, vec_mode))
    8348              :         return false;
    8349              :     }
    8350              : 
    8351      1387206 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
    8352      1387206 :   grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
    8353      2559598 :                    && !mat_gather_scatter_p (memory_access_type));
    8354      1172392 :   if (grouped_store)
    8355              :     {
    8356      1172392 :       first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
    8357      1172392 :       first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
    8358      1172392 :       group_size = DR_GROUP_SIZE (first_stmt_info);
    8359              :     }
    8360              :   else
    8361              :     {
    8362      1387206 :       first_stmt_info = stmt_info;
    8363      1387206 :       first_dr_info = dr_info;
    8364              :       group_size = 1;
    8365              :     }
    8366              : 
    8367      1387206 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
    8368              :     {
    8369         1076 :       if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
    8370              :                              mask_node, memory_access_type))
    8371              :         return false;
    8372              :     }
    8373              : 
    8374      1936950 :   bool costing_p = cost_vec;
    8375      1386438 :   if (costing_p) /* transformation not required.  */
    8376              :     {
    8377       836694 :       if (loop_vinfo
    8378       169382 :           && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
    8379        79482 :         check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
    8380              :                                               vls_type, group_size, &ls,
    8381              :                                               mask_node);
    8382              : 
    8383       836694 :       if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
    8384       836694 :           || (mask_node
    8385         1051 :               && !vect_maybe_update_slp_op_vectype (mask_node,
    8386              :                                                     mask_vectype)))
    8387              :         {
    8388            0 :           if (dump_enabled_p ())
    8389            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8390              :                              "incompatible vector types for invariants\n");
    8391              :           return false;
    8392              :         }
    8393              : 
    8394       836694 :       if (dump_enabled_p ()
    8395              :           && memory_access_type != VMAT_ELEMENTWISE
    8396        15867 :           && memory_access_type != VMAT_STRIDED_SLP
    8397        15200 :           && memory_access_type != VMAT_INVARIANT
    8398       851894 :           && alignment_support_scheme != dr_aligned)
    8399         5366 :         dump_printf_loc (MSG_NOTE, vect_location,
    8400              :                          "Vectorizing an unaligned access.\n");
    8401              :     }
    8402              : 
    8403              :   /* Transform.  */
    8404              : 
    8405              :   if (!costing_p)
    8406       550512 :     ensure_base_align (dr_info);
    8407              : 
    8408      1387206 :   if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
    8409              :     {
    8410         1024 :       gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
    8411         1024 :       gcc_assert (SLP_TREE_LANES (slp_node) == 1);
    8412         1024 :       if (costing_p)
    8413              :         {
    8414          512 :           unsigned int inside_cost = 0, prologue_cost = 0;
    8415          512 :           if (vls_type == VLS_STORE_INVARIANT)
    8416            0 :             prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
    8417              :                                                slp_node, 0, vect_prologue);
    8418          512 :           vect_get_store_cost (vinfo, stmt_info, slp_node, 1,
    8419              :                                alignment_support_scheme, misalignment,
    8420              :                                &inside_cost, cost_vec);
    8421              : 
    8422          512 :           if (dump_enabled_p ())
    8423          492 :             dump_printf_loc (MSG_NOTE, vect_location,
    8424              :                              "vect_model_store_cost: inside_cost = %d, "
    8425              :                              "prologue_cost = %d .\n",
    8426              :                              inside_cost, prologue_cost);
    8427              : 
    8428          512 :           SLP_TREE_TYPE (slp_node) = store_vec_info_type;
    8429          512 :           slp_node->data = new vect_load_store_data (std::move (ls));
    8430              : 
    8431          512 :           return true;
    8432              :         }
    8433          512 :       return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
    8434              :     }
    8435              : 
    8436              :   /* FORNOW */
    8437      1386182 :   gcc_assert (!grouped_store
    8438              :               || !loop
    8439              :               || !nested_in_vect_loop_p (loop, stmt_info));
    8440              : 
    8441      1386182 :   grouped_store = false;
    8442      1386182 :   first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
    8443      1386182 :   gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
    8444              :               || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
    8445      1386182 :   first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
    8446              : 
    8447      1386182 :   ref_type = get_group_alias_ptr_type (first_stmt_info);
    8448              : 
    8449      1386182 :   if (!costing_p && dump_enabled_p ())
    8450        12812 :     dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n");
    8451              : 
    8452      1386182 :   if (memory_access_type == VMAT_ELEMENTWISE
    8453      1386182 :       || memory_access_type == VMAT_STRIDED_SLP)
    8454              :     {
    8455        29784 :       unsigned inside_cost = 0, prologue_cost = 0;
    8456        29784 :       gimple_stmt_iterator incr_gsi;
    8457        29784 :       bool insert_after;
    8458        29784 :       tree offvar = NULL_TREE;
    8459        29784 :       tree ivstep;
    8460        29784 :       tree running_off;
    8461        29784 :       tree stride_base, stride_step, alias_off;
    8462        29784 :       tree vec_oprnd = NULL_TREE;
    8463        29784 :       tree dr_offset;
    8464              :       /* Checked by get_load_store_type.  */
    8465        29784 :       unsigned int const_nunits = nunits.to_constant ();
    8466              : 
    8467        29784 :       gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
    8468        29784 :       gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
    8469              : 
    8470        29784 :       dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
    8471        29784 :       stride_base
    8472        29784 :         = fold_build_pointer_plus
    8473              :             (DR_BASE_ADDRESS (first_dr_info->dr),
    8474              :              size_binop (PLUS_EXPR,
    8475              :                          convert_to_ptrofftype (dr_offset),
    8476              :                          convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
    8477        29784 :       stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
    8478              : 
    8479              :       /* For a store with loop-invariant (but other than power-of-2)
    8480              :          stride (i.e. not a grouped access) like so:
    8481              : 
    8482              :            for (i = 0; i < n; i += stride)
    8483              :              array[i] = ...;
    8484              : 
    8485              :          we generate a new induction variable and new stores from
    8486              :          the components of the (vectorized) rhs:
    8487              : 
    8488              :            for (j = 0; ; j += VF*stride)
    8489              :              vectemp = ...;
    8490              :              tmp1 = vectemp[0];
    8491              :              array[j] = tmp1;
    8492              :              tmp2 = vectemp[1];
    8493              :              array[j + stride] = tmp2;
    8494              :              ...
    8495              :          */
    8496              : 
    8497              :       /* ???  Modify local copies of alignment_support_scheme and
    8498              :          misalignment, but this part of analysis should be done
    8499              :          earlier and remembered, likewise the chosen load mode.  */
    8500        29784 :       const dr_alignment_support tem = alignment_support_scheme;
    8501        29784 :       dr_alignment_support alignment_support_scheme = tem;
    8502        29784 :       const int tem2 = misalignment;
    8503        29784 :       int misalignment = tem2;
    8504              : 
    8505        29784 :       unsigned nstores = const_nunits;
    8506        29784 :       unsigned lnel = 1;
    8507        29784 :       tree ltype = elem_type;
    8508        29784 :       tree lvectype = vectype;
    8509        29784 :       HOST_WIDE_INT n = gcd (group_size, const_nunits);
    8510        29784 :       if (n == const_nunits)
    8511              :         {
    8512         2978 :           int mis_align = dr_misalignment (first_dr_info, vectype);
    8513              :           /* With VF > 1 we advance the DR by step, if that is constant
    8514              :              and only aligned when performed VF times, DR alignment
    8515              :              analysis can analyze this as aligned since it assumes
    8516              :              contiguous accesses.  But that is not how we code generate
    8517              :              here, so adjust for this.  */
    8518         2978 :           if (maybe_gt (vf, 1u)
    8519         4514 :               && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
    8520         4279 :                               DR_TARGET_ALIGNMENT (first_dr_info)))
    8521          235 :             mis_align = -1;
    8522         2978 :           dr_alignment_support dr_align
    8523         2978 :               = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
    8524              :                                                mis_align);
    8525         2978 :           if (dr_align == dr_aligned
    8526         2978 :               || dr_align == dr_unaligned_supported)
    8527              :             {
    8528        29784 :               nstores = 1;
    8529        29784 :               lnel = const_nunits;
    8530        29784 :               ltype = vectype;
    8531        29784 :               lvectype = vectype;
    8532        29784 :               alignment_support_scheme = dr_align;
    8533        29784 :               misalignment = mis_align;
    8534              :             }
    8535              :         }
    8536        26806 :       else if (n > 1)
    8537              :         {
    8538         1969 :           nstores = const_nunits / n;
    8539         1969 :           lnel = n;
    8540         1969 :           ltype = build_vector_type (elem_type, n);
    8541         1969 :           lvectype = vectype;
    8542         1969 :           int mis_align = dr_misalignment (first_dr_info, ltype);
    8543         1969 :           if (maybe_gt (vf, 1u)
    8544         3938 :               && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
    8545         3292 :                               DR_TARGET_ALIGNMENT (first_dr_info)))
    8546          646 :             mis_align = -1;
    8547         1969 :           dr_alignment_support dr_align
    8548         1969 :             = vect_supportable_dr_alignment (vinfo, dr_info, ltype,
    8549              :                                              mis_align);
    8550         1969 :           alignment_support_scheme = dr_align;
    8551         1969 :           misalignment = mis_align;
    8552              : 
    8553              :           /* First check if vec_extract optab doesn't support extraction
    8554              :              of vector elts directly.  */
    8555         1969 :           scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
    8556         1969 :           machine_mode vmode;
    8557         3938 :           if (!VECTOR_MODE_P (TYPE_MODE (vectype))
    8558         1969 :               || !related_vector_mode (TYPE_MODE (vectype), elmode,
    8559         1969 :                                        n).exists (&vmode)
    8560         2141 :               || can_vec_extract (TYPE_MODE (vectype), vmode) != vmode
    8561         1969 :               || !(dr_align == dr_aligned
    8562          172 :                    || dr_align == dr_unaligned_supported))
    8563              :             {
    8564              :               /* Try to avoid emitting an extract of vector elements
    8565              :                  by performing the extracts using an integer type of the
    8566              :                  same size, extracting from a vector of those and then
    8567              :                  re-interpreting it as the original vector type if
    8568              :                  supported.  */
    8569         1797 :               unsigned lsize = n * GET_MODE_BITSIZE (elmode);
    8570         1797 :               unsigned int lnunits = const_nunits / n;
    8571              :               /* If we can't construct such a vector fall back to
    8572              :                  element extracts from the original vector type and
    8573              :                  element size stores.  */
    8574         1797 :               if (int_mode_for_size (lsize, 0).exists (&elmode)
    8575         1797 :                   && VECTOR_MODE_P (TYPE_MODE (vectype))
    8576         1797 :                   && related_vector_mode (TYPE_MODE (vectype), elmode,
    8577         1797 :                                           lnunits).exists (&vmode)
    8578         1769 :                   && can_vec_extract (vmode, elmode).exists ())
    8579              :                 {
    8580         1769 :                   nstores = lnunits;
    8581         1769 :                   lnel = n;
    8582         1769 :                   ltype = build_nonstandard_integer_type (lsize, 1);
    8583         1769 :                   lvectype = build_vector_type (ltype, nstores);
    8584              :                 }
    8585              :               /* Else fall back to vector extraction anyway.
    8586              :                  Fewer stores are more important than avoiding spilling
    8587              :                  of the vector we extract from.  Compared to the
    8588              :                  construction case in vectorizable_load no store-forwarding
    8589              :                  issue exists here for reasonable archs.  But only
    8590              :                  if the store is supported.  */
    8591           28 :                  else if (!(dr_align == dr_aligned
    8592           28 :                             || dr_align == dr_unaligned_supported))
    8593              :                    {
    8594        29784 :                      nstores = const_nunits;
    8595        29784 :                      lnel = 1;
    8596        29784 :                      ltype = elem_type;
    8597        29784 :                      lvectype = vectype;
    8598              :                    }
    8599              :             }
    8600              :         }
    8601              : 
    8602        29784 :       if (costing_p)
    8603              :         {
    8604              :           /* Record the decomposition type for target access during costing.  */
    8605        26331 :           ls.ls_type = lvectype;
    8606        26331 :           ls.ls_eltype = ltype;
    8607              :         }
    8608              :       else
    8609         3453 :         gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
    8610              : 
    8611        29784 :       unsigned align;
    8612        29784 :       if (alignment_support_scheme == dr_aligned)
    8613         1263 :         align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
    8614              :       else
    8615        28521 :         align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
    8616              :       /* Alignment is at most the access size if we do multiple stores.  */
    8617        29784 :       if (nstores > 1)
    8618        26806 :         align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
    8619        29784 :       ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
    8620        29784 :       int ncopies = vec_num;
    8621              : 
    8622        29784 :       if (!costing_p)
    8623              :         {
    8624         3453 :           ivstep = stride_step;
    8625              : 
    8626         3453 :           tree increment = fold_convert (TREE_TYPE (ivstep),
    8627              :                                          LOOP_VINFO_IV_INCREMENT (loop_vinfo));
    8628              : 
    8629         3453 :           ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
    8630              :                                 increment);
    8631              : 
    8632         3453 :           standard_iv_increment_position (loop, &incr_gsi, &insert_after);
    8633              : 
    8634         3453 :           stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
    8635         3453 :           if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
    8636         3453 :             ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
    8637              :           else
    8638            0 :             ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
    8639              :                                                true, NULL_TREE, true,
    8640              :                                                GSI_SAME_STMT);
    8641              : 
    8642         3453 :           create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
    8643              :                      insert_after, &offvar, NULL,
    8644         3453 :                      LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
    8645              : 
    8646         3453 :           stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
    8647              :         }
    8648              : 
    8649        29784 :       alias_off = build_int_cst (ref_type, 0);
    8650        29784 :       auto_vec<tree> vec_oprnds;
    8651              :       /* For costing some adjacent vector stores, we'd like to cost with
    8652              :          the total number of them once instead of cost each one by one. */
    8653        29784 :       unsigned int n_adjacent_stores = 0;
    8654        29784 :       running_off = offvar;
    8655        29784 :       if (!costing_p)
    8656         3453 :         vect_get_slp_defs (op_node, &vec_oprnds);
    8657        29784 :       unsigned int group_el = 0;
    8658        29784 :       unsigned HOST_WIDE_INT elsz
    8659        29784 :         = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
    8660        70883 :       for (j = 0; j < ncopies; j++)
    8661              :         {
    8662        41099 :           if (!costing_p)
    8663              :             {
    8664         5347 :               vec_oprnd = vec_oprnds[j];
    8665              :               /* Pun the vector to extract from if necessary.  */
    8666         5347 :               if (lvectype != vectype)
    8667              :                 {
    8668         1010 :                   tree tem = make_ssa_name (lvectype);
    8669         1010 :                   tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd);
    8670         1010 :                   gimple *pun = gimple_build_assign (tem, cvt);
    8671         1010 :                   vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
    8672         1010 :                   vec_oprnd = tem;
    8673              :                 }
    8674              :             }
    8675       182776 :           for (i = 0; i < nstores; i++)
    8676              :             {
    8677       141677 :               if (costing_p)
    8678              :                 {
    8679       125495 :                   n_adjacent_stores++;
    8680       125495 :                   continue;
    8681              :                 }
    8682        16182 :               tree newref, newoff;
    8683        16182 :               gimple *incr, *assign;
    8684        16182 :               tree size = TYPE_SIZE (ltype);
    8685              :               /* Extract the i'th component.  */
    8686        16182 :               tree pos = fold_build2 (MULT_EXPR, bitsizetype,
    8687              :                                       bitsize_int (i), size);
    8688        16182 :               tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
    8689              :                                        size, pos);
    8690              : 
    8691        16182 :               elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true,
    8692              :                                                GSI_SAME_STMT);
    8693              : 
    8694        16182 :               tree this_off = build_int_cst (TREE_TYPE (alias_off),
    8695        16182 :                                              group_el * elsz);
    8696        16182 :               newref = build2 (MEM_REF, ltype, running_off, this_off);
    8697        16182 :               vect_copy_ref_info (newref, DR_REF (first_dr_info->dr));
    8698              : 
    8699              :               /* And store it to *running_off.  */
    8700        16182 :               assign = gimple_build_assign (newref, elem);
    8701        16182 :               vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
    8702              : 
    8703        16182 :               group_el += lnel;
    8704        16182 :               if (group_el == group_size)
    8705              :                 {
    8706        14506 :                   newoff = copy_ssa_name (running_off, NULL);
    8707        14506 :                   incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
    8708              :                                               running_off, stride_step);
    8709        14506 :                   vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
    8710              : 
    8711        14506 :                   running_off = newoff;
    8712        14506 :                   group_el = 0;
    8713              :                 }
    8714              :             }
    8715              :         }
    8716              : 
    8717        29784 :       if (costing_p)
    8718              :         {
    8719        26331 :           if (n_adjacent_stores > 0)
    8720              :             {
    8721              :               /* Take a single lane vector type store as scalar
    8722              :                  store to avoid ICE like 110776.  */
    8723        26331 :               if (VECTOR_TYPE_P (ltype)
    8724        26331 :                   && maybe_ne (TYPE_VECTOR_SUBPARTS (ltype), 1U))
    8725         1616 :                 vect_get_store_cost (vinfo, stmt_info, slp_node,
    8726              :                                      n_adjacent_stores, alignment_support_scheme,
    8727              :                                      misalignment, &inside_cost, cost_vec);
    8728              :               else
    8729        24715 :                 inside_cost
    8730        24715 :                   += record_stmt_cost (cost_vec, n_adjacent_stores,
    8731              :                                        scalar_store, slp_node, 0, vect_body);
    8732              :               /* Only need vector deconstruction when there is more
    8733              :                  than one store.  */
    8734        26331 :               if (nstores > 1)
    8735        24299 :                 inside_cost
    8736        24299 :                   += record_stmt_cost (cost_vec, ncopies,
    8737              :                                        vec_deconstruct, slp_node, 0, vect_body);
    8738              :             }
    8739        26331 :           if (dump_enabled_p ())
    8740          667 :             dump_printf_loc (MSG_NOTE, vect_location,
    8741              :                              "vect_model_store_cost: inside_cost = %d, "
    8742              :                              "prologue_cost = %d .\n",
    8743              :                              inside_cost, prologue_cost);
    8744              : 
    8745        26331 :           SLP_TREE_TYPE (slp_node) = store_vec_info_type;
    8746        26331 :           slp_node->data = new vect_load_store_data (std::move (ls));
    8747              :         }
    8748              : 
    8749        29784 :       return true;
    8750        29784 :     }
    8751              : 
    8752      1356398 :   gcc_assert (alignment_support_scheme);
    8753      1356398 :   vec_loop_masks *loop_masks
    8754       204347 :     = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
    8755      1356398 :        ? &LOOP_VINFO_MASKS (loop_vinfo)
    8756           16 :        : NULL);
    8757           16 :   vec_loop_lens *loop_lens
    8758       204347 :     = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
    8759              :        ? &LOOP_VINFO_LENS (loop_vinfo)
    8760            0 :        : NULL);
    8761              : 
    8762              :   /* The vect_transform_stmt and vect_analyze_stmt will go here but there
    8763              :      are some difference here.  We cannot enable both the lens and masks
    8764              :      during transform but it is allowed during analysis.
    8765              :      Shouldn't go with length-based approach if fully masked.  */
    8766      1356398 :   if (cost_vec == NULL)
    8767              :     /* The cost_vec is NULL during transform.  */
    8768       546547 :     gcc_assert ((!loop_lens || !loop_masks));
    8769              : 
    8770              :   /* Targets with store-lane instructions must not require explicit
    8771              :      realignment.  vect_supportable_dr_alignment always returns either
    8772              :      dr_aligned or dr_unaligned_supported for masked operations.  */
    8773      1356398 :   gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
    8774              :                && !mask_node
    8775              :                && !loop_masks)
    8776              :               || alignment_support_scheme == dr_aligned
    8777              :               || alignment_support_scheme == dr_unaligned_supported);
    8778              : 
    8779      1356398 :   tree offset = NULL_TREE;
    8780      1356398 :   if (!known_eq (poffset, 0))
    8781         4699 :     offset = size_int (poffset);
    8782              : 
    8783      1356398 :   tree dr_increment;
    8784      1356398 :   tree dr_bump;
    8785              : 
    8786      1356398 :   tree vec_offset = NULL_TREE;
    8787      1356398 :   if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    8788              :     {
    8789         1383 :       aggr_type = NULL_TREE;
    8790         1383 :       dr_increment = NULL_TREE;
    8791         1383 :       dr_bump = NULL_TREE;
    8792              :     }
    8793      1355015 :   else if (mat_gather_scatter_p (memory_access_type))
    8794              :     {
    8795            0 :       aggr_type = elem_type;
    8796            0 :       if (!costing_p)
    8797              :         {
    8798            0 :           tree vtype = ls.ls_type ? ls.ls_type : vectype;
    8799            0 :           vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
    8800              :                                            ls.strided_offset_vectype,
    8801              :                                            loop_vinfo, gsi,
    8802              :                                            &dr_increment, &dr_bump,
    8803              :                                            &vec_offset);
    8804              :         }
    8805              :     }
    8806              :   else
    8807              :     {
    8808      1355015 :       if (memory_access_type == VMAT_LOAD_STORE_LANES)
    8809            0 :         aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
    8810              :       else
    8811              :         aggr_type = vectype;
    8812      1355015 :       if (!costing_p)
    8813              :         {
    8814       546073 :           dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
    8815              :                                                  memory_access_type);
    8816       546073 :           dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
    8817              :                                             memory_access_type);
    8818              :         }
    8819              :     }
    8820              : 
    8821      1356398 :   if (loop_vinfo && mask_node && !costing_p)
    8822          540 :     LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
    8823              : 
    8824              :   /* In case the vectorization factor (VF) is bigger than the number
    8825              :      of elements that we can fit in a vectype (nunits), we have to generate
    8826              :      more than one vector stmt - i.e - we need to "unroll" the
    8827              :      vector stmt by a factor VF/nunits.  */
    8828              : 
    8829      1356398 :   auto_vec<tree> dr_chain (group_size);
    8830      1356398 :   auto_vec<tree> vec_masks;
    8831      1356398 :   tree vec_mask = NULL;
    8832      1356398 :   auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
    8833      6106063 :   for (i = 0; i < group_size; i++)
    8834      3393267 :     gvec_oprnds.quick_push (new auto_vec<tree> ());
    8835              : 
    8836      1356398 :   if (memory_access_type == VMAT_LOAD_STORE_LANES)
    8837              :     {
    8838            0 :       const internal_fn lanes_ifn = ls.lanes_ifn;
    8839              : 
    8840            0 :       if (costing_p)
    8841              :         /* Update all incoming store operand nodes, the general handling
    8842              :            above only handles the mask and the first store operand node.  */
    8843            0 :         for (slp_tree child : SLP_TREE_CHILDREN (slp_node))
    8844            0 :           if (child != mask_node
    8845            0 :               && !vect_maybe_update_slp_op_vectype (child, vectype))
    8846              :             {
    8847            0 :               if (dump_enabled_p ())
    8848            0 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8849              :                                  "incompatible vector types for invariants\n");
    8850              :               return false;
    8851              :             }
    8852            0 :       unsigned inside_cost = 0, prologue_cost = 0;
    8853              :       /* For costing some adjacent vector stores, we'd like to cost with
    8854              :          the total number of them once instead of cost each one by one. */
    8855            0 :       unsigned int n_adjacent_stores = 0;
    8856            0 :       int ncopies = vec_num / group_size;
    8857            0 :       for (j = 0; j < ncopies; j++)
    8858              :         {
    8859            0 :           if (j == 0)
    8860              :             {
    8861            0 :               if (!costing_p)
    8862              :                 {
    8863            0 :                   if (mask_node)
    8864              :                     {
    8865            0 :                       vect_get_slp_defs (mask_node, &vec_masks);
    8866            0 :                       vec_mask = vec_masks[0];
    8867              :                     }
    8868            0 :                   dataref_ptr
    8869            0 :                     = vect_create_data_ref_ptr (vinfo, first_stmt_info,
    8870              :                                                 aggr_type, NULL, offset, &dummy,
    8871              :                                                 gsi, NULL, false, dr_increment);
    8872              :                 }
    8873              :             }
    8874            0 :           else if (!costing_p)
    8875              :             {
    8876            0 :               gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
    8877            0 :               if (mask_node)
    8878            0 :                 vec_mask = vec_masks[j];
    8879            0 :               dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
    8880              :                                              stmt_info, dr_bump);
    8881              :             }
    8882              : 
    8883            0 :           if (costing_p)
    8884              :             {
    8885            0 :               n_adjacent_stores += group_size;
    8886            0 :               continue;
    8887              :             }
    8888              : 
    8889              :           /* Get an array into which we can store the individual vectors.  */
    8890            0 :           tree vec_array = create_vector_array (vectype, group_size);
    8891              : 
    8892              :           /* Invalidate the current contents of VEC_ARRAY.  This should
    8893              :              become an RTL clobber too, which prevents the vector registers
    8894              :              from being upward-exposed.  */
    8895            0 :           vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
    8896              : 
    8897              :           /* Store the individual vectors into the array.  */
    8898            0 :           for (i = 0; i < group_size; i++)
    8899              :             {
    8900            0 :               slp_tree child;
    8901            0 :               if (i == 0 || !mask_node)
    8902            0 :                 child = SLP_TREE_CHILDREN (slp_node)[i];
    8903              :               else
    8904            0 :                 child = SLP_TREE_CHILDREN (slp_node)[i + 1];
    8905            0 :               vec_oprnd = SLP_TREE_VEC_DEFS (child)[j];
    8906            0 :               write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
    8907              :                                   i);
    8908              :             }
    8909              : 
    8910            0 :           tree final_mask = NULL;
    8911            0 :           tree final_len = NULL;
    8912            0 :           tree bias = NULL;
    8913            0 :           if (loop_masks)
    8914            0 :             final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
    8915              :                                              ncopies, vectype, j);
    8916            0 :           if (vec_mask)
    8917            0 :             final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
    8918              :                                            vec_mask, gsi);
    8919              : 
    8920            0 :           if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
    8921              :             {
    8922            0 :               if (loop_lens)
    8923            0 :                 final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
    8924              :                                                ncopies, vectype, j, 1, true);
    8925              :               else
    8926            0 :                 final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
    8927            0 :               signed char biasval
    8928            0 :                 = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    8929            0 :               bias = build_int_cst (intQI_type_node, biasval);
    8930            0 :               if (!final_mask)
    8931              :                 {
    8932            0 :                   mask_vectype = truth_type_for (vectype);
    8933            0 :                   final_mask = build_minus_one_cst (mask_vectype);
    8934              :                 }
    8935              :             }
    8936              : 
    8937            0 :           gcall *call;
    8938            0 :           if (final_len && final_mask)
    8939              :             {
    8940              :               /* Emit:
    8941              :                    MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
    8942              :                                          LEN, BIAS, VEC_ARRAY).  */
    8943            0 :               unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
    8944            0 :               tree alias_ptr = build_int_cst (ref_type, align);
    8945            0 :               call = gimple_build_call_internal (IFN_MASK_LEN_STORE_LANES, 6,
    8946              :                                                  dataref_ptr, alias_ptr,
    8947              :                                                  final_mask, final_len, bias,
    8948              :                                                  vec_array);
    8949              :             }
    8950            0 :           else if (final_mask)
    8951              :             {
    8952              :               /* Emit:
    8953              :                    MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
    8954              :                                      VEC_ARRAY).  */
    8955            0 :               unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
    8956            0 :               tree alias_ptr = build_int_cst (ref_type, align);
    8957            0 :               call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
    8958              :                                                  dataref_ptr, alias_ptr,
    8959              :                                                  final_mask, vec_array);
    8960              :             }
    8961              :           else
    8962              :             {
    8963              :               /* Emit:
    8964              :                    MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY).  */
    8965            0 :               data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
    8966            0 :               call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
    8967            0 :               gimple_call_set_lhs (call, data_ref);
    8968              :             }
    8969            0 :           gimple_call_set_nothrow (call, true);
    8970            0 :           vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    8971              : 
    8972              :           /* Record that VEC_ARRAY is now dead.  */
    8973            0 :           vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
    8974              :         }
    8975              : 
    8976            0 :       if (costing_p)
    8977              :         {
    8978            0 :           if (n_adjacent_stores > 0)
    8979            0 :             vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
    8980              :                                  alignment_support_scheme, misalignment,
    8981              :                                  &inside_cost, cost_vec);
    8982            0 :           if (dump_enabled_p ())
    8983            0 :             dump_printf_loc (MSG_NOTE, vect_location,
    8984              :                              "vect_model_store_cost: inside_cost = %d, "
    8985              :                              "prologue_cost = %d .\n",
    8986              :                              inside_cost, prologue_cost);
    8987              : 
    8988            0 :           SLP_TREE_TYPE (slp_node) = store_vec_info_type;
    8989            0 :           slp_node->data = new vect_load_store_data (std::move (ls));
    8990              :         }
    8991              : 
    8992              :       return true;
    8993              :     }
    8994              : 
    8995      1356398 :   if (mat_gather_scatter_p (memory_access_type))
    8996              :     {
    8997         1383 :       gcc_assert (!grouped_store || ls.ls_type);
    8998         1383 :       if (ls.ls_type)
    8999            0 :         vectype = ls.ls_type;
    9000         1383 :       auto_vec<tree> vec_offsets;
    9001         1383 :       unsigned int inside_cost = 0, prologue_cost = 0;
    9002         1383 :       int num_stmts = vec_num;
    9003         3023 :       for (j = 0; j < num_stmts; j++)
    9004              :         {
    9005         1640 :           gimple *new_stmt;
    9006         1640 :           if (j == 0)
    9007              :             {
    9008         1383 :               if (costing_p && vls_type == VLS_STORE_INVARIANT)
    9009          148 :                 prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
    9010              :                                                    slp_node, 0, vect_prologue);
    9011              :               else if (!costing_p)
    9012              :                 {
    9013              :                   /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
    9014              :                      DR_CHAIN is of size 1.  */
    9015          474 :                   gcc_assert (group_size == 1);
    9016          474 :                   vect_get_slp_defs (op_node, gvec_oprnds[0]);
    9017          474 :                   if (mask_node)
    9018           70 :                     vect_get_slp_defs (mask_node, &vec_masks);
    9019              : 
    9020          474 :                   if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    9021          474 :                     vect_get_gather_scatter_ops (loop, slp_node,
    9022              :                                                  &dataref_ptr, &vec_offsets);
    9023              :                   else
    9024            0 :                     dataref_ptr
    9025            0 :                       = vect_create_data_ref_ptr (vinfo, first_stmt_info,
    9026              :                                                   aggr_type, NULL, offset,
    9027              :                                                   &dummy, gsi, NULL, false,
    9028              :                                                   dr_increment);
    9029              :                 }
    9030              :             }
    9031          257 :           else if (!costing_p)
    9032              :             {
    9033           35 :               gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
    9034           35 :               if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    9035            0 :                 dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
    9036              :                                                gsi, stmt_info, dr_bump);
    9037              :             }
    9038              : 
    9039         2297 :           new_stmt = NULL;
    9040          657 :           if (!costing_p)
    9041              :             {
    9042          509 :               vec_oprnd = (*gvec_oprnds[0])[j];
    9043          509 :               if (mask_node)
    9044           90 :                 vec_mask = vec_masks[j];
    9045              :               /* We should have caught mismatched types earlier.  */
    9046          509 :               gcc_assert (ls.ls_type
    9047              :                           || useless_type_conversion_p
    9048              :                           (vectype, TREE_TYPE (vec_oprnd)));
    9049              :             }
    9050          509 :           tree final_mask = NULL_TREE;
    9051         2149 :           tree final_len = NULL_TREE;
    9052         2149 :           tree bias = NULL_TREE;
    9053          509 :           if (!costing_p)
    9054              :             {
    9055          509 :               if (loop_masks)
    9056            0 :                 final_mask = vect_get_loop_mask (loop_vinfo, gsi,
    9057              :                                                  loop_masks, num_stmts,
    9058              :                                                  vectype, j);
    9059          509 :               if (vec_mask)
    9060           90 :                 final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
    9061              :                                                final_mask, vec_mask, gsi);
    9062              :             }
    9063              : 
    9064         1640 :           unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
    9065         1640 :           tree alias_align_ptr = build_int_cst (ref_type, align);
    9066         1640 :           if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
    9067              :             {
    9068            0 :               if (costing_p)
    9069              :                 {
    9070            0 :                   if (ls.supported_offset_vectype
    9071            0 :                       && !tree_nop_conversion_p (ls.supported_offset_vectype,
    9072              :                                                  vec_offset))
    9073            0 :                     inside_cost
    9074            0 :                       += record_stmt_cost (cost_vec, 1, vector_stmt,
    9075              :                                            slp_node, 0, vect_body);
    9076            0 :                   if (ls.supported_scale)
    9077            0 :                     inside_cost
    9078            0 :                       += record_stmt_cost (cost_vec, 1, vector_stmt,
    9079              :                                            slp_node, 0, vect_body);
    9080              : 
    9081            0 :                   unsigned int cnunits = vect_nunits_for_cost (vectype);
    9082            0 :                   inside_cost
    9083            0 :                     += record_stmt_cost (cost_vec, cnunits, scalar_store,
    9084              :                                          slp_node, 0, vect_body);
    9085            0 :                   continue;
    9086            0 :                 }
    9087              : 
    9088            0 :               if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    9089            0 :                 vec_offset = vec_offsets[j];
    9090              : 
    9091            0 :               tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
    9092            0 :               bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
    9093              : 
    9094              :               /* Perform the offset conversion and scaling if necessary.  */
    9095            0 :               if (!strided
    9096            0 :                   && (ls.supported_offset_vectype || ls.supported_scale))
    9097              :                 {
    9098            0 :                   gimple_seq stmts = NULL;
    9099            0 :                   if (ls.supported_offset_vectype)
    9100            0 :                     vec_offset = gimple_convert
    9101            0 :                       (&stmts, ls.supported_offset_vectype, vec_offset);
    9102            0 :                   if (ls.supported_scale)
    9103              :                     {
    9104              :                       /* Only scale the vec_offset if we haven't already.  */
    9105            0 :                       if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
    9106            0 :                           || j == 0)
    9107              :                         {
    9108            0 :                           tree mult_cst = build_int_cst
    9109            0 :                             (TREE_TYPE (TREE_TYPE (vec_offset)),
    9110            0 :                              SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
    9111            0 :                           tree mult = build_vector_from_val
    9112            0 :                             (TREE_TYPE (vec_offset), mult_cst);
    9113            0 :                           vec_offset = gimple_build
    9114            0 :                             (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
    9115              :                              vec_offset, mult);
    9116              :                         }
    9117            0 :                       scale = size_int (ls.supported_scale);
    9118              :                     }
    9119            0 :                   gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    9120              :                 }
    9121              : 
    9122            0 :               if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
    9123              :                 {
    9124            0 :                   if (loop_lens)
    9125            0 :                     final_len = vect_get_loop_len (loop_vinfo, gsi,
    9126              :                                                    loop_lens, num_stmts,
    9127              :                                                    vectype, j, 1, true);
    9128              :                   else
    9129            0 :                     final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
    9130              : 
    9131            0 :                   signed char biasval
    9132            0 :                     = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    9133            0 :                   bias = build_int_cst (intQI_type_node, biasval);
    9134            0 :                   if (!final_mask)
    9135              :                     {
    9136            0 :                       mask_vectype = truth_type_for (vectype);
    9137            0 :                       final_mask = build_minus_one_cst (mask_vectype);
    9138              :                     }
    9139              :                 }
    9140              : 
    9141            0 :               if (ls.ls_type)
    9142              :                 {
    9143            0 :                   gimple *conv_stmt
    9144            0 :                     = gimple_build_assign (make_ssa_name (vectype),
    9145              :                                            VIEW_CONVERT_EXPR,
    9146              :                                            build1 (VIEW_CONVERT_EXPR, vectype,
    9147              :                                                    vec_oprnd));
    9148            0 :                   vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
    9149              :                                                gsi);
    9150            0 :                   vec_oprnd = gimple_get_lhs (conv_stmt);
    9151              :                 }
    9152              : 
    9153            0 :               gcall *call;
    9154            0 :               if (final_len && final_mask)
    9155              :                 {
    9156            0 :                   if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
    9157            0 :                     call = gimple_build_call_internal (
    9158              :                             IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
    9159              :                             alias_align_ptr,
    9160              :                             vec_offset, scale, vec_oprnd, final_mask, final_len,
    9161              :                             bias);
    9162              :                   else
    9163              :                     /* Non-vector offset indicates that prefer to take
    9164              :                        MASK_LEN_STRIDED_STORE instead of the
    9165              :                        IFN_MASK_SCATTER_STORE with direct stride arg.
    9166              :                        Similar to the gather case we have checked the
    9167              :                        alignment for a scatter already and assume
    9168              :                        that the strided store has the same requirements.  */
    9169            0 :                     call = gimple_build_call_internal (
    9170              :                             IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
    9171              :                             vec_offset, vec_oprnd, final_mask, final_len, bias);
    9172              :                 }
    9173            0 :               else if (final_mask)
    9174            0 :                 call = gimple_build_call_internal
    9175            0 :                              (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
    9176              :                               alias_align_ptr,
    9177              :                               vec_offset, scale, vec_oprnd, final_mask);
    9178              :               else
    9179            0 :                 call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
    9180              :                                                    dataref_ptr,
    9181              :                                                    alias_align_ptr,
    9182              :                                                    vec_offset,
    9183              :                                                    scale, vec_oprnd);
    9184            0 :               gimple_call_set_nothrow (call, true);
    9185            0 :               vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    9186            0 :               new_stmt = call;
    9187              :             }
    9188         1640 :           else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
    9189              :             {
    9190              :               /* The builtin decls path for scatter is legacy, x86 only.  */
    9191          331 :               gcc_assert (nunits.is_constant ()
    9192              :                           && (!final_mask
    9193              :                               || SCALAR_INT_MODE_P
    9194              :                                    (TYPE_MODE (TREE_TYPE (final_mask)))));
    9195          331 :               if (costing_p)
    9196              :                 {
    9197          200 :                   unsigned int cnunits = vect_nunits_for_cost (vectype);
    9198          200 :                   inside_cost
    9199          200 :                     += record_stmt_cost (cost_vec, cnunits, scalar_store,
    9200              :                                          slp_node, 0, vect_body);
    9201          200 :                   continue;
    9202          200 :                 }
    9203              : 
    9204          131 :                 tree offset_vectype = TREE_TYPE (vec_offsets[0]);
    9205          131 :                 poly_uint64 offset_nunits
    9206          131 :                   = TYPE_VECTOR_SUBPARTS (offset_vectype);
    9207          131 :                 if (known_eq (nunits, offset_nunits))
    9208              :                   {
    9209           55 :                     new_stmt = vect_build_one_scatter_store_call
    9210          110 :                                    (vinfo, stmt_info, slp_node, gsi,
    9211           55 :                                     ls.gs.decl, dataref_ptr, vec_offsets[j],
    9212              :                                     vec_oprnd, final_mask);
    9213           55 :                     vect_finish_stmt_generation (vinfo, stmt_info,
    9214              :                                                  new_stmt, gsi);
    9215              :                   }
    9216           76 :                 else if (known_eq (nunits, offset_nunits * 2))
    9217              :                   {
    9218              :                     /* We have a offset vector with half the number of
    9219              :                        lanes but the builtins will store full vectype
    9220              :                        data from the lower lanes.  */
    9221           30 :                     new_stmt = vect_build_one_scatter_store_call
    9222           60 :                                    (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
    9223           30 :                                     dataref_ptr, vec_offsets[2 * j],
    9224              :                                     vec_oprnd, final_mask);
    9225           30 :                     vect_finish_stmt_generation (vinfo, stmt_info,
    9226              :                                                    new_stmt, gsi);
    9227           30 :                     int count = nunits.to_constant ();
    9228           30 :                     vec_perm_builder sel (count, count, 1);
    9229           30 :                     sel.quick_grow (count);
    9230          412 :                     for (int i = 0; i < count; ++i)
    9231          352 :                       sel[i] = i | (count / 2);
    9232           30 :                     vec_perm_indices indices (sel, 2, count);
    9233           30 :                     tree perm_mask
    9234           30 :                       = vect_gen_perm_mask_checked (vectype, indices);
    9235           30 :                     new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
    9236              :                                                     vec_oprnd, vec_oprnd,
    9237              :                                                     perm_mask);
    9238           30 :                     vec_oprnd = make_ssa_name (vectype);
    9239           30 :                     gimple_set_lhs (new_stmt, vec_oprnd);
    9240           30 :                     vect_finish_stmt_generation (vinfo, stmt_info,
    9241              :                                                  new_stmt, gsi);
    9242           30 :                     if (final_mask)
    9243              :                       {
    9244           20 :                         new_stmt = gimple_build_assign (NULL_TREE,
    9245              :                                                         VEC_UNPACK_HI_EXPR,
    9246              :                                                         final_mask);
    9247           20 :                         final_mask = make_ssa_name
    9248           20 :                                       (truth_type_for (offset_vectype));
    9249           20 :                         gimple_set_lhs (new_stmt, final_mask);
    9250           20 :                         vect_finish_stmt_generation (vinfo, stmt_info,
    9251              :                                                      new_stmt, gsi);
    9252              :                         }
    9253              : 
    9254           30 :                     new_stmt = vect_build_one_scatter_store_call
    9255           60 :                                   (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
    9256           30 :                                    dataref_ptr, vec_offsets[2 * j + 1],
    9257              :                                    vec_oprnd, final_mask);
    9258           30 :                     vect_finish_stmt_generation (vinfo, stmt_info,
    9259              :                                                  new_stmt, gsi);
    9260           30 :                   }
    9261           46 :                 else if (known_eq (nunits * 2, offset_nunits))
    9262              :                   {
    9263              :                     /* We have a offset vector with double the number of
    9264              :                        lanes.  Select the low/high part accordingly.  */
    9265           46 :                     vec_offset = vec_offsets[j / 2];
    9266           46 :                     if (j & 1)
    9267              :                       {
    9268           23 :                         int count = offset_nunits.to_constant ();
    9269           23 :                         vec_perm_builder sel (count, count, 1);
    9270           23 :                         sel.quick_grow (count);
    9271          286 :                         for (int i = 0; i < count; ++i)
    9272          240 :                           sel[i] = i | (count / 2);
    9273           23 :                         vec_perm_indices indices (sel, 2, count);
    9274           23 :                         tree perm_mask = vect_gen_perm_mask_checked
    9275           23 :                                            (TREE_TYPE (vec_offset), indices);
    9276           23 :                         new_stmt = gimple_build_assign (NULL_TREE,
    9277              :                                                         VEC_PERM_EXPR,
    9278              :                                                         vec_offset,
    9279              :                                                         vec_offset,
    9280              :                                                         perm_mask);
    9281           23 :                         vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
    9282           23 :                         gimple_set_lhs (new_stmt, vec_offset);
    9283           23 :                         vect_finish_stmt_generation (vinfo, stmt_info,
    9284              :                                                      new_stmt, gsi);
    9285           23 :                       }
    9286              : 
    9287           46 :                     new_stmt = vect_build_one_scatter_store_call
    9288           46 :                                    (vinfo, stmt_info, slp_node, gsi,
    9289              :                                     ls.gs.decl, dataref_ptr, vec_offset,
    9290              :                                     vec_oprnd, final_mask);
    9291           46 :                     vect_finish_stmt_generation (vinfo, stmt_info,
    9292              :                                                  new_stmt, gsi);
    9293              :                   }
    9294              :                 else
    9295            0 :                   gcc_unreachable ();
    9296              :             }
    9297              :           else
    9298              :             {
    9299              :               /* Emulated scatter.  */
    9300         1309 :               gcc_assert (!final_mask);
    9301         1309 :               if (costing_p)
    9302              :                 {
    9303          931 :                   unsigned int cnunits = vect_nunits_for_cost (vectype);
    9304              :                   /* For emulated scatter N offset vector element extracts
    9305              :                      (we assume the scalar scaling and ptr + offset add is
    9306              :                      consumed by the load).  */
    9307          931 :                   inside_cost
    9308          931 :                     += record_stmt_cost (cost_vec, 1, vec_deconstruct,
    9309              :                                          slp_node, 0, vect_body);
    9310              :                   /* N scalar stores plus extracting the elements.  */
    9311          931 :                   inside_cost
    9312          931 :                     += record_stmt_cost (cost_vec, 1, vec_deconstruct,
    9313              :                                          slp_node, 0, vect_body);
    9314          931 :                   inside_cost
    9315          931 :                     += record_stmt_cost (cost_vec, cnunits, scalar_store,
    9316              :                                          slp_node, 0, vect_body);
    9317          931 :                   continue;
    9318          931 :                 }
    9319              : 
    9320          378 :               tree offset_vectype = TREE_TYPE (vec_offsets[0]);
    9321          378 :               unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
    9322          378 :               unsigned HOST_WIDE_INT const_offset_nunits
    9323          378 :                 = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
    9324          378 :               vec<constructor_elt, va_gc> *ctor_elts;
    9325          378 :               vec_alloc (ctor_elts, const_nunits);
    9326          378 :               gimple_seq stmts = NULL;
    9327          378 :               tree elt_type = TREE_TYPE (vectype);
    9328          378 :               unsigned HOST_WIDE_INT elt_size
    9329          378 :                 = tree_to_uhwi (TYPE_SIZE (elt_type));
    9330              :               /* We support offset vectors with more elements
    9331              :                  than the data vector for now.  */
    9332          378 :               unsigned HOST_WIDE_INT factor
    9333              :                 = const_offset_nunits / const_nunits;
    9334          378 :               vec_offset = vec_offsets[j / factor];
    9335          378 :               unsigned elt_offset
    9336          378 :                 = (j % factor) * const_nunits;
    9337          378 :               tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
    9338          378 :               tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
    9339          378 :               tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
    9340         1914 :               for (unsigned k = 0; k < const_nunits; ++k)
    9341              :                 {
    9342              :                   /* Compute the offsetted pointer.  */
    9343         1158 :                   tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
    9344              :                                           bitsize_int (k + elt_offset));
    9345         1158 :                   tree idx
    9346         2316 :                     = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
    9347         1158 :                                     vec_offset, TYPE_SIZE (idx_type), boff);
    9348         1158 :                   idx = gimple_convert (&stmts, sizetype, idx);
    9349         1158 :                   idx = gimple_build (&stmts, MULT_EXPR, sizetype,
    9350              :                                       idx, scale);
    9351         1158 :                   tree ptr
    9352         1158 :                     = gimple_build (&stmts, PLUS_EXPR,
    9353         1158 :                                     TREE_TYPE (dataref_ptr),
    9354              :                                     dataref_ptr, idx);
    9355         1158 :                   ptr = gimple_convert (&stmts, ptr_type_node, ptr);
    9356              :                   /* Extract the element to be stored.  */
    9357         1158 :                   tree elt
    9358         2316 :                     = gimple_build (&stmts, BIT_FIELD_REF,
    9359         1158 :                                     TREE_TYPE (vectype),
    9360         1158 :                                     vec_oprnd, TYPE_SIZE (elt_type),
    9361         1158 :                                     bitsize_int (k * elt_size));
    9362         1158 :                   gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    9363         1158 :                   stmts = NULL;
    9364         1158 :                   tree ref
    9365         1158 :                     = build2 (MEM_REF, ltype, ptr,
    9366              :                               build_int_cst (ref_type, 0));
    9367         1158 :                   new_stmt = gimple_build_assign (ref, elt);
    9368         1158 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    9369              :                 }
    9370              : 
    9371          378 :               slp_node->push_vec_def (new_stmt);
    9372              :             }
    9373              :         }
    9374              : 
    9375         1383 :       if (costing_p)
    9376              :         {
    9377          909 :           if (dump_enabled_p ())
    9378           78 :             dump_printf_loc (MSG_NOTE, vect_location,
    9379              :                              "vect_model_store_cost: inside_cost = %d, "
    9380              :                              "prologue_cost = %d .\n",
    9381              :                              inside_cost, prologue_cost);
    9382          909 :           SLP_TREE_TYPE (slp_node) = store_vec_info_type;
    9383          909 :           slp_node->data = new vect_load_store_data (std::move (ls));
    9384              :         }
    9385              : 
    9386         1383 :       return true;
    9387         1383 :     }
    9388              : 
    9389      1355015 :   gcc_assert (memory_access_type == VMAT_CONTIGUOUS
    9390              :               || memory_access_type == VMAT_CONTIGUOUS_DOWN
    9391              :               || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
    9392              : 
    9393      1355015 :   unsigned inside_cost = 0, prologue_cost = 0;
    9394              :   /* For costing some adjacent vector stores, we'd like to cost with
    9395              :      the total number of them once instead of cost each one by one. */
    9396      1355015 :   unsigned int n_adjacent_stores = 0;
    9397      1355015 :   auto_vec<tree> result_chain (group_size);
    9398      1355015 :   auto_vec<tree, 1> vec_oprnds;
    9399      1355015 :   gimple *new_stmt;
    9400      1355015 :   if (!costing_p)
    9401              :     {
    9402              :       /* Get vectorized arguments for SLP_NODE.  */
    9403       546073 :       vect_get_slp_defs (op_node, &vec_oprnds);
    9404       546073 :       vec_oprnd = vec_oprnds[0];
    9405       546073 :       if (mask_node)
    9406              :         {
    9407          470 :           vect_get_slp_defs (mask_node, &vec_masks);
    9408          470 :           vec_mask = vec_masks[0];
    9409              :         }
    9410              :     }
    9411              : 
    9412              :   /* We should have caught mismatched types earlier.  */
    9413       546073 :   gcc_assert (costing_p
    9414              :               || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
    9415      1355015 :   bool simd_lane_access_p
    9416      1355015 :       = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
    9417      1355015 :   if (!costing_p
    9418      1355015 :       && simd_lane_access_p
    9419         4365 :       && !loop_masks
    9420         4365 :       && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
    9421         4365 :       && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
    9422         4365 :       && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
    9423         4365 :       && integer_zerop (DR_INIT (first_dr_info->dr))
    9424      1359380 :       && alias_sets_conflict_p (get_alias_set (aggr_type),
    9425         4365 :                                 get_alias_set (TREE_TYPE (ref_type))))
    9426              :     {
    9427         4357 :       dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
    9428         4357 :       dataref_offset = build_int_cst (ref_type, 0);
    9429              :     }
    9430      1350658 :   else if (!costing_p)
    9431      1083424 :     dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
    9432              :                                             simd_lane_access_p ? loop : NULL,
    9433              :                                             offset, &dummy, gsi, NULL,
    9434              :                                             simd_lane_access_p, dr_increment);
    9435              : 
    9436      1355015 :   new_stmt = NULL;
    9437      1355015 :   gcc_assert (!grouped_store);
    9438      3012704 :   for (i = 0; i < vec_num; i++)
    9439              :     {
    9440      1657689 :       if (!costing_p)
    9441       677103 :         vec_oprnd = vec_oprnds[i];
    9442              : 
    9443      1657689 :       if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
    9444              :         {
    9445         3391 :           if (costing_p)
    9446         2241 :             inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
    9447              :                                              slp_node, 0, vect_body);
    9448              :           else
    9449              :             {
    9450         1150 :               tree perm_mask = perm_mask_for_reverse (vectype);
    9451         1150 :               tree new_temp = make_ssa_name (vectype);
    9452              : 
    9453              :               /* Generate the permute statement.  */
    9454         1150 :               gimple *perm_stmt
    9455         1150 :                 = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
    9456              :                                        vec_oprnd, perm_mask);
    9457         1150 :               vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
    9458              : 
    9459         1150 :               perm_stmt = SSA_NAME_DEF_STMT (new_temp);
    9460      1657689 :               vec_oprnd = new_temp;
    9461              :             }
    9462              :         }
    9463              : 
    9464      1657689 :       if (costing_p)
    9465              :         {
    9466       980586 :           n_adjacent_stores++;
    9467       980586 :           continue;
    9468              :         }
    9469              : 
    9470       677103 :       tree final_mask = NULL_TREE;
    9471       677103 :       tree final_len = NULL_TREE;
    9472       677103 :       tree bias = NULL_TREE;
    9473       677103 :       if (loop_masks)
    9474           83 :         final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
    9475              :                                          vec_num, vectype, i);
    9476       677103 :       if (vec_mask)
    9477          672 :         vec_mask = vec_masks[i];
    9478          672 :       if (vec_mask)
    9479          672 :         final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
    9480              :                                        vec_mask, gsi);
    9481              : 
    9482       677103 :       if (i > 0)
    9483              :         /* Bump the vector pointer.  */
    9484       131030 :         dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
    9485              :                                        dr_bump);
    9486              : 
    9487       677103 :       unsigned misalign;
    9488       677103 :       unsigned HOST_WIDE_INT align;
    9489       677103 :       align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
    9490       677103 :       if (alignment_support_scheme == dr_aligned)
    9491              :         misalign = 0;
    9492       310711 :       else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
    9493              :         {
    9494       162852 :           align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
    9495       162852 :           misalign = 0;
    9496              :         }
    9497              :       else
    9498       147859 :         misalign = misalignment;
    9499       677103 :       if (dataref_offset == NULL_TREE
    9500       671737 :           && TREE_CODE (dataref_ptr) == SSA_NAME)
    9501       188041 :         set_ssa_name_alignment (dataref_ptr, align, misalign);
    9502       677103 :       align = least_bit_hwi (misalign | align);
    9503              : 
    9504              :       /* Compute IFN when LOOP_LENS or final_mask valid.  */
    9505       677103 :       machine_mode vmode = TYPE_MODE (vectype);
    9506       677103 :       machine_mode new_vmode = vmode;
    9507       677103 :       internal_fn partial_ifn = IFN_LAST;
    9508       677103 :       if (loop_lens)
    9509              :         {
    9510            0 :           opt_machine_mode new_ovmode
    9511            0 :             = get_len_load_store_mode (vmode, false, &partial_ifn);
    9512            0 :           new_vmode = new_ovmode.require ();
    9513            0 :           unsigned factor
    9514            0 :             = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
    9515            0 :           final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
    9516              :                                          vec_num, vectype, i, factor, true);
    9517              :         }
    9518       677103 :       else if (final_mask)
    9519              :         {
    9520          688 :           if (!can_vec_mask_load_store_p (vmode,
    9521          688 :                                           TYPE_MODE (TREE_TYPE (final_mask)),
    9522              :                                           false, &partial_ifn))
    9523            0 :             gcc_unreachable ();
    9524              :         }
    9525              : 
    9526       677103 :       if (partial_ifn == IFN_MASK_LEN_STORE)
    9527              :         {
    9528            0 :           if (!final_len)
    9529              :             {
    9530              :               /* Pass VF value to 'len' argument of
    9531              :                  MASK_LEN_STORE if LOOP_LENS is invalid.  */
    9532            0 :               final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
    9533              :             }
    9534            0 :           if (!final_mask)
    9535              :             {
    9536              :               /* Pass all ones value to 'mask' argument of
    9537              :                  MASK_LEN_STORE if final_mask is invalid.  */
    9538            0 :               mask_vectype = truth_type_for (vectype);
    9539            0 :               final_mask = build_minus_one_cst (mask_vectype);
    9540              :             }
    9541              :         }
    9542       677103 :       if (final_len)
    9543              :         {
    9544            0 :           signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    9545            0 :           bias = build_int_cst (intQI_type_node, biasval);
    9546              :         }
    9547              : 
    9548              :       /* Arguments are ready.  Create the new vector stmt.  */
    9549            0 :       if (final_len)
    9550              :         {
    9551            0 :           gcall *call;
    9552            0 :           tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
    9553              :           /* Need conversion if it's wrapped with VnQI.  */
    9554            0 :           if (vmode != new_vmode)
    9555              :             {
    9556            0 :               tree new_vtype
    9557            0 :                 = build_vector_type_for_mode (unsigned_intQI_type_node,
    9558              :                                               new_vmode);
    9559            0 :               tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
    9560            0 :               vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
    9561            0 :               gassign *new_stmt
    9562            0 :                 = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
    9563            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    9564            0 :               vec_oprnd = var;
    9565              :             }
    9566              : 
    9567            0 :           if (partial_ifn == IFN_MASK_LEN_STORE)
    9568            0 :             call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
    9569              :                                                dataref_ptr, ptr, final_mask,
    9570              :                                                final_len, bias, vec_oprnd);
    9571              :           else
    9572            0 :             call = gimple_build_call_internal (IFN_LEN_STORE, 5,
    9573              :                                                dataref_ptr, ptr, final_len,
    9574              :                                                bias, vec_oprnd);
    9575            0 :           gimple_call_set_nothrow (call, true);
    9576            0 :           vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    9577            0 :           new_stmt = call;
    9578              :         }
    9579       677103 :       else if (final_mask)
    9580              :         {
    9581          688 :           tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
    9582          688 :           gcall *call
    9583          688 :             = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
    9584              :                                           ptr, final_mask, vec_oprnd);
    9585          688 :           gimple_call_set_nothrow (call, true);
    9586          688 :           vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
    9587          688 :           new_stmt = call;
    9588              :         }
    9589              :       else
    9590              :         {
    9591       676415 :           data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr,
    9592              :                                   dataref_offset ? dataref_offset
    9593              :                                   : build_int_cst (ref_type, 0));
    9594       676415 :           if (alignment_support_scheme == dr_aligned
    9595       676415 :               && align >= TYPE_ALIGN_UNIT (vectype))
    9596              :             ;
    9597              :           else
    9598       310184 :             TREE_TYPE (data_ref)
    9599       620368 :               = build_aligned_type (TREE_TYPE (data_ref),
    9600              :                                     align * BITS_PER_UNIT);
    9601       676415 :           vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
    9602       676415 :           new_stmt = gimple_build_assign (data_ref, vec_oprnd);
    9603       676415 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
    9604              :         }
    9605              :     }
    9606              : 
    9607      1355015 :   if (costing_p)
    9608              :     {
    9609       808942 :       if (n_adjacent_stores > 0)
    9610       808942 :         vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
    9611              :                              alignment_support_scheme, misalignment,
    9612              :                              &inside_cost, cost_vec);
    9613              : 
    9614              :       /* When vectorizing a store into the function result assign
    9615              :          a penalty if the function returns in a multi-register location.
    9616              :          In this case we assume we'll end up with having to spill the
    9617              :          vector result and do piecewise loads as a conservative estimate.  */
    9618       808942 :       tree base = get_base_address (STMT_VINFO_DATA_REF (stmt_info)->ref);
    9619       808942 :       if (base
    9620       808942 :           && (TREE_CODE (base) == RESULT_DECL
    9621       758521 :               || (DECL_P (base) && cfun_returns (base)))
    9622       871186 :           && !aggregate_value_p (base, cfun->decl))
    9623              :         {
    9624        11237 :           rtx reg = hard_function_value (TREE_TYPE (base), cfun->decl, 0, 1);
    9625              :           /* ???  Handle PARALLEL in some way.  */
    9626        11237 :           if (REG_P (reg))
    9627              :             {
    9628        11028 :               int nregs = hard_regno_nregs (REGNO (reg), GET_MODE (reg));
    9629              :               /* Assume that a single reg-reg move is possible and cheap,
    9630              :                  do not account for vector to gp register move cost.  */
    9631        11028 :               if (nregs > 1)
    9632              :                 {
    9633              :                   /* Spill.  */
    9634        10151 :                   prologue_cost
    9635        10151 :                     += record_stmt_cost (cost_vec, 1, vector_store,
    9636              :                                          slp_node, 0, vect_epilogue);
    9637              :                   /* Loads.  */
    9638        10151 :                   prologue_cost
    9639        10151 :                     += record_stmt_cost (cost_vec, nregs, scalar_load,
    9640              :                                          slp_node, 0, vect_epilogue);
    9641              :                 }
    9642              :             }
    9643              :         }
    9644       808942 :       if (dump_enabled_p ())
    9645        14630 :         dump_printf_loc (MSG_NOTE, vect_location,
    9646              :                          "vect_model_store_cost: inside_cost = %d, "
    9647              :                          "prologue_cost = %d .\n",
    9648              :                          inside_cost, prologue_cost);
    9649              : 
    9650       808942 :       SLP_TREE_TYPE (slp_node) = store_vec_info_type;
    9651       808942 :       slp_node->data = new vect_load_store_data (std::move (ls));
    9652              :     }
    9653              : 
    9654      1355015 :   return true;
    9655      1387924 : }
    9656              : 
    9657              : /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
    9658              :    VECTOR_CST mask.  No checks are made that the target platform supports the
    9659              :    mask, so callers may wish to test can_vec_perm_const_p separately, or use
    9660              :    vect_gen_perm_mask_checked.  */
    9661              : 
    9662              : tree
    9663        62072 : vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
    9664              : {
    9665        62072 :   tree mask_type;
    9666              : 
    9667        62072 :   poly_uint64 nunits = sel.length ();
    9668        62072 :   gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
    9669              : 
    9670        62072 :   mask_type = build_vector_type (ssizetype, nunits);
    9671        62072 :   return vec_perm_indices_to_tree (mask_type, sel);
    9672              : }
    9673              : 
    9674              : /* Checked version of vect_gen_perm_mask_any.  Asserts can_vec_perm_const_p,
    9675              :    i.e. that the target supports the pattern _for arbitrary input vectors_.  */
    9676              : 
    9677              : tree
    9678        59387 : vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
    9679              : {
    9680        59387 :   machine_mode vmode = TYPE_MODE (vectype);
    9681        59387 :   gcc_assert (can_vec_perm_const_p (vmode, vmode, sel));
    9682        59387 :   return vect_gen_perm_mask_any (vectype, sel);
    9683              : }
    9684              : 
    9685              : /* Given a vector variable X and Y, that was generated for the scalar
    9686              :    STMT_INFO, generate instructions to permute the vector elements of X and Y
    9687              :    using permutation mask MASK_VEC, insert them at *GSI and return the
    9688              :    permuted vector variable.  */
    9689              : 
    9690              : static tree
    9691         1488 : permute_vec_elements (vec_info *vinfo,
    9692              :                       tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
    9693              :                       gimple_stmt_iterator *gsi)
    9694              : {
    9695         1488 :   tree vectype = TREE_TYPE (x);
    9696         1488 :   tree perm_dest, data_ref;
    9697         1488 :   gimple *perm_stmt;
    9698              : 
    9699         1488 :   tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
    9700         1488 :   if (scalar_dest && TREE_CODE (scalar_dest) == SSA_NAME)
    9701         1488 :     perm_dest = vect_create_destination_var (scalar_dest, vectype);
    9702              :   else
    9703            0 :     perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
    9704         1488 :   data_ref = make_ssa_name (perm_dest);
    9705              : 
    9706              :   /* Generate the permute statement.  */
    9707         1488 :   perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
    9708         1488 :   vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
    9709              : 
    9710         1488 :   return data_ref;
    9711              : }
    9712              : 
    9713              : /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
    9714              :    inserting them on the loops preheader edge.  Returns true if we
    9715              :    were successful in doing so (and thus STMT_INFO can be moved then),
    9716              :    otherwise returns false.  HOIST_P indicates if we want to hoist the
    9717              :    definitions of all SSA uses, it would be false when we are costing.  */
    9718              : 
    9719              : static bool
    9720         4032 : hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
    9721              : {
    9722         4032 :   ssa_op_iter i;
    9723         4032 :   use_operand_p use_p;
    9724         4032 :   auto_vec<use_operand_p, 8> to_hoist;
    9725              : 
    9726         7640 :   FOR_EACH_SSA_USE_OPERAND (use_p, stmt, i, SSA_OP_USE)
    9727              :     {
    9728         3638 :       gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
    9729         3638 :       if (!gimple_nop_p (def_stmt)
    9730         3638 :           && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
    9731              :         {
    9732              :           /* Make sure we don't need to recurse.  While we could do
    9733              :              so in simple cases when there are more complex use webs
    9734              :              we don't have an easy way to preserve stmt order to fulfil
    9735              :              dependencies within them.  */
    9736           87 :           tree op2;
    9737           87 :           ssa_op_iter i2;
    9738           87 :           if (gimple_code (def_stmt) == GIMPLE_PHI
    9739           87 :               || (single_ssa_def_operand (def_stmt, SSA_OP_DEF)
    9740              :                   == NULL_DEF_OPERAND_P))
    9741           30 :             return false;
    9742          176 :           FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
    9743              :             {
    9744          119 :               gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
    9745          119 :               if (!gimple_nop_p (def_stmt2)
    9746          119 :                   && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
    9747              :                 return false;
    9748              :             }
    9749           57 :           to_hoist.safe_push (use_p);
    9750              :         }
    9751              :     }
    9752              : 
    9753         8034 :   if (to_hoist.is_empty ())
    9754              :     return true;
    9755              : 
    9756           33 :   if (!hoist_p)
    9757              :     return true;
    9758              : 
    9759              :   /* Instead of moving defs we copy them so we can zero their UID to not
    9760              :      confuse dominance queries in the preheader.  */
    9761            3 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    9762           12 :   for (use_operand_p use_p : to_hoist)
    9763              :     {
    9764            3 :       gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
    9765            3 :       gimple *copy = gimple_copy (def_stmt);
    9766            3 :       assign_discriminators_to_stmt
    9767            3 :         (copy, 0, allocate_copyid_base (gimple_location (copy), 1));
    9768            3 :       gimple_set_uid (copy, 0);
    9769            3 :       def_operand_p def_p = single_ssa_def_operand (def_stmt, SSA_OP_DEF);
    9770            3 :       tree new_def = duplicate_ssa_name (DEF_FROM_PTR (def_p), copy);
    9771            3 :       update_stmt (copy);
    9772            3 :       def_p = single_ssa_def_operand (copy, SSA_OP_DEF);
    9773            3 :       SET_DEF (def_p, new_def);
    9774            3 :       SET_USE (use_p, new_def);
    9775            3 :       gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
    9776              :     }
    9777              : 
    9778              :   return true;
    9779         4032 : }
    9780              : 
    9781              : /* vectorizable_load.
    9782              : 
    9783              :    Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
    9784              :    that can be vectorized.
    9785              :    If COST_VEC is passed, calculate costs but don't change anything,
    9786              :    otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
    9787              :    it, and insert it at GSI.
    9788              :    Return true if STMT_INFO is vectorizable in this way.  */
    9789              : 
    9790              : static bool
    9791      2240149 : vectorizable_load (vec_info *vinfo,
    9792              :                    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    9793              :                    slp_tree slp_node,
    9794              :                    stmt_vector_for_cost *cost_vec)
    9795              : {
    9796      2240149 :   tree scalar_dest;
    9797      2240149 :   tree vec_dest = NULL;
    9798      2240149 :   tree data_ref = NULL;
    9799      2240149 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    9800      2240149 :   class loop *loop = NULL;
    9801      2240149 :   class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
    9802      2240149 :   bool nested_in_vect_loop = false;
    9803      2240149 :   tree elem_type;
    9804              :   /* Avoid false positive uninitialized warning, see PR110652.  */
    9805      2240149 :   tree new_temp = NULL_TREE;
    9806      2240149 :   machine_mode mode;
    9807      2240149 :   tree dummy;
    9808      2240149 :   tree dataref_ptr = NULL_TREE;
    9809      2240149 :   tree dataref_offset = NULL_TREE;
    9810      2240149 :   int i, j;
    9811      2240149 :   unsigned int group_size;
    9812      2240149 :   poly_uint64 group_gap_adj;
    9813      2240149 :   tree msq = NULL_TREE, lsq;
    9814      2240149 :   tree realignment_token = NULL_TREE;
    9815      2240149 :   gphi *phi = NULL;
    9816      2240149 :   bool grouped_load = false;
    9817      2240149 :   stmt_vec_info first_stmt_info;
    9818      2240149 :   stmt_vec_info first_stmt_info_for_drptr = NULL;
    9819      2240149 :   bool compute_in_loop = false;
    9820      2240149 :   class loop *at_loop;
    9821      2240149 :   int vec_num;
    9822      2240149 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    9823      2240149 :   poly_uint64 vf;
    9824      2240149 :   tree aggr_type;
    9825      2240149 :   tree ref_type;
    9826      2240149 :   enum vect_def_type mask_dt = vect_unknown_def_type;
    9827      2240149 :   enum vect_def_type els_dt = vect_unknown_def_type;
    9828              : 
    9829      2240149 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
    9830              :     return false;
    9831              : 
    9832      2240149 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    9833       251571 :       && cost_vec)
    9834              :     return false;
    9835              : 
    9836      1988578 :   if (!STMT_VINFO_DATA_REF (stmt_info))
    9837              :     return false;
    9838              : 
    9839      1579798 :   tree mask_vectype = NULL_TREE;
    9840      1579798 :   tree els = NULL_TREE; tree els_vectype = NULL_TREE;
    9841              : 
    9842      1579798 :   int mask_index = -1;
    9843      1579798 :   int els_index = -1;
    9844      1579798 :   slp_tree mask_node = NULL;
    9845      1579798 :   slp_tree els_op = NULL;
    9846      1579798 :   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
    9847              :     {
    9848      1575659 :       scalar_dest = gimple_assign_lhs (assign);
    9849      1575659 :       if (TREE_CODE (scalar_dest) != SSA_NAME)
    9850              :         return false;
    9851              : 
    9852       739558 :       tree_code code = gimple_assign_rhs_code (assign);
    9853       739558 :       if (code != ARRAY_REF
    9854       739558 :           && code != BIT_FIELD_REF
    9855       739558 :           && code != INDIRECT_REF
    9856       507998 :           && code != COMPONENT_REF
    9857       507998 :           && code != IMAGPART_EXPR
    9858       367072 :           && code != REALPART_EXPR
    9859       367072 :           && code != MEM_REF
    9860          285 :           && TREE_CODE_CLASS (code) != tcc_declaration)
    9861              :         return false;
    9862              :     }
    9863              :   else
    9864              :     {
    9865         4139 :       gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
    9866         4139 :       if (!call || !gimple_call_internal_p (call))
    9867              :         return false;
    9868              : 
    9869         4139 :       internal_fn ifn = gimple_call_internal_fn (call);
    9870         4139 :       if (!internal_load_fn_p (ifn))
    9871              :         return false;
    9872              : 
    9873         2796 :       scalar_dest = gimple_call_lhs (call);
    9874         2796 :       if (!scalar_dest)
    9875              :         return false;
    9876              : 
    9877         2796 :       mask_index = internal_fn_mask_index (ifn);
    9878         2796 :       if (mask_index >= 0)
    9879         2796 :         mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
    9880         2796 :       if (mask_index >= 0
    9881         2796 :           && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
    9882              :                                       &mask_node, &mask_dt, &mask_vectype))
    9883              :         return false;
    9884              : 
    9885         2796 :       els_index = internal_fn_else_index (ifn);
    9886         2796 :       if (els_index >= 0)
    9887         2796 :         els_index = vect_slp_child_index_for_operand (stmt_info, els_index);
    9888         2796 :       if (els_index >= 0
    9889         2796 :           && !vect_is_simple_use (vinfo, slp_node, els_index,
    9890              :                                   &els, &els_op, &els_dt, &els_vectype))
    9891              :         return false;
    9892              :     }
    9893              : 
    9894       742287 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    9895       742287 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    9896              : 
    9897       742287 :   if (loop_vinfo)
    9898              :     {
    9899       523676 :       loop = LOOP_VINFO_LOOP (loop_vinfo);
    9900       523676 :       nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
    9901       523676 :       vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    9902              :     }
    9903              :   else
    9904              :     vf = 1;
    9905              : 
    9906       742287 :   vec_num = vect_get_num_copies (vinfo, slp_node);
    9907              : 
    9908              :   /* FORNOW. This restriction should be relaxed.  */
    9909       742287 :   if (nested_in_vect_loop && vec_num > 1)
    9910              :     {
    9911          319 :       if (dump_enabled_p ())
    9912           69 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9913              :                          "multiple types in nested loop.\n");
    9914              :       return false;
    9915              :     }
    9916              : 
    9917       741968 :   elem_type = TREE_TYPE (vectype);
    9918       741968 :   mode = TYPE_MODE (vectype);
    9919              : 
    9920              :   /* FORNOW. In some cases can vectorize even if data-type not supported
    9921              :     (e.g. - data copies).  */
    9922       741968 :   if (!can_implement_p (mov_optab, mode))
    9923              :     {
    9924            0 :       if (dump_enabled_p ())
    9925            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9926              :                          "Aligned load, but unsupported type.\n");
    9927              :       return false;
    9928              :     }
    9929              : 
    9930              :   /* Check if the load is a part of an interleaving chain.  */
    9931       741968 :   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    9932              :     {
    9933       310874 :       grouped_load = true;
    9934              :       /* FORNOW */
    9935       310874 :       gcc_assert (!nested_in_vect_loop);
    9936       310874 :       gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
    9937              : 
    9938       310874 :       first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
    9939       310874 :       group_size = DR_GROUP_SIZE (first_stmt_info);
    9940              : 
    9941              :       /* Invalidate assumptions made by dependence analysis when vectorization
    9942              :          on the unrolled body effectively re-orders stmts.  */
    9943       310874 :       if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
    9944       310874 :           && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
    9945              :                        STMT_VINFO_MIN_NEG_DIST (stmt_info)))
    9946              :         {
    9947            0 :           if (dump_enabled_p ())
    9948            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9949              :                              "cannot perform implicit CSE when performing "
    9950              :                              "group loads with negative dependence distance\n");
    9951              :           return false;
    9952              :         }
    9953              :     }
    9954              :   else
    9955              :     group_size = 1;
    9956              : 
    9957       741968 :   vect_load_store_data _ls_data{};
    9958       741968 :   vect_load_store_data &ls = slp_node->get_data (_ls_data);
    9959       741968 :   if (cost_vec
    9960       741968 :       && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
    9961              :                                VLS_LOAD, &ls))
    9962              :     return false;
    9963              :   /* Temporary aliases to analysis data, should not be modified through
    9964              :      these.  */
    9965       623767 :   const vect_memory_access_type memory_access_type = ls.memory_access_type;
    9966       623767 :   const dr_alignment_support alignment_support_scheme
    9967              :     = ls.alignment_support_scheme;
    9968       623767 :   const int misalignment = ls.misalignment;
    9969       623767 :   const poly_int64 poffset = ls.poffset;
    9970       623767 :   const vec<int> &elsvals = ls.elsvals;
    9971              : 
    9972       623767 :   int maskload_elsval = 0;
    9973       623767 :   bool need_zeroing = false;
    9974              : 
    9975              :   /* We might need to explicitly zero inactive elements if there are
    9976              :      padding bits in the type that might leak otherwise.
    9977              :      Refer to PR115336.  */
    9978       623767 :   tree scalar_type = TREE_TYPE (scalar_dest);
    9979       623767 :   bool type_mode_padding_p
    9980      1247534 :     = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
    9981              : 
    9982       623767 :   if (TREE_CODE (scalar_type) == BITINT_TYPE
    9983       623767 :       && !type_has_mode_precision_p (scalar_type))
    9984              :     {
    9985          140 :       if (dump_enabled_p ())
    9986           18 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9987              :                          "_BitInt loads are not fully supported.\n");
    9988              :       return false;
    9989              :     }
    9990              : 
    9991       623627 :   if (slp_node->ldst_lanes
    9992            0 :       && memory_access_type != VMAT_LOAD_STORE_LANES)
    9993              :     {
    9994            0 :       if (dump_enabled_p ())
    9995            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9996              :                          "discovered load-lane but cannot use it.\n");
    9997              :       return false;
    9998              :     }
    9999              : 
   10000       623627 :   if (mask_node)
   10001              :     {
   10002         2670 :       if (memory_access_type == VMAT_CONTIGUOUS)
   10003              :         {
   10004         2105 :           machine_mode vec_mode = TYPE_MODE (vectype);
   10005          730 :           if (!VECTOR_MODE_P (vec_mode)
   10006         4210 :               || !can_vec_mask_load_store_p (vec_mode,
   10007         2105 :                                              TYPE_MODE (mask_vectype),
   10008              :                                              true, NULL, &ls.elsvals))
   10009              :             return false;
   10010              :         }
   10011          565 :       else if (memory_access_type == VMAT_ELEMENTWISE
   10012          565 :                || memory_access_type == VMAT_STRIDED_SLP)
   10013              :         {
   10014            0 :           if (dump_enabled_p ())
   10015            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10016              :                              "unsupported masked strided access.\n");
   10017              :           return false;
   10018              :         }
   10019          565 :       else if (memory_access_type != VMAT_LOAD_STORE_LANES
   10020          565 :                && !mat_gather_scatter_p (memory_access_type))
   10021              :         {
   10022           32 :           if (dump_enabled_p ())
   10023            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10024              :                              "unsupported access type for masked load.\n");
   10025              :           return false;
   10026              :         }
   10027          533 :       else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
   10028              :         {
   10029          212 :           if (dump_enabled_p ())
   10030           28 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10031              :                              "unsupported masked emulated gather.\n");
   10032              :           return false;
   10033              :         }
   10034              :     }
   10035              : 
   10036       623028 :   bool costing_p = cost_vec;
   10037              : 
   10038       623028 :   if (costing_p) /* transformation not required.  */
   10039              :     {
   10040       465362 :       if (loop_vinfo
   10041       329757 :           && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
   10042       224730 :         check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
   10043              :                                               VLS_LOAD, group_size, &ls,
   10044              :                                               mask_node, &ls.elsvals);
   10045              : 
   10046              :       /* If the type needs padding we must zero inactive elements.
   10047              :          Check if we can do that with a VEC_COND_EXPR and store the
   10048              :          elsval we choose in MASKLOAD_ELSVAL.  */
   10049       465362 :       if (ls.elsvals.length ()
   10050        62129 :           && type_mode_padding_p
   10051            7 :           && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
   10052        62129 :           && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
   10053              :         {
   10054            0 :           if (dump_enabled_p ())
   10055            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10056              :                              "cannot zero inactive elements.\n");
   10057              :           return false;
   10058              :         }
   10059              : 
   10060       465362 :       if (mask_node
   10061       465362 :           && !vect_maybe_update_slp_op_vectype (mask_node,
   10062              :                                                 mask_vectype))
   10063              :         {
   10064            0 :           if (dump_enabled_p ())
   10065            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10066              :                              "incompatible vector types for invariants\n");
   10067              :           return false;
   10068              :         }
   10069              : 
   10070       465362 :       if (dump_enabled_p ()
   10071        27327 :           && memory_access_type != VMAT_ELEMENTWISE
   10072        27216 :           && !mat_gather_scatter_p (memory_access_type)
   10073        26901 :           && memory_access_type != VMAT_STRIDED_SLP
   10074        26901 :           && memory_access_type != VMAT_INVARIANT
   10075       491292 :           && alignment_support_scheme != dr_aligned)
   10076        10774 :         dump_printf_loc (MSG_NOTE, vect_location,
   10077              :                          "Vectorizing an unaligned access.\n");
   10078              : 
   10079       465362 :       if (memory_access_type == VMAT_LOAD_STORE_LANES)
   10080            0 :         vinfo->any_known_not_updated_vssa = true;
   10081              :     }
   10082              : 
   10083              :   /* For now just use the first available else value.
   10084              :      get_supported_else_vals tries MASK_LOAD_ELSE_ZERO first so we will
   10085              :      select it here if it is supported.  */
   10086       623028 :   if (elsvals.length ())
   10087        85783 :     maskload_elsval = *elsvals.begin ();
   10088              : 
   10089       623028 :   if (dump_enabled_p () && !costing_p)
   10090        17214 :     dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n");
   10091              : 
   10092              :   /* Transform.  */
   10093              : 
   10094       623028 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
   10095       623028 :   if (!costing_p)
   10096       157666 :     ensure_base_align (dr_info);
   10097              : 
   10098       623028 :   if (memory_access_type == VMAT_INVARIANT)
   10099              :     {
   10100         5942 :       gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
   10101              :       /* If we have versioned for aliasing or the loop doesn't
   10102              :          have any data dependencies that would preclude this,
   10103              :          then we are sure this is a loop invariant load and
   10104              :          thus we can insert it on the preheader edge.
   10105              :          TODO: hoist_defs_of_uses should ideally be computed
   10106              :          once at analysis time, remembered and used in the
   10107              :          transform time.  */
   10108        11884 :       bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
   10109         5942 :                       && !nested_in_vect_loop);
   10110              : 
   10111         5942 :       bool uniform_p = true;
   10112        24592 :       for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
   10113              :         {
   10114              :           /* It is unsafe to hoist a conditional load over the conditions that
   10115              :              make it valid.  When early break this means that any invariant load
   10116              :              can't be hoisted unless it's in the loop header or if we know
   10117              :              something else has verified the load is valid to do.  Alignment
   10118              :              peeling would do this since getting through the prologue means the
   10119              :              load was done at least once and so the vector main body is free to
   10120              :              hoist it.  However today GCC will hoist the load above the PFA
   10121              :              loop.  As such that makes it still invalid and so we can't allow it
   10122              :              today.  */
   10123         6766 :           if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
   10124         2912 :               && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo))
   10125         9642 :               && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo)))
   10126         2876 :                   != loop->header)
   10127              :             {
   10128         2778 :               if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
   10129         2778 :                   && dump_enabled_p ())
   10130            6 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10131              :                              "not hoisting invariant load due to early break"
   10132              :                              "constraints\n");
   10133         2772 :               else if (dump_enabled_p ())
   10134           26 :                dump_printf_loc (MSG_NOTE, vect_location,
   10135              :                              "not hoisting invariant load due to early break"
   10136              :                              "constraints\n");
   10137              :             hoist_p = false;
   10138              :           }
   10139              : 
   10140         4020 :           hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false);
   10141         6766 :           if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0])
   10142          321 :             uniform_p = false;
   10143              :         }
   10144         5942 :       if (costing_p)
   10145              :         {
   10146         5112 :           if (!uniform_p && (!hoist_p || !vf.is_constant ()))
   10147              :             {
   10148            0 :               if (dump_enabled_p ())
   10149            0 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10150              :                                  "not vectorizing non-uniform invariant "
   10151              :                                  "load\n");
   10152              :               return false;
   10153              :             }
   10154         3292 :           enum vect_cost_model_location cost_loc
   10155         5112 :             = hoist_p ? vect_prologue : vect_body;
   10156         5236 :           unsigned int cost = record_stmt_cost (cost_vec,
   10157              :                                                 uniform_p
   10158          124 :                                                 ? 1 : SLP_TREE_LANES (slp_node),
   10159              :                                                 scalar_load,
   10160              :                                                 slp_node, 0, cost_loc);
   10161         5112 :           cost += record_stmt_cost (cost_vec, 1,
   10162              :                                     uniform_p ? scalar_to_vec : vec_construct,
   10163              :                                     slp_node, 0, cost_loc);
   10164         5112 :           unsigned int prologue_cost = hoist_p ? cost : 0;
   10165         3292 :           unsigned int inside_cost = hoist_p ? 0 : cost;
   10166         5112 :           if (dump_enabled_p ())
   10167          560 :             dump_printf_loc (MSG_NOTE, vect_location,
   10168              :                              "vect_model_load_cost: inside_cost = %d, "
   10169              :                              "prologue_cost = %d .\n",
   10170              :                              inside_cost, prologue_cost);
   10171         5112 :           SLP_TREE_TYPE (slp_node) = load_vec_info_type;
   10172         5112 :           slp_node->data = new vect_load_store_data (std::move (ls));
   10173         5112 :           return true;
   10174              :         }
   10175          830 :       if (hoist_p)
   10176              :         {
   10177              :           /* ???  For non-uniform lanes there could be still duplicates.
   10178              :              We're leaving those to post-vectorizer CSE for the moment.  */
   10179          629 :           auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node));
   10180         2054 :           for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
   10181              :             {
   10182          733 :               gassign *stmt = as_a <gassign *> (sinfo->stmt);
   10183          733 :               if (dump_enabled_p ())
   10184          379 :                 dump_printf_loc (MSG_NOTE, vect_location,
   10185              :                                  "hoisting out of the vectorized loop: %G",
   10186              :                                  (gimple *) stmt);
   10187          733 :               scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt));
   10188          733 :               tree rhs = unshare_expr (gimple_assign_rhs1 (stmt));
   10189          733 :               edge pe = loop_preheader_edge (loop);
   10190          733 :               gphi *vphi = get_virtual_phi (loop->header);
   10191          733 :               tree vuse;
   10192          733 :               if (vphi)
   10193          726 :                 vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
   10194              :               else
   10195            7 :                 vuse = gimple_vuse (gsi_stmt (*gsi));
   10196          733 :               gimple *new_stmt = gimple_build_assign (scalar_dest, rhs);
   10197          733 :               gimple_set_vuse (new_stmt, vuse);
   10198          733 :               gsi_insert_on_edge_immediate (pe, new_stmt);
   10199          733 :               hoist_defs_of_uses (new_stmt, loop, true);
   10200          733 :               if (!useless_type_conversion_p (TREE_TYPE (vectype),
   10201          733 :                                               TREE_TYPE (scalar_dest)))
   10202              :                 {
   10203           10 :                   tree tem = make_ssa_name (TREE_TYPE (vectype));
   10204           10 :                   new_stmt = gimple_build_assign (tem,
   10205              :                                                   NOP_EXPR, scalar_dest);
   10206           10 :                   gsi_insert_on_edge_immediate (pe, new_stmt);
   10207           10 :                   scalar_dest = tem;
   10208              :                 }
   10209          733 :               scalar_defs.quick_push (scalar_dest);
   10210          733 :               if (uniform_p)
   10211              :                 break;
   10212              :             }
   10213          629 :           if (!uniform_p)
   10214              :             {
   10215           63 :               unsigned const_nunits
   10216           63 :                 = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
   10217          144 :               for (j = 0; j < (int) vec_num; ++j)
   10218              :                 {
   10219           81 :                   vec<constructor_elt, va_gc> *v = NULL;
   10220           81 :                   vec_safe_reserve (v, const_nunits, true);
   10221          522 :                   for (unsigned i = 0; i < const_nunits; ++i)
   10222              :                     {
   10223          360 :                       unsigned def_idx
   10224          360 :                         = (j * const_nunits + i) % SLP_TREE_LANES (slp_node);
   10225          360 :                       CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
   10226              :                                               scalar_defs[def_idx]);
   10227              :                     }
   10228           81 :                   scalar_dest = build_constructor (vectype, v);
   10229           81 :                   new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
   10230              :                                                vectype, NULL);
   10231           81 :                   slp_node->push_vec_def (new_temp);
   10232              :                 }
   10233           63 :               return true;
   10234              :             }
   10235          566 :           new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
   10236              :                                        vectype, NULL);
   10237          629 :         }
   10238              :       else
   10239              :         {
   10240          201 :           gcc_assert (uniform_p);
   10241          201 :           gimple_stmt_iterator gsi2 = *gsi;
   10242          201 :           gsi_next (&gsi2);
   10243          201 :           new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
   10244              :                                        vectype, &gsi2);
   10245              :         }
   10246         1609 :       for (j = 0; j < (int) vec_num; ++j)
   10247          842 :         slp_node->push_vec_def (new_temp);
   10248              :       return true;
   10249              :     }
   10250              : 
   10251       617086 :   if (memory_access_type == VMAT_ELEMENTWISE
   10252       617086 :       || memory_access_type == VMAT_STRIDED_SLP)
   10253              :     {
   10254        24458 :       gimple_stmt_iterator incr_gsi;
   10255        24458 :       bool insert_after;
   10256        24458 :       tree offvar = NULL_TREE;
   10257        24458 :       tree ivstep;
   10258        24458 :       tree running_off;
   10259        24458 :       vec<constructor_elt, va_gc> *v = NULL;
   10260        24458 :       tree stride_base, stride_step = NULL_TREE, alias_off;
   10261              :       /* Checked by get_load_store_type.  */
   10262        24458 :       unsigned int const_nunits = nunits.to_constant ();
   10263        24458 :       unsigned HOST_WIDE_INT cst_offset = 0;
   10264        24458 :       tree dr_offset;
   10265        24458 :       unsigned int inside_cost = 0;
   10266              : 
   10267        24458 :       gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
   10268        24458 :       gcc_assert (!nested_in_vect_loop);
   10269              : 
   10270        24458 :       if (grouped_load)
   10271              :         {
   10272              :           /* If we elided a consecutive load permutation, don't
   10273              :              use the original first statement (which could be elided)
   10274              :              but the one the load permutation starts with.
   10275              :              This ensures the stride_base below is correct.  */
   10276        11150 :           if (!ls.subchain_p)
   10277        11068 :             first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
   10278              :           else
   10279           82 :             first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
   10280        11150 :           first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
   10281        11150 :           ref_type = get_group_alias_ptr_type (first_stmt_info);
   10282              :         }
   10283              :       else
   10284              :         {
   10285        13308 :           first_stmt_info = stmt_info;
   10286        13308 :           first_dr_info = dr_info;
   10287        13308 :           ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
   10288              :         }
   10289              : 
   10290        24458 :       if (grouped_load)
   10291              :         {
   10292        11150 :           if (memory_access_type == VMAT_STRIDED_SLP)
   10293              :             {
   10294              :               /* If we elided a consecutive load permutation, adjust
   10295              :                  the group size here.  */
   10296         4291 :               if (!ls.subchain_p)
   10297         4209 :                 group_size = DR_GROUP_SIZE (first_stmt_info);
   10298              :               else
   10299           82 :                 group_size = SLP_TREE_LANES (slp_node);
   10300              :             }
   10301              :           else /* VMAT_ELEMENTWISE */
   10302         6859 :             group_size = SLP_TREE_LANES (slp_node);
   10303              :         }
   10304              :       else
   10305              :         group_size = 1;
   10306              : 
   10307        24458 :       if (!costing_p)
   10308              :         {
   10309         3563 :           dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
   10310         3563 :           stride_base = fold_build_pointer_plus (
   10311              :             DR_BASE_ADDRESS (first_dr_info->dr),
   10312              :             size_binop (PLUS_EXPR, convert_to_ptrofftype (dr_offset),
   10313              :                         convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
   10314         3563 :           stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
   10315              : 
   10316              :           /* For a load with loop-invariant (but other than power-of-2)
   10317              :              stride (i.e. not a grouped access) like so:
   10318              : 
   10319              :                for (i = 0; i < n; i += stride)
   10320              :                  ... = array[i];
   10321              : 
   10322              :              we generate a new induction variable and new accesses to
   10323              :              form a new vector (or vectors, depending on ncopies):
   10324              : 
   10325              :                for (j = 0; ; j += VF*stride)
   10326              :                  tmp1 = array[j];
   10327              :                  tmp2 = array[j + stride];
   10328              :                  ...
   10329              :                  vectemp = {tmp1, tmp2, ...}
   10330              :              */
   10331              : 
   10332         3563 :           tree increment = fold_convert (TREE_TYPE (stride_step),
   10333              :                                          LOOP_VINFO_IV_INCREMENT (loop_vinfo));
   10334         3563 :           ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
   10335              :                                 stride_step, increment);
   10336              : 
   10337         3563 :           standard_iv_increment_position (loop, &incr_gsi, &insert_after);
   10338              : 
   10339         3563 :           stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
   10340         3563 :           if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
   10341         3563 :             ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
   10342              :           else
   10343            0 :             ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
   10344              :                                                true, NULL_TREE, true,
   10345              :                                                GSI_SAME_STMT);
   10346         3563 :           create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
   10347              :                      insert_after, &offvar, NULL, true);
   10348              : 
   10349         3563 :           stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
   10350              :         }
   10351              : 
   10352        24458 :       running_off = offvar;
   10353        24458 :       alias_off = build_int_cst (ref_type, 0);
   10354        24458 :       int nloads = const_nunits;
   10355        24458 :       int lnel = 1;
   10356        24458 :       tree ltype = TREE_TYPE (vectype);
   10357        24458 :       tree lvectype = vectype;
   10358        24458 :       auto_vec<tree> dr_chain;
   10359              :       /* ???  Modify local copies of alignment_support_scheme and
   10360              :          misalignment, but this part of analysis should be done
   10361              :          earlier and remembered, likewise the chosen load mode.  */
   10362        24458 :       const dr_alignment_support tem = alignment_support_scheme;
   10363        24458 :       dr_alignment_support alignment_support_scheme = tem;
   10364        24458 :       const int tem2 = misalignment;
   10365        24458 :       int misalignment = tem2;
   10366        24458 :       if (memory_access_type == VMAT_STRIDED_SLP)
   10367              :         {
   10368        17599 :           HOST_WIDE_INT n = gcd (group_size, const_nunits);
   10369              :           /* Use the target vector type if the group size is a multiple
   10370              :              of it.  */
   10371        17599 :           if (n == const_nunits)
   10372              :             {
   10373         2266 :               int mis_align = dr_misalignment (first_dr_info, vectype);
   10374              :               /* With VF > 1 we advance the DR by step, if that is constant
   10375              :                  and only aligned when performed VF times, DR alignment
   10376              :                  analysis can analyze this as aligned since it assumes
   10377              :                  contiguous accesses.  But that is not how we code generate
   10378              :                  here, so adjust for this.  */
   10379         2266 :               if (maybe_gt (vf, 1u)
   10380         3634 :                   && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
   10381         3418 :                                   DR_TARGET_ALIGNMENT (first_dr_info)))
   10382          216 :                 mis_align = -1;
   10383         2266 :               dr_alignment_support dr_align
   10384         2266 :                 = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
   10385              :                                                  mis_align);
   10386         2266 :               if (dr_align == dr_aligned
   10387         2266 :                   || dr_align == dr_unaligned_supported)
   10388              :                 {
   10389        17599 :                   nloads = 1;
   10390        17599 :                   lnel = const_nunits;
   10391        17599 :                   ltype = vectype;
   10392        17599 :                   alignment_support_scheme = dr_align;
   10393        17599 :                   misalignment = mis_align;
   10394              :                 }
   10395              :             }
   10396              :           /* Else use the biggest vector we can load the group without
   10397              :              accessing excess elements.  */
   10398        15333 :           else if (n > 1)
   10399              :             {
   10400         2014 :               tree ptype;
   10401         2014 :               tree vtype
   10402         2014 :                 = vector_vector_composition_type (vectype, const_nunits / n,
   10403              :                                                   &ptype);
   10404         2014 :               if (vtype != NULL_TREE)
   10405              :                 {
   10406         1976 :                   dr_alignment_support dr_align;
   10407         1976 :                   int mis_align = 0;
   10408         1976 :                   if (VECTOR_TYPE_P (ptype))
   10409              :                     {
   10410         1056 :                       mis_align = dr_misalignment (first_dr_info, ptype);
   10411         1056 :                       if (maybe_gt (vf, 1u)
   10412         2082 :                           && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
   10413         1062 :                                           DR_TARGET_ALIGNMENT (first_dr_info)))
   10414         1020 :                         mis_align = -1;
   10415         1056 :                       dr_align
   10416         1056 :                         = vect_supportable_dr_alignment (vinfo, dr_info, ptype,
   10417              :                                                          mis_align);
   10418              :                     }
   10419              :                   else
   10420              :                     dr_align = dr_unaligned_supported;
   10421         1976 :                   if (dr_align == dr_aligned
   10422         1976 :                       || dr_align == dr_unaligned_supported)
   10423              :                     {
   10424         1976 :                       nloads = const_nunits / n;
   10425         1976 :                       lnel = n;
   10426         1976 :                       lvectype = vtype;
   10427         1976 :                       ltype = ptype;
   10428         1976 :                       alignment_support_scheme = dr_align;
   10429         1976 :                       misalignment = mis_align;
   10430              :                     }
   10431              :                 }
   10432              :             }
   10433        17599 :           unsigned align;
   10434        17599 :           if (alignment_support_scheme == dr_aligned)
   10435           14 :             align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
   10436              :           else
   10437        17585 :             align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
   10438              :           /* Alignment is at most the access size if we do multiple loads.  */
   10439        17599 :           if (nloads > 1)
   10440        15333 :             align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
   10441        17599 :           ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
   10442              :         }
   10443              : 
   10444        24458 :       if (costing_p)
   10445              :         {
   10446              :           /* Record the composition type for target access during costing.  */
   10447        20895 :           ls.ls_type = lvectype;
   10448        20895 :           ls.ls_eltype = ltype;
   10449              :         }
   10450              :       else
   10451         3563 :         gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
   10452              : 
   10453              :       /* For SLP permutation support we need to load the whole group,
   10454              :          not only the number of vector stmts the permutation result
   10455              :          fits in.  */
   10456        24458 :       int ncopies;
   10457        24458 :       if (ls.slp_perm)
   10458              :         {
   10459         2871 :           gcc_assert (memory_access_type != VMAT_ELEMENTWISE);
   10460              :           /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
   10461              :              variable VF.  */
   10462         2871 :           unsigned int const_vf = vf.to_constant ();
   10463         2871 :           ncopies = CEIL (group_size * const_vf, const_nunits);
   10464         2871 :           dr_chain.create (ncopies);
   10465              :         }
   10466              :       else
   10467              :         ncopies = vec_num;
   10468              : 
   10469        24458 :       unsigned int group_el = 0;
   10470        24458 :       unsigned HOST_WIDE_INT
   10471        24458 :         elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
   10472        24458 :       unsigned int n_groups = 0;
   10473              :       /* For costing some adjacent vector loads, we'd like to cost with
   10474              :          the total number of them once instead of cost each one by one. */
   10475        24458 :       unsigned int n_adjacent_loads = 0;
   10476        58172 :       for (j = 0; j < ncopies; j++)
   10477              :         {
   10478        33714 :           if (nloads > 1 && !costing_p)
   10479         3275 :             vec_alloc (v, nloads);
   10480        33714 :           gimple *new_stmt = NULL;
   10481       141980 :           for (i = 0; i < nloads; i++)
   10482              :             {
   10483       108266 :               if (costing_p)
   10484              :                 {
   10485              :                   /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
   10486              :                      avoid ICE, see PR110776.  */
   10487        97850 :                   if (VECTOR_TYPE_P (ltype)
   10488         5918 :                       && memory_access_type != VMAT_ELEMENTWISE)
   10489         5918 :                     n_adjacent_loads++;
   10490              :                   else
   10491        91932 :                     inside_cost += record_stmt_cost (cost_vec, 1, scalar_load,
   10492              :                                                      slp_node, 0, vect_body);
   10493        97850 :                   continue;
   10494              :                 }
   10495        10416 :               unsigned int load_el = group_el;
   10496              :               /* For elementwise accesses apply a load permutation directly.  */
   10497        10416 :               if (memory_access_type == VMAT_ELEMENTWISE
   10498        10416 :                   && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
   10499         2266 :                 load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el];
   10500        10416 :               tree this_off = build_int_cst (TREE_TYPE (alias_off),
   10501        10416 :                                              load_el * elsz + cst_offset);
   10502        10416 :               tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
   10503        10416 :               vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
   10504        10416 :               new_temp = make_ssa_name (ltype);
   10505        10416 :               new_stmt = gimple_build_assign (new_temp, data_ref);
   10506        10416 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   10507        10416 :               if (nloads > 1)
   10508         8800 :                 CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp);
   10509              : 
   10510        10416 :               group_el += lnel;
   10511        10416 :               if (group_el == group_size)
   10512              :                 {
   10513        10047 :                   n_groups++;
   10514              :                   /* When doing SLP make sure to not load elements from
   10515              :                      the next vector iteration, those will not be accessed
   10516              :                      so just use the last element again.  See PR107451.  */
   10517        10047 :                   if (known_lt (n_groups, vf))
   10518              :                     {
   10519         6465 :                       tree newoff = copy_ssa_name (running_off);
   10520         6465 :                       gimple *incr
   10521         6465 :                         = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
   10522              :                                                running_off, stride_step);
   10523         6465 :                       vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
   10524         6465 :                       running_off = newoff;
   10525              :                     }
   10526              :                   group_el = 0;
   10527              :                 }
   10528              :             }
   10529              : 
   10530        33714 :           if (nloads > 1)
   10531              :             {
   10532        24956 :               if (costing_p)
   10533        21681 :                 inside_cost += record_stmt_cost (cost_vec, 1, vec_construct,
   10534              :                                                  slp_node, 0, vect_body);
   10535              :               else
   10536              :                 {
   10537         3275 :                   tree vec_inv = build_constructor (lvectype, v);
   10538         3275 :                   new_temp = vect_init_vector (vinfo, stmt_info, vec_inv,
   10539              :                                                lvectype, gsi);
   10540         3275 :                   new_stmt = SSA_NAME_DEF_STMT (new_temp);
   10541         3275 :                   if (lvectype != vectype)
   10542              :                     {
   10543          398 :                       new_stmt
   10544          398 :                         = gimple_build_assign (make_ssa_name (vectype),
   10545              :                                                VIEW_CONVERT_EXPR,
   10546              :                                                build1 (VIEW_CONVERT_EXPR,
   10547              :                                                        vectype, new_temp));
   10548          398 :                       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
   10549              :                                                    gsi);
   10550              :                     }
   10551              :                 }
   10552              :             }
   10553         8758 :           else if (!costing_p && ltype != vectype)
   10554              :             {
   10555         1599 :               new_stmt = gimple_build_assign (make_ssa_name (vectype),
   10556              :                                               VIEW_CONVERT_EXPR,
   10557              :                                               build1 (VIEW_CONVERT_EXPR,
   10558              :                                                       vectype, new_temp));
   10559         1599 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
   10560              :                                            gsi);
   10561              :             }
   10562              : 
   10563        33714 :           if (!costing_p)
   10564              :             {
   10565         4891 :               if (ls.slp_perm)
   10566         1682 :                 dr_chain.quick_push (gimple_assign_lhs (new_stmt));
   10567              :               else
   10568         3209 :                 slp_node->push_vec_def (new_stmt);
   10569              :             }
   10570              :         }
   10571        24458 :       if (ls.slp_perm)
   10572              :         {
   10573         2871 :           if (costing_p)
   10574              :             {
   10575         2078 :               gcc_assert (ls.n_perms != -1U);
   10576         2078 :               inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
   10577              :                                                slp_node, 0, vect_body);
   10578              :             }
   10579              :           else
   10580              :             {
   10581          793 :               unsigned n_perms2;
   10582          793 :               vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
   10583              :                                             false, &n_perms2);
   10584          793 :               gcc_assert (ls.n_perms == n_perms2);
   10585              :             }
   10586              :         }
   10587              : 
   10588        24458 :       if (costing_p)
   10589              :         {
   10590        20895 :           if (n_adjacent_loads > 0)
   10591         2192 :             vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
   10592              :                                 alignment_support_scheme, misalignment, false,
   10593              :                                 &inside_cost, nullptr, cost_vec, cost_vec,
   10594              :                                 true);
   10595        20895 :           if (dump_enabled_p ())
   10596          522 :             dump_printf_loc (MSG_NOTE, vect_location,
   10597              :                              "vect_model_load_cost: inside_cost = %u, "
   10598              :                              "prologue_cost = 0 .\n",
   10599              :                              inside_cost);
   10600        20895 :           SLP_TREE_TYPE (slp_node) = load_vec_info_type;
   10601        20895 :           slp_node->data = new vect_load_store_data (std::move (ls));
   10602              :         }
   10603              : 
   10604        24458 :       return true;
   10605        24458 :     }
   10606              : 
   10607       592628 :   if (mat_gather_scatter_p (memory_access_type)
   10608       592628 :       && !ls.ls_type)
   10609              :     grouped_load = false;
   10610              : 
   10611       589638 :   if (grouped_load
   10612       592628 :       || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
   10613              :     {
   10614       261670 :       if (grouped_load)
   10615              :         {
   10616       261197 :           first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
   10617       261197 :           group_size = DR_GROUP_SIZE (first_stmt_info);
   10618              :         }
   10619              :       else
   10620              :         {
   10621              :           first_stmt_info = stmt_info;
   10622              :           group_size = 1;
   10623              :         }
   10624              :       /* For SLP vectorization we directly vectorize a subchain
   10625              :          without permutation.  */
   10626       261670 :       if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
   10627       204194 :         first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
   10628              :       /* For BB vectorization always use the first stmt to base
   10629              :          the data ref pointer on.  */
   10630       261670 :       if (bb_vinfo)
   10631       210286 :         first_stmt_info_for_drptr
   10632       210286 :           = vect_find_first_scalar_stmt_in_slp (slp_node);
   10633              : 
   10634       261670 :       first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
   10635       261670 :       group_gap_adj = 0;
   10636              : 
   10637              :       /* VEC_NUM is the number of vect stmts to be created for this group.  */
   10638       261670 :       grouped_load = false;
   10639              :       /* If an SLP permutation is from N elements to N elements,
   10640              :          and if one vector holds a whole number of N, we can load
   10641              :          the inputs to the permutation in the same way as an
   10642              :          unpermuted sequence.  In other cases we need to load the
   10643              :          whole group, not only the number of vector stmts the
   10644              :          permutation result fits in.  */
   10645       261670 :       unsigned scalar_lanes = SLP_TREE_LANES (slp_node);
   10646       261670 :       if (nested_in_vect_loop)
   10647              :         /* We do not support grouped accesses in a nested loop,
   10648              :            instead the access is contiguous but it might be
   10649              :            permuted.  No gap adjustment is needed though.  */
   10650              :         ;
   10651       261668 :       else if (ls.slp_perm
   10652       261668 :                && (group_size != scalar_lanes
   10653        10076 :                    || !multiple_p (nunits, group_size)))
   10654              :         {
   10655              :           /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
   10656              :              variable VF; see vect_transform_slp_perm_load.  */
   10657        48582 :           unsigned int const_vf = vf.to_constant ();
   10658        48582 :           unsigned int const_nunits = nunits.to_constant ();
   10659        48582 :           vec_num = CEIL (group_size * const_vf, const_nunits);
   10660        48582 :           group_gap_adj = vf * group_size - nunits * vec_num;
   10661              :         }
   10662              :       else
   10663              :         {
   10664       213086 :           group_gap_adj = group_size - scalar_lanes;
   10665              :         }
   10666              : 
   10667       261670 :       ref_type = get_group_alias_ptr_type (first_stmt_info);
   10668              :     }
   10669              :   else
   10670              :     {
   10671       330958 :       first_stmt_info = stmt_info;
   10672       330958 :       first_dr_info = dr_info;
   10673       330958 :       group_size = 1;
   10674       330958 :       group_gap_adj = 0;
   10675       330958 :       ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
   10676              :     }
   10677              : 
   10678       592628 :   vec_loop_masks *loop_masks
   10679       382342 :     = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
   10680       592628 :        ? &LOOP_VINFO_MASKS (loop_vinfo)
   10681           38 :        : NULL);
   10682           38 :   vec_loop_lens *loop_lens
   10683       382342 :     = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
   10684              :        ? &LOOP_VINFO_LENS (loop_vinfo)
   10685            0 :        : NULL);
   10686              : 
   10687              :   /* The vect_transform_stmt and vect_analyze_stmt will go here but there
   10688              :      are some difference here.  We cannot enable both the lens and masks
   10689              :      during transform but it is allowed during analysis.
   10690              :      Shouldn't go with length-based approach if fully masked.  */
   10691       592628 :   if (cost_vec == NULL)
   10692              :     /* The cost_vec is NULL during transform.  */
   10693       153273 :     gcc_assert ((!loop_lens || !loop_masks));
   10694              : 
   10695              :   /* Targets with store-lane instructions must not require explicit
   10696              :      realignment.  vect_supportable_dr_alignment always returns either
   10697              :      dr_aligned or dr_unaligned_supported for (non-length) masked
   10698              :      operations.  */
   10699       592628 :   gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
   10700              :                && !mask_node
   10701              :                && !loop_masks)
   10702              :               || mat_gather_scatter_p (memory_access_type)
   10703              :               || alignment_support_scheme == dr_aligned
   10704              :               || alignment_support_scheme == dr_unaligned_supported);
   10705              : 
   10706              :   /* In case the vectorization factor (VF) is bigger than the number
   10707              :      of elements that we can fit in a vectype (nunits), we have to generate
   10708              :      more than one vector stmt - i.e - we need to "unroll" the
   10709              :      vector stmt by a factor VF/nunits.  In doing so, we record a pointer
   10710              :      from one copy of the vector stmt to the next, in the field
   10711              :      STMT_VINFO_RELATED_STMT.  This is necessary in order to allow following
   10712              :      stages to find the correct vector defs to be used when vectorizing
   10713              :      stmts that use the defs of the current stmt.  The example below
   10714              :      illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
   10715              :      need to create 4 vectorized stmts):
   10716              : 
   10717              :      before vectorization:
   10718              :                                 RELATED_STMT    VEC_STMT
   10719              :         S1:     x = memref      -               -
   10720              :         S2:     z = x + 1       -               -
   10721              : 
   10722              :      step 1: vectorize stmt S1:
   10723              :         We first create the vector stmt VS1_0, and, as usual, record a
   10724              :         pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
   10725              :         Next, we create the vector stmt VS1_1, and record a pointer to
   10726              :         it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
   10727              :         Similarly, for VS1_2 and VS1_3.  This is the resulting chain of
   10728              :         stmts and pointers:
   10729              :                                 RELATED_STMT    VEC_STMT
   10730              :         VS1_0:  vx0 = memref0   VS1_1           -
   10731              :         VS1_1:  vx1 = memref1   VS1_2           -
   10732              :         VS1_2:  vx2 = memref2   VS1_3           -
   10733              :         VS1_3:  vx3 = memref3   -               -
   10734              :         S1:     x = load        -               VS1_0
   10735              :         S2:     z = x + 1       -               -
   10736              :   */
   10737              : 
   10738              :   /* If the data reference is aligned (dr_aligned) or potentially unaligned
   10739              :      on a target that supports unaligned accesses (dr_unaligned_supported)
   10740              :      we generate the following code:
   10741              :          p = initial_addr;
   10742              :          indx = 0;
   10743              :          loop {
   10744              :            p = p + indx * vectype_size;
   10745              :            vec_dest = *(p);
   10746              :            indx = indx + 1;
   10747              :          }
   10748              : 
   10749              :      Otherwise, the data reference is potentially unaligned on a target that
   10750              :      does not support unaligned accesses (dr_explicit_realign_optimized) -
   10751              :      then generate the following code, in which the data in each iteration is
   10752              :      obtained by two vector loads, one from the previous iteration, and one
   10753              :      from the current iteration:
   10754              :          p1 = initial_addr;
   10755              :          msq_init = *(floor(p1))
   10756              :          p2 = initial_addr + VS - 1;
   10757              :          realignment_token = call target_builtin;
   10758              :          indx = 0;
   10759              :          loop {
   10760              :            p2 = p2 + indx * vectype_size
   10761              :            lsq = *(floor(p2))
   10762              :            vec_dest = realign_load (msq, lsq, realignment_token)
   10763              :            indx = indx + 1;
   10764              :            msq = lsq;
   10765              :          }   */
   10766              : 
   10767              :   /* If the misalignment remains the same throughout the execution of the
   10768              :      loop, we can create the init_addr and permutation mask at the loop
   10769              :      preheader.  Otherwise, it needs to be created inside the loop.
   10770              :      This can only occur when vectorizing memory accesses in the inner-loop
   10771              :      nested within an outer-loop that is being vectorized.  */
   10772              : 
   10773       592628 :   if (nested_in_vect_loop
   10774       592628 :       && !multiple_p (DR_STEP_ALIGNMENT (dr_info->dr),
   10775         1234 :                       GET_MODE_SIZE (TYPE_MODE (vectype))))
   10776              :     {
   10777          195 :       gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
   10778              :       compute_in_loop = true;
   10779              :     }
   10780              : 
   10781       592628 :   bool diff_first_stmt_info
   10782       592628 :     = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
   10783              : 
   10784       592628 :   tree offset = NULL_TREE;
   10785       592628 :   if ((alignment_support_scheme == dr_explicit_realign_optimized
   10786       592628 :        || alignment_support_scheme == dr_explicit_realign)
   10787            0 :       && !compute_in_loop)
   10788              :     {
   10789              :       /* If we have different first_stmt_info, we can't set up realignment
   10790              :          here, since we can't guarantee first_stmt_info DR has been
   10791              :          initialized yet, use first_stmt_info_for_drptr DR by bumping the
   10792              :          distance from first_stmt_info DR instead as below.  */
   10793            0 :       if (!costing_p)
   10794              :         {
   10795            0 :           if (!diff_first_stmt_info)
   10796            0 :             msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi,
   10797              :                                           &realignment_token,
   10798              :                                           alignment_support_scheme, NULL_TREE,
   10799              :                                           &at_loop);
   10800            0 :           if (alignment_support_scheme == dr_explicit_realign_optimized)
   10801              :             {
   10802            0 :               phi = as_a<gphi *> (SSA_NAME_DEF_STMT (msq));
   10803            0 :               offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
   10804              :                                    size_one_node);
   10805            0 :               gcc_assert (!first_stmt_info_for_drptr);
   10806              :             }
   10807              :         }
   10808              :     }
   10809              :   else
   10810       592628 :     at_loop = loop;
   10811              : 
   10812       592628 :   if (!known_eq (poffset, 0))
   10813         4781 :     offset = (offset
   10814         4781 :               ? size_binop (PLUS_EXPR, offset, size_int (poffset))
   10815         4781 :               : size_int (poffset));
   10816              : 
   10817       592628 :   tree dr_increment;
   10818       592628 :   tree dr_bump;
   10819       592628 :   tree vec_offset = NULL_TREE;
   10820              : 
   10821       592628 :   auto_vec<tree> vec_offsets;
   10822       592628 :   auto_vec<tree> vec_masks;
   10823       592628 :   if (mask_node && !costing_p)
   10824          630 :     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
   10825              :                        &vec_masks);
   10826              : 
   10827       592628 :   tree vec_mask = NULL_TREE;
   10828       592628 :   tree vec_els = NULL_TREE;
   10829       592628 :   if (memory_access_type == VMAT_LOAD_STORE_LANES)
   10830              :     {
   10831            0 :       const internal_fn lanes_ifn = ls.lanes_ifn;
   10832              : 
   10833            0 :       gcc_assert (alignment_support_scheme == dr_aligned
   10834              :                   || alignment_support_scheme == dr_unaligned_supported);
   10835              : 
   10836            0 :       aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
   10837            0 :       if (!costing_p)
   10838              :         {
   10839            0 :           dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
   10840              :                                                       memory_access_type);
   10841            0 :           dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
   10842              :                                             memory_access_type);
   10843              :         }
   10844              : 
   10845            0 :       unsigned int inside_cost = 0, prologue_cost = 0;
   10846              :       /* For costing some adjacent vector loads, we'd like to cost with
   10847              :          the total number of them once instead of cost each one by one. */
   10848            0 :       unsigned int n_adjacent_loads = 0;
   10849            0 :       int ncopies = vec_num / group_size;
   10850            0 :       for (j = 0; j < ncopies; j++)
   10851              :         {
   10852            0 :           if (costing_p)
   10853              :             {
   10854              :               /* An IFN_LOAD_LANES will load all its vector results,
   10855              :                  regardless of which ones we actually need.  Account
   10856              :                  for the cost of unused results.  */
   10857            0 :               if (first_stmt_info == stmt_info)
   10858              :                 {
   10859            0 :                   unsigned int gaps = DR_GROUP_SIZE (first_stmt_info);
   10860            0 :                   stmt_vec_info next_stmt_info = first_stmt_info;
   10861            0 :                   do
   10862              :                     {
   10863            0 :                       gaps -= 1;
   10864            0 :                       next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
   10865              :                     }
   10866            0 :                   while (next_stmt_info);
   10867            0 :                   if (gaps)
   10868              :                     {
   10869            0 :                       if (dump_enabled_p ())
   10870            0 :                         dump_printf_loc (MSG_NOTE, vect_location,
   10871              :                                          "vect_model_load_cost: %d "
   10872              :                                          "unused vectors.\n",
   10873              :                                          gaps);
   10874            0 :                       vect_get_load_cost (vinfo, stmt_info, slp_node, gaps,
   10875              :                                           alignment_support_scheme,
   10876              :                                           misalignment, false, &inside_cost,
   10877              :                                           &prologue_cost, cost_vec, cost_vec,
   10878              :                                           true);
   10879              :                     }
   10880              :                 }
   10881            0 :               n_adjacent_loads++;
   10882            0 :               continue;
   10883            0 :             }
   10884              : 
   10885              :           /* 1. Create the vector or array pointer update chain.  */
   10886            0 :           if (j == 0)
   10887            0 :             dataref_ptr
   10888            0 :               = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
   10889              :                                           at_loop, offset, &dummy, gsi,
   10890              :                                           NULL, false, dr_increment);
   10891              :           else
   10892              :             {
   10893            0 :               gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
   10894            0 :               dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
   10895              :                                              stmt_info, dr_bump);
   10896              :             }
   10897            0 :           if (mask_node)
   10898            0 :             vec_mask = vec_masks[j];
   10899              : 
   10900            0 :           tree vec_array = create_vector_array (vectype, group_size);
   10901              : 
   10902            0 :           tree final_mask = NULL_TREE;
   10903            0 :           tree final_len = NULL_TREE;
   10904            0 :           tree bias = NULL_TREE;
   10905            0 :           if (loop_masks)
   10906            0 :             final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
   10907              :                                              ncopies, vectype, j);
   10908            0 :           if (vec_mask)
   10909            0 :             final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
   10910              :                                            vec_mask, gsi);
   10911              : 
   10912            0 :           if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
   10913              :             {
   10914            0 :               if (loop_lens)
   10915            0 :                 final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
   10916              :                                                ncopies, vectype, j, 1, true);
   10917              :               else
   10918            0 :                 final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
   10919            0 :               signed char biasval
   10920            0 :                 = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
   10921            0 :               bias = build_int_cst (intQI_type_node, biasval);
   10922            0 :               if (!final_mask)
   10923              :                 {
   10924            0 :                   mask_vectype = truth_type_for (vectype);
   10925            0 :                   final_mask = build_minus_one_cst (mask_vectype);
   10926              :                 }
   10927              :             }
   10928              : 
   10929            0 :           if (final_mask)
   10930              :             {
   10931            0 :               vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
   10932            0 :               if (type_mode_padding_p
   10933            0 :                   && maskload_elsval != MASK_LOAD_ELSE_ZERO)
   10934            0 :                 need_zeroing = true;
   10935              :             }
   10936              : 
   10937            0 :           gcall *call;
   10938            0 :           if (final_len && final_mask)
   10939              :             {
   10940              :               /* Emit:
   10941              :                    VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
   10942              :                                                     VEC_MASK, LEN, BIAS).  */
   10943            0 :               unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
   10944            0 :               tree alias_ptr = build_int_cst (ref_type, align);
   10945            0 :               call = gimple_build_call_internal (IFN_MASK_LEN_LOAD_LANES, 6,
   10946              :                                                  dataref_ptr, alias_ptr,
   10947              :                                                  final_mask, vec_els,
   10948              :                                                  final_len, bias);
   10949              :             }
   10950            0 :           else if (final_mask)
   10951              :             {
   10952              :               /* Emit:
   10953              :                    VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
   10954              :                                                 VEC_MASK).  */
   10955            0 :               unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
   10956            0 :               tree alias_ptr = build_int_cst (ref_type, align);
   10957            0 :               call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 4,
   10958              :                                                  dataref_ptr, alias_ptr,
   10959              :                                                  final_mask, vec_els);
   10960              :             }
   10961              :           else
   10962              :             {
   10963              :               /* Emit:
   10964              :                    VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]).  */
   10965            0 :               data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
   10966            0 :               call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
   10967              :             }
   10968            0 :           gimple_call_set_lhs (call, vec_array);
   10969            0 :           gimple_call_set_nothrow (call, true);
   10970            0 :           vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
   10971              : 
   10972              :           /* Extract each vector into an SSA_NAME.  */
   10973            0 :           for (unsigned i = 0; i < group_size; i++)
   10974              :             {
   10975            0 :               new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
   10976              :                                             vec_array, i, need_zeroing,
   10977              :                                             final_mask);
   10978            0 :               slp_node->push_vec_def (new_temp);
   10979              :             }
   10980              : 
   10981              :           /* Record that VEC_ARRAY is now dead.  */
   10982            0 :           vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
   10983              :         }
   10984              : 
   10985            0 :       if (costing_p)
   10986              :         {
   10987            0 :           if (n_adjacent_loads > 0)
   10988            0 :             vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
   10989              :                                 alignment_support_scheme, misalignment, false,
   10990              :                                 &inside_cost, &prologue_cost, cost_vec,
   10991              :                                 cost_vec, true);
   10992            0 :           if (dump_enabled_p ())
   10993            0 :             dump_printf_loc (MSG_NOTE, vect_location,
   10994              :                              "vect_model_load_cost: inside_cost = %u, "
   10995              :                              "prologue_cost = %u .\n",
   10996              :                              inside_cost, prologue_cost);
   10997            0 :           SLP_TREE_TYPE (slp_node) = load_vec_info_type;
   10998            0 :           slp_node->data = new vect_load_store_data (std::move (ls));
   10999              :         }
   11000              : 
   11001            0 :       return true;
   11002              :     }
   11003              : 
   11004       592628 :   if (mat_gather_scatter_p (memory_access_type))
   11005              :     {
   11006         2990 :       gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
   11007              : 
   11008         2990 :       auto_vec<tree> dr_chain (vec_num);
   11009              : 
   11010              :       /* If we pun the original vectype the loads as well as costing, length,
   11011              :          etc. is performed with the new type.  After loading we VIEW_CONVERT
   11012              :          the data to the original vectype.  */
   11013         2990 :       tree original_vectype = vectype;
   11014         2990 :       if (ls.ls_type)
   11015            0 :         vectype = ls.ls_type;
   11016              : 
   11017              :       /* 1. Create the vector or array pointer update chain.  */
   11018         2990 :       if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
   11019              :         {
   11020         2990 :           aggr_type = NULL_TREE;
   11021         2990 :           dr_increment = NULL_TREE;
   11022         2990 :           if (!costing_p)
   11023          778 :             vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
   11024              :                                          &vec_offsets);
   11025              :         }
   11026              :       else
   11027              :         {
   11028            0 :           aggr_type = elem_type;
   11029            0 :           if (!costing_p)
   11030              :             {
   11031            0 :               vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
   11032              :                                                ls.strided_offset_vectype,
   11033              :                                                loop_vinfo, gsi,
   11034              :                                                &dr_increment, &dr_bump,
   11035              :                                                &vec_offset);
   11036            0 :               dataref_ptr
   11037            0 :                   = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
   11038              :                                               at_loop, offset, &dummy, gsi,
   11039              :                                               NULL, false, dr_increment);
   11040              :             }
   11041              :         }
   11042              : 
   11043         2990 :       unsigned int inside_cost = 0, prologue_cost = 0;
   11044              : 
   11045         2990 :       gimple *new_stmt = NULL;
   11046         6768 :       for (i = 0; i < vec_num; i++)
   11047              :         {
   11048         3778 :           tree final_mask = NULL_TREE;
   11049         3778 :           tree final_len = NULL_TREE;
   11050         3778 :           tree bias = NULL_TREE;
   11051         3778 :           if (!costing_p)
   11052              :             {
   11053          994 :               if (mask_node)
   11054          148 :                 vec_mask = vec_masks[i];
   11055          994 :               if (loop_masks)
   11056            0 :                 final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
   11057              :                                                  vec_num, vectype, i);
   11058          994 :               if (vec_mask)
   11059          148 :                 final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
   11060              :                                                final_mask, vec_mask, gsi);
   11061              : 
   11062          994 :               if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
   11063            0 :                 dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
   11064              :                                                stmt_info, dr_bump);
   11065              :             }
   11066              : 
   11067              :           /* 2. Create the vector-load in the loop.  */
   11068         3778 :           unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
   11069         3778 :           tree alias_align_ptr = build_int_cst (ref_type, align);
   11070         3778 :           if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
   11071              :             {
   11072            0 :               if (costing_p)
   11073              :                 {
   11074            0 :                   if (ls.supported_offset_vectype
   11075            0 :                       && !tree_nop_conversion_p (ls.supported_offset_vectype,
   11076              :                                                  vec_offset))
   11077            0 :                     inside_cost
   11078            0 :                       += record_stmt_cost (cost_vec, 1, vector_stmt,
   11079              :                                            slp_node, 0, vect_body);
   11080            0 :                   if (ls.supported_scale)
   11081            0 :                     inside_cost
   11082            0 :                       += record_stmt_cost (cost_vec, 1, vector_stmt,
   11083              :                                            slp_node, 0, vect_body);
   11084              : 
   11085            0 :                   unsigned int cnunits = vect_nunits_for_cost (vectype);
   11086            0 :                   inside_cost
   11087            0 :                     = record_stmt_cost (cost_vec, cnunits, scalar_load,
   11088              :                                         slp_node, 0, vect_body);
   11089            0 :                   continue;
   11090            0 :                 }
   11091            0 :               if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
   11092            0 :                 vec_offset = vec_offsets[i];
   11093            0 :               tree zero = build_zero_cst (vectype);
   11094            0 :               tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
   11095            0 :               bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
   11096              : 
   11097              :               /* Perform the offset conversion and scaling if necessary.  */
   11098            0 :               if (!strided
   11099            0 :                   && (ls.supported_offset_vectype || ls.supported_scale))
   11100              :                 {
   11101            0 :                   gimple_seq stmts = NULL;
   11102            0 :                   if (ls.supported_offset_vectype)
   11103            0 :                     vec_offset = gimple_convert
   11104            0 :                       (&stmts, ls.supported_offset_vectype, vec_offset);
   11105            0 :                   if (ls.supported_scale)
   11106              :                     {
   11107              :                       /* Only scale the vec_offset if we haven't already.  */
   11108            0 :                       if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
   11109            0 :                           || i == 0)
   11110              :                         {
   11111            0 :                           tree mult_cst = build_int_cst
   11112            0 :                             (TREE_TYPE (TREE_TYPE (vec_offset)),
   11113            0 :                              SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
   11114            0 :                           tree mult = build_vector_from_val
   11115            0 :                             (TREE_TYPE (vec_offset), mult_cst);
   11116            0 :                           vec_offset = gimple_build
   11117            0 :                             (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
   11118              :                              vec_offset, mult);
   11119              :                         }
   11120            0 :                       scale = size_int (ls.supported_scale);
   11121              :                     }
   11122            0 :                   gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
   11123              :                 }
   11124              : 
   11125            0 :               if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
   11126              :                 {
   11127            0 :                   if (loop_lens)
   11128            0 :                     final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
   11129              :                                                    vec_num, vectype, i, 1, true);
   11130              :                   else
   11131            0 :                     final_len = build_int_cst (sizetype,
   11132            0 :                                                TYPE_VECTOR_SUBPARTS (vectype));
   11133            0 :                   signed char biasval
   11134            0 :                     = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
   11135            0 :                   bias = build_int_cst (intQI_type_node, biasval);
   11136            0 :                   if (!final_mask)
   11137              :                     {
   11138            0 :                       mask_vectype = truth_type_for (vectype);
   11139            0 :                       final_mask = build_minus_one_cst (mask_vectype);
   11140              :                     }
   11141              :                 }
   11142              : 
   11143            0 :               if (final_mask)
   11144              :                 {
   11145            0 :                   vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
   11146            0 :                   if (type_mode_padding_p
   11147            0 :                       && maskload_elsval != MASK_LOAD_ELSE_ZERO)
   11148            0 :                     need_zeroing = true;
   11149              :                 }
   11150              : 
   11151            0 :               gcall *call;
   11152            0 :               if (final_len && final_mask)
   11153              :                 {
   11154            0 :                   if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
   11155            0 :                     call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
   11156              :                                                        9, dataref_ptr,
   11157              :                                                        alias_align_ptr,
   11158              :                                                        vec_offset, scale, zero,
   11159              :                                                        final_mask, vec_els,
   11160              :                                                        final_len, bias);
   11161              :                   else
   11162              :                     /* Non-vector offset indicates that prefer to take
   11163              :                        MASK_LEN_STRIDED_LOAD instead of the
   11164              :                        MASK_LEN_GATHER_LOAD with direct stride arg.  */
   11165            0 :                     call = gimple_build_call_internal
   11166            0 :                              (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr,
   11167              :                               vec_offset, zero, final_mask, vec_els, final_len,
   11168              :                               bias);
   11169              :                 }
   11170            0 :               else if (final_mask)
   11171            0 :                 call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
   11172              :                                                    7, dataref_ptr,
   11173              :                                                    alias_align_ptr,
   11174              :                                                    vec_offset, scale,
   11175              :                                                    zero, final_mask, vec_els);
   11176              :               else
   11177            0 :                 call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
   11178              :                                                    dataref_ptr,
   11179              :                                                    alias_align_ptr,
   11180              :                                                    vec_offset, scale, zero);
   11181            0 :               gimple_call_set_nothrow (call, true);
   11182            0 :               new_stmt = call;
   11183            0 :               data_ref = NULL_TREE;
   11184              :             }
   11185         3778 :           else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
   11186              :             {
   11187              :               /* The builtin decls path for gather is legacy, x86 only.  */
   11188          828 :               gcc_assert (!final_len && nunits.is_constant ());
   11189          828 :               if (costing_p)
   11190              :                 {
   11191          554 :                   unsigned int cnunits = vect_nunits_for_cost (vectype);
   11192          554 :                   inside_cost
   11193          554 :                     = record_stmt_cost (cost_vec, cnunits, scalar_load,
   11194              :                                         slp_node, 0, vect_body);
   11195          554 :                   continue;
   11196          554 :                 }
   11197          274 :               tree offset_vectype = TREE_TYPE (vec_offsets[0]);
   11198          274 :               poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype);
   11199          274 :               if (known_eq (nunits, offset_nunits))
   11200              :                 {
   11201          133 :                   new_stmt = vect_build_one_gather_load_call
   11202          133 :                                (vinfo, stmt_info, slp_node, vectype, gsi,
   11203          133 :                                 ls.gs.decl, dataref_ptr, vec_offsets[i],
   11204              :                                 final_mask);
   11205          133 :                   data_ref = NULL_TREE;
   11206              :                 }
   11207          141 :               else if (known_eq (nunits, offset_nunits * 2))
   11208              :                 {
   11209              :                   /* We have a offset vector with half the number of
   11210              :                      lanes but the builtins will produce full vectype
   11211              :                      data with just the lower lanes filled.  */
   11212           63 :                   new_stmt = vect_build_one_gather_load_call
   11213          126 :                                (vinfo, stmt_info, slp_node, vectype, gsi,
   11214           63 :                                 ls.gs.decl, dataref_ptr, vec_offsets[2 * i],
   11215              :                                 final_mask);
   11216           63 :                   tree low = make_ssa_name (vectype);
   11217           63 :                   gimple_set_lhs (new_stmt, low);
   11218           63 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11219              : 
   11220              :                   /* now put upper half of final_mask in final_mask low. */
   11221           63 :                   if (final_mask
   11222           63 :                       && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask))))
   11223              :                     {
   11224           11 :                       int count = nunits.to_constant ();
   11225           11 :                       vec_perm_builder sel (count, count, 1);
   11226           11 :                       sel.quick_grow (count);
   11227           98 :                       for (int i = 0; i < count; ++i)
   11228           76 :                         sel[i] = i | (count / 2);
   11229           11 :                       vec_perm_indices indices (sel, 2, count);
   11230           11 :                       tree perm_mask = vect_gen_perm_mask_checked
   11231           11 :                                          (TREE_TYPE (final_mask), indices);
   11232           11 :                       new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
   11233              :                                                       final_mask, final_mask,
   11234              :                                                       perm_mask);
   11235           11 :                       final_mask = make_ssa_name (TREE_TYPE (final_mask));
   11236           11 :                       gimple_set_lhs (new_stmt, final_mask);
   11237           11 :                       vect_finish_stmt_generation (vinfo, stmt_info,
   11238              :                                                    new_stmt, gsi);
   11239           11 :                     }
   11240           52 :                   else if (final_mask)
   11241              :                     {
   11242           24 :                       new_stmt = gimple_build_assign (NULL_TREE,
   11243              :                                                       VEC_UNPACK_HI_EXPR,
   11244              :                                                       final_mask);
   11245           24 :                       final_mask = make_ssa_name
   11246           24 :                                     (truth_type_for (offset_vectype));
   11247           24 :                       gimple_set_lhs (new_stmt, final_mask);
   11248           24 :                       vect_finish_stmt_generation (vinfo, stmt_info,
   11249              :                                                    new_stmt, gsi);
   11250              :                     }
   11251              : 
   11252           63 :                   new_stmt = vect_build_one_gather_load_call
   11253          126 :                                (vinfo, stmt_info, slp_node, vectype, gsi,
   11254              :                                 ls.gs.decl, dataref_ptr,
   11255           63 :                                 vec_offsets[2 * i + 1], final_mask);
   11256           63 :                   tree high = make_ssa_name (vectype);
   11257           63 :                   gimple_set_lhs (new_stmt, high);
   11258           63 :                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11259              : 
   11260              :                   /* compose low + high.  */
   11261           63 :                   int count = nunits.to_constant ();
   11262           63 :                   vec_perm_builder sel (count, count, 1);
   11263           63 :                   sel.quick_grow (count);
   11264          710 :                   for (int i = 0; i < count; ++i)
   11265          584 :                     sel[i] = i < count / 2 ? i : i + count / 2;
   11266           63 :                   vec_perm_indices indices (sel, 2, count);
   11267           63 :                   tree perm_mask
   11268           63 :                     = vect_gen_perm_mask_checked (vectype, indices);
   11269           63 :                   new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
   11270              :                                                   low, high, perm_mask);
   11271           63 :                   data_ref = NULL_TREE;
   11272           63 :                 }
   11273           78 :               else if (known_eq (nunits * 2, offset_nunits))
   11274              :                 {
   11275              :                   /* We have a offset vector with double the number of
   11276              :                      lanes.  Select the low/high part accordingly.  */
   11277           78 :                   vec_offset = vec_offsets[i / 2];
   11278           78 :                   if (i & 1)
   11279              :                     {
   11280           39 :                       int count = offset_nunits.to_constant ();
   11281           39 :                       vec_perm_builder sel (count, count, 1);
   11282           39 :                       sel.quick_grow (count);
   11283          474 :                       for (int i = 0; i < count; ++i)
   11284          396 :                         sel[i] = i | (count / 2);
   11285           39 :                       vec_perm_indices indices (sel, 2, count);
   11286           39 :                       tree perm_mask = vect_gen_perm_mask_checked
   11287           39 :                                          (TREE_TYPE (vec_offset), indices);
   11288           39 :                       new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
   11289              :                                                       vec_offset, vec_offset,
   11290              :                                                       perm_mask);
   11291           39 :                       vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
   11292           39 :                       gimple_set_lhs (new_stmt, vec_offset);
   11293           39 :                       vect_finish_stmt_generation (vinfo, stmt_info,
   11294              :                                                    new_stmt, gsi);
   11295           39 :                     }
   11296           78 :                   new_stmt = vect_build_one_gather_load_call
   11297           78 :                                (vinfo, stmt_info, slp_node, vectype, gsi,
   11298              :                                 ls.gs.decl,
   11299              :                                 dataref_ptr, vec_offset, final_mask);
   11300           78 :                   data_ref = NULL_TREE;
   11301              :                 }
   11302              :               else
   11303            0 :                 gcc_unreachable ();
   11304              :             }
   11305              :           else
   11306              :             {
   11307              :               /* Emulated gather-scatter.  */
   11308         2950 :               gcc_assert (!final_mask);
   11309         2950 :               unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
   11310         2950 :               if (costing_p)
   11311              :                 {
   11312              :                   /* For emulated gathers N offset vector element
   11313              :                      offset add is consumed by the load).  */
   11314         2230 :                   inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
   11315              :                                                   slp_node, 0, vect_body);
   11316              :                   /* N scalar loads plus gathering them into a
   11317              :                      vector.  */
   11318         2230 :                   inside_cost
   11319         2230 :                     = record_stmt_cost (cost_vec, const_nunits, scalar_load,
   11320              :                                         slp_node, 0, vect_body);
   11321         2230 :                   inside_cost
   11322         2230 :                     = record_stmt_cost (cost_vec, 1, vec_construct,
   11323              :                                         slp_node, 0, vect_body);
   11324         2230 :                   continue;
   11325              :                 }
   11326          720 :               tree offset_vectype = TREE_TYPE (vec_offsets[0]);
   11327          720 :               unsigned HOST_WIDE_INT const_offset_nunits
   11328          720 :                 = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
   11329          720 :               vec<constructor_elt, va_gc> *ctor_elts;
   11330          720 :               vec_alloc (ctor_elts, const_nunits);
   11331          720 :               gimple_seq stmts = NULL;
   11332              :               /* We support offset vectors with more elements
   11333              :                  than the data vector for now.  */
   11334          720 :               unsigned HOST_WIDE_INT factor
   11335              :                 = const_offset_nunits / const_nunits;
   11336          720 :               vec_offset = vec_offsets[i / factor];
   11337          720 :               unsigned elt_offset = (i % factor) * const_nunits;
   11338          720 :               tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
   11339          720 :               tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
   11340          720 :               tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
   11341         3616 :               for (unsigned k = 0; k < const_nunits; ++k)
   11342              :                 {
   11343         2176 :                   tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
   11344              :                                           bitsize_int (k + elt_offset));
   11345         6528 :                   tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
   11346         2176 :                                            vec_offset, TYPE_SIZE (idx_type),
   11347              :                                            boff);
   11348         2176 :                   idx = gimple_convert (&stmts, sizetype, idx);
   11349         2176 :                   idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
   11350         2176 :                   tree ptr = gimple_build (&stmts, PLUS_EXPR,
   11351         2176 :                                            TREE_TYPE (dataref_ptr),
   11352              :                                            dataref_ptr, idx);
   11353         2176 :                   ptr = gimple_convert (&stmts, ptr_type_node, ptr);
   11354         2176 :                   tree elt = make_ssa_name (TREE_TYPE (vectype));
   11355         2176 :                   tree ref = build2 (MEM_REF, ltype, ptr,
   11356              :                                      build_int_cst (ref_type, 0));
   11357         2176 :                   new_stmt = gimple_build_assign (elt, ref);
   11358         4352 :                   gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi)));
   11359         2176 :                   gimple_seq_add_stmt (&stmts, new_stmt);
   11360         2176 :                   CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt);
   11361              :                 }
   11362          720 :               gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
   11363          720 :               new_stmt = gimple_build_assign (NULL_TREE,
   11364              :                                               build_constructor (vectype,
   11365              :                                                                  ctor_elts));
   11366          720 :               data_ref = NULL_TREE;
   11367              :             }
   11368              : 
   11369          994 :           vec_dest = vect_create_destination_var (scalar_dest, vectype);
   11370              :           /* DATA_REF is null if we've already built the statement.  */
   11371          994 :           if (data_ref)
   11372              :             {
   11373              :               vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
   11374              :               new_stmt = gimple_build_assign (vec_dest, data_ref);
   11375              :             }
   11376         1988 :           new_temp = (need_zeroing
   11377          994 :                       ? make_ssa_name (vectype)
   11378          994 :                       : make_ssa_name (vec_dest, new_stmt));
   11379          994 :           gimple_set_lhs (new_stmt, new_temp);
   11380          994 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11381              : 
   11382              :           /* If we need to explicitly zero inactive elements emit a
   11383              :              VEC_COND_EXPR that does so.  */
   11384          994 :           if (need_zeroing)
   11385              :             {
   11386            0 :               vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
   11387              :                                                  vectype);
   11388              : 
   11389            0 :               tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
   11390            0 :               new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
   11391              :                                               final_mask, new_temp, vec_els);
   11392            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11393            0 :               new_temp = new_temp2;
   11394              :             }
   11395              : 
   11396          994 :           if (ls.ls_type)
   11397              :             {
   11398            0 :               new_stmt = gimple_build_assign (make_ssa_name
   11399              :                                               (original_vectype),
   11400              :                                               VIEW_CONVERT_EXPR,
   11401              :                                               build1 (VIEW_CONVERT_EXPR,
   11402              :                                                       original_vectype,
   11403              :                                                       new_temp));
   11404            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11405              :             }
   11406              : 
   11407              :           /* Store vector loads in the corresponding SLP_NODE.  */
   11408          994 :           if (!costing_p)
   11409              :             {
   11410          994 :               if (ls.slp_perm)
   11411            0 :                 dr_chain.quick_push (gimple_assign_lhs (new_stmt));
   11412              :               else
   11413          994 :                 slp_node->push_vec_def (new_stmt);
   11414              :             }
   11415              :         }
   11416              : 
   11417         2990 :       if (ls.slp_perm)
   11418              :         {
   11419            0 :           if (costing_p)
   11420              :             {
   11421            0 :               gcc_assert (ls.n_perms != -1U);
   11422            0 :               inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
   11423              :                                                slp_node, 0, vect_body);
   11424              :             }
   11425              :           else
   11426              :             {
   11427            0 :               unsigned n_perms2;
   11428            0 :               vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
   11429              :                                             false, &n_perms2);
   11430            0 :               gcc_assert (ls.n_perms == n_perms2);
   11431              :             }
   11432              :         }
   11433              : 
   11434         2990 :       if (costing_p)
   11435              :         {
   11436         2212 :           if (dump_enabled_p ())
   11437          315 :             dump_printf_loc (MSG_NOTE, vect_location,
   11438              :                              "vect_model_load_cost: inside_cost = %u, "
   11439              :                              "prologue_cost = %u .\n",
   11440              :                              inside_cost, prologue_cost);
   11441         2212 :           SLP_TREE_TYPE (slp_node) = load_vec_info_type;
   11442         2212 :           slp_node->data = new vect_load_store_data (std::move (ls));
   11443              :         }
   11444         2990 :       return true;
   11445         2990 :     }
   11446              : 
   11447       589638 :   gcc_assert (memory_access_type == VMAT_CONTIGUOUS
   11448              :               || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
   11449              : 
   11450       589638 :   aggr_type = vectype;
   11451       589638 :   if (!costing_p)
   11452              :     {
   11453       152495 :       dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
   11454              :                                                   memory_access_type);
   11455       152495 :       dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
   11456              :                                         memory_access_type);
   11457              :     }
   11458              : 
   11459       589638 :   poly_uint64 group_elt = 0;
   11460       589638 :   unsigned int inside_cost = 0, prologue_cost = 0;
   11461              :   /* For costing some adjacent vector loads, we'd like to cost with
   11462              :      the total number of them once instead of cost each one by one. */
   11463       589638 :   unsigned int n_adjacent_loads = 0;
   11464              : 
   11465              :   /* 1. Create the vector or array pointer update chain.  */
   11466       589638 :   if (!costing_p)
   11467              :     {
   11468       152495 :       bool simd_lane_access_p
   11469       152495 :           = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
   11470       152495 :       if (simd_lane_access_p
   11471         1634 :           && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
   11472         1634 :           && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
   11473         1634 :           && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
   11474         1634 :           && integer_zerop (DR_INIT (first_dr_info->dr))
   11475         1634 :           && alias_sets_conflict_p (get_alias_set (aggr_type),
   11476         1634 :                                     get_alias_set (TREE_TYPE (ref_type)))
   11477       152495 :           && (alignment_support_scheme == dr_aligned
   11478         1634 :               || alignment_support_scheme == dr_unaligned_supported))
   11479              :         {
   11480         1634 :           dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
   11481         1634 :           dataref_offset = build_int_cst (ref_type, 0);
   11482              :         }
   11483       150861 :       else if (diff_first_stmt_info)
   11484              :         {
   11485         3435 :           dataref_ptr
   11486         3435 :             = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
   11487              :                                         aggr_type, at_loop, offset, &dummy,
   11488              :                                         gsi, NULL, simd_lane_access_p,
   11489              :                                         dr_increment);
   11490              :           /* Adjust the pointer by the difference to first_stmt.  */
   11491         3435 :           data_reference_p ptrdr
   11492              :             = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
   11493         3435 :           tree diff = fold_convert (sizetype,
   11494              :                                     size_binop (MINUS_EXPR,
   11495              :                                                 DR_INIT (first_dr_info->dr),
   11496              :                                                 DR_INIT (ptrdr)));
   11497         3435 :           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
   11498              :                                          stmt_info, diff);
   11499         3435 :           if (alignment_support_scheme == dr_explicit_realign)
   11500              :             {
   11501            0 :               msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr,
   11502              :                                             vectype, gsi,
   11503              :                                             &realignment_token,
   11504              :                                             alignment_support_scheme,
   11505              :                                             dataref_ptr, &at_loop);
   11506            0 :               gcc_assert (!compute_in_loop);
   11507              :             }
   11508              :         }
   11509              :       else
   11510       147426 :         dataref_ptr
   11511       147426 :           = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
   11512              :                                       at_loop,
   11513              :                                       offset, &dummy, gsi, NULL,
   11514              :                                       simd_lane_access_p, dr_increment);
   11515              :     }
   11516              : 
   11517       589638 :   auto_vec<tree> dr_chain;
   11518       589638 :   if (grouped_load || ls.slp_perm)
   11519        57476 :     dr_chain.create (vec_num);
   11520              : 
   11521       589638 :   gimple *new_stmt = NULL;
   11522      1541659 :   for (i = 0; i < vec_num; i++)
   11523              :     {
   11524       952021 :       tree final_mask = NULL_TREE;
   11525       952021 :       tree final_len = NULL_TREE;
   11526       952021 :       tree bias = NULL_TREE;
   11527              : 
   11528       952021 :       if (!costing_p)
   11529              :         {
   11530       240864 :           if (mask_node)
   11531          661 :             vec_mask = vec_masks[i];
   11532       240864 :           if (loop_masks)
   11533           58 :             final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
   11534              :                                              vec_num, vectype, i);
   11535       240864 :           if (vec_mask)
   11536          661 :             final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
   11537              :                                            final_mask, vec_mask, gsi);
   11538              : 
   11539       240864 :           if (i > 0)
   11540        88369 :             dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
   11541              :                                            dr_bump);
   11542              :         }
   11543              : 
   11544              :       /* 2. Create the vector-load in the loop.  */
   11545       952021 :       switch (alignment_support_scheme)
   11546              :         {
   11547       952021 :         case dr_aligned:
   11548       952021 :         case dr_unaligned_supported:
   11549       952021 :           {
   11550       952021 :             if (costing_p)
   11551              :               break;
   11552              : 
   11553       240864 :             unsigned int misalign;
   11554       240864 :             unsigned HOST_WIDE_INT align;
   11555       240864 :             align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
   11556       240864 :             if (alignment_support_scheme == dr_aligned)
   11557              :               misalign = 0;
   11558       156479 :             else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
   11559              :               {
   11560       119071 :                 align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
   11561       119071 :                 misalign = 0;
   11562              :               }
   11563              :             else
   11564        37408 :               misalign = misalignment;
   11565       240864 :             if (dataref_offset == NULL_TREE
   11566       238732 :                 && TREE_CODE (dataref_ptr) == SSA_NAME)
   11567       172765 :               set_ssa_name_alignment (dataref_ptr, align, misalign);
   11568       240864 :             align = least_bit_hwi (misalign | align);
   11569              : 
   11570              :             /* Compute IFN when LOOP_LENS or final_mask valid.  */
   11571       240864 :             machine_mode vmode = TYPE_MODE (vectype);
   11572       240864 :             machine_mode new_vmode = vmode;
   11573       240864 :             internal_fn partial_ifn = IFN_LAST;
   11574       240864 :             if (loop_lens)
   11575              :               {
   11576            0 :                 opt_machine_mode new_ovmode
   11577            0 :                   = get_len_load_store_mode (vmode, true, &partial_ifn);
   11578            0 :                 new_vmode = new_ovmode.require ();
   11579            0 :                 unsigned factor
   11580            0 :                   = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
   11581            0 :                 final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
   11582              :                                                vec_num, vectype, i, factor, true);
   11583              :               }
   11584       240864 :             else if (final_mask)
   11585              :               {
   11586          699 :                 if (!can_vec_mask_load_store_p (vmode,
   11587          699 :                                                 TYPE_MODE
   11588              :                                                   (TREE_TYPE (final_mask)),
   11589              :                                                 true, &partial_ifn))
   11590            0 :                   gcc_unreachable ();
   11591              :               }
   11592              : 
   11593       240864 :             if (partial_ifn == IFN_MASK_LEN_LOAD)
   11594              :               {
   11595            0 :                 if (!final_len)
   11596              :                   {
   11597              :                     /* Pass VF value to 'len' argument of
   11598              :                        MASK_LEN_LOAD if LOOP_LENS is invalid.  */
   11599            0 :                     final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
   11600              :                   }
   11601            0 :                 if (!final_mask)
   11602              :                   {
   11603              :                     /* Pass all ones value to 'mask' argument of
   11604              :                        MASK_LEN_LOAD if final_mask is invalid.  */
   11605            0 :                     mask_vectype = truth_type_for (vectype);
   11606            0 :                     final_mask = build_minus_one_cst (mask_vectype);
   11607              :                   }
   11608              :               }
   11609       240864 :             if (final_len)
   11610              :               {
   11611            0 :                 signed char biasval
   11612            0 :                   = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
   11613            0 :                 bias = build_int_cst (intQI_type_node, biasval);
   11614              :               }
   11615              : 
   11616            0 :             tree vec_els;
   11617              : 
   11618            0 :             if (final_len)
   11619              :               {
   11620            0 :                 tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
   11621            0 :                 gcall *call;
   11622              : 
   11623              :                 /* Need conversion if the vectype is punned by VnQI.  */
   11624            0 :                 els_vectype = vectype;
   11625            0 :                 if (vmode != new_vmode)
   11626            0 :                   els_vectype
   11627            0 :                     = build_vector_type_for_mode (unsigned_intQI_type_node,
   11628              :                                                   new_vmode);
   11629            0 :                 vec_els = vect_get_mask_load_else (maskload_elsval,
   11630              :                                                    els_vectype);
   11631              : 
   11632            0 :                 if (partial_ifn == IFN_MASK_LEN_LOAD)
   11633              :                   {
   11634            0 :                     if (type_mode_padding_p
   11635            0 :                         && maskload_elsval != MASK_LOAD_ELSE_ZERO)
   11636            0 :                       need_zeroing = true;
   11637            0 :                     call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
   11638              :                                                        6, dataref_ptr, ptr,
   11639              :                                                        final_mask, vec_els,
   11640              :                                                        final_len, bias);
   11641              :                   }
   11642              :                 else
   11643            0 :                   call = gimple_build_call_internal (IFN_LEN_LOAD, 5,
   11644              :                                                      dataref_ptr, ptr,
   11645              :                                                      vec_els, final_len,
   11646              :                                                      bias);
   11647            0 :                 gimple_call_set_nothrow (call, true);
   11648            0 :                 new_stmt = call;
   11649            0 :                 data_ref = NULL_TREE;
   11650              : 
   11651              :                 /* Need conversion if it's wrapped with VnQI.  */
   11652            0 :                 if (vmode != new_vmode)
   11653              :                   {
   11654            0 :                     tree new_vtype
   11655            0 :                       = build_vector_type_for_mode (unsigned_intQI_type_node,
   11656              :                                                     new_vmode);
   11657            0 :                     tree var = vect_get_new_ssa_name (new_vtype,
   11658              :                                                       vect_simple_var);
   11659            0 :                     gimple_set_lhs (call, var);
   11660            0 :                     vect_finish_stmt_generation (vinfo, stmt_info, call,
   11661              :                                                  gsi);
   11662            0 :                     tree op = build1 (VIEW_CONVERT_EXPR, vectype, var);
   11663            0 :                     new_stmt = gimple_build_assign (vec_dest,
   11664              :                                                     VIEW_CONVERT_EXPR, op);
   11665              :                   }
   11666              :               }
   11667       240864 :             else if (final_mask)
   11668              :               {
   11669          699 :                 tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
   11670          699 :                 vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
   11671          699 :                 if (type_mode_padding_p
   11672          699 :                     && maskload_elsval != MASK_LOAD_ELSE_ZERO)
   11673            0 :                   need_zeroing = true;
   11674          699 :                 gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
   11675              :                                                           dataref_ptr, ptr,
   11676              :                                                           final_mask,
   11677              :                                                           vec_els);
   11678          699 :                 gimple_call_set_nothrow (call, true);
   11679          699 :                 new_stmt = call;
   11680          699 :                 data_ref = NULL_TREE;
   11681              :               }
   11682              :             else
   11683              :               {
   11684       240165 :                 tree ltype = vectype;
   11685       240165 :                 tree new_vtype = NULL_TREE;
   11686       240165 :                 unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
   11687       240165 :                 unsigned HOST_WIDE_INT dr_size
   11688       240165 :                   = vect_get_scalar_dr_size (first_dr_info);
   11689       240165 :                 poly_int64 off = 0;
   11690       240165 :                 if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
   11691         1488 :                   off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
   11692       240165 :                 unsigned int vect_align
   11693       240165 :                   = vect_known_alignment_in_bytes (first_dr_info, vectype,
   11694       240165 :                                                    off);
   11695              :                 /* Try to use a single smaller load when we are about
   11696              :                    to load excess elements compared to the unrolled
   11697              :                    scalar loop.  */
   11698       240165 :                 if (known_gt ((i + 1) * nunits,
   11699              :                               (group_size * vf - gap)))
   11700              :                   {
   11701         7035 :                     poly_uint64 remain = ((group_size * vf - gap) - i * nunits);
   11702         7035 :                     if (known_ge ((i + 1) * nunits - (group_size * vf - gap),
   11703              :                                   nunits))
   11704              :                       /* DR will be unused.  */
   11705              :                       ltype = NULL_TREE;
   11706         2381 :                     else if (known_ge (vect_align,
   11707              :                                        tree_to_poly_uint64
   11708              :                                          (TYPE_SIZE_UNIT (vectype))))
   11709              :                       /* Aligned access to excess elements is OK if
   11710              :                          at least one element is accessed in the
   11711              :                          scalar loop.  */
   11712              :                       ;
   11713         1977 :                     else if (known_gt (vect_align,
   11714              :                                        ((nunits - remain) * dr_size)))
   11715              :                       /* Aligned access to the gap area when there's
   11716              :                          at least one element in it is OK.  */
   11717              :                       ;
   11718              :                     else
   11719              :                       {
   11720              :                         /* remain should now be > 0 and < nunits.  */
   11721         1972 :                         unsigned num;
   11722         1972 :                         if (known_ne (remain, 0u)
   11723         1972 :                             && constant_multiple_p (nunits, remain, &num))
   11724              :                           {
   11725         1480 :                             tree ptype;
   11726         1480 :                             new_vtype
   11727         1480 :                               = vector_vector_composition_type (vectype, num,
   11728              :                                                                 &ptype);
   11729         1480 :                             if (new_vtype)
   11730         1480 :                               ltype = ptype;
   11731              :                           }
   11732              :                         /* Else use multiple loads or a masked load?  */
   11733              :                         /* For loop vectorization we now should have
   11734              :                            an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
   11735              :                            set.  */
   11736         1972 :                         if (loop_vinfo)
   11737         1730 :                           gcc_assert (new_vtype
   11738              :                                       || LOOP_VINFO_PEELING_FOR_GAPS
   11739              :                                            (loop_vinfo));
   11740              :                         /* But still reduce the access size to the next
   11741              :                            required power-of-two so peeling a single
   11742              :                            scalar iteration is sufficient.  */
   11743         1972 :                         unsigned HOST_WIDE_INT cremain;
   11744         1972 :                         if (remain.is_constant (&cremain))
   11745              :                           {
   11746         1972 :                             unsigned HOST_WIDE_INT cpart_size
   11747         1972 :                               = 1 << ceil_log2 (cremain);
   11748         1972 :                             if (known_gt (nunits, cpart_size)
   11749         1972 :                                 && constant_multiple_p (nunits, cpart_size,
   11750              :                                                         &num))
   11751              :                               {
   11752         1492 :                                 tree ptype;
   11753         1492 :                                 new_vtype
   11754         2984 :                                   = vector_vector_composition_type (vectype,
   11755         1492 :                                                                     num,
   11756              :                                                                     &ptype);
   11757         1492 :                                 if (new_vtype)
   11758         1492 :                                   ltype = ptype;
   11759              :                               }
   11760              :                           }
   11761              :                       }
   11762              :                   }
   11763       240165 :                 tree offset = (dataref_offset ? dataref_offset
   11764       238033 :                                : build_int_cst (ref_type, 0));
   11765       240165 :                 if (!ltype)
   11766              :                   ;
   11767       235511 :                 else if (ltype != vectype
   11768       235511 :                          && memory_access_type == VMAT_CONTIGUOUS_REVERSE)
   11769              :                   {
   11770           21 :                     poly_uint64 gap_offset
   11771           21 :                       = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype))
   11772           21 :                          - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype)));
   11773           21 :                     tree gapcst = build_int_cstu (ref_type, gap_offset);
   11774           21 :                     offset = size_binop (PLUS_EXPR, offset, gapcst);
   11775              :                   }
   11776       235511 :                 if (ltype)
   11777              :                   {
   11778       235511 :                     data_ref = fold_build2 (MEM_REF, ltype,
   11779              :                                             dataref_ptr, offset);
   11780       235511 :                     if (alignment_support_scheme == dr_aligned
   11781       235511 :                         && align >= TYPE_ALIGN_UNIT (ltype))
   11782              :                       ;
   11783              :                     else
   11784       154793 :                       TREE_TYPE (data_ref)
   11785       309586 :                         = build_aligned_type (TREE_TYPE (data_ref),
   11786              :                                               align * BITS_PER_UNIT);
   11787              :                   }
   11788       235511 :                 if (!ltype)
   11789         4654 :                   data_ref = build_constructor (vectype, NULL);
   11790       235511 :                 else if (ltype != vectype)
   11791              :                   {
   11792         1492 :                     vect_copy_ref_info (data_ref,
   11793         1492 :                                         DR_REF (first_dr_info->dr));
   11794         1492 :                     tree tem = make_ssa_name (ltype);
   11795         1492 :                     new_stmt = gimple_build_assign (tem, data_ref);
   11796         1492 :                     vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
   11797              :                                                  gsi);
   11798         1492 :                     data_ref = NULL;
   11799         1492 :                     vec<constructor_elt, va_gc> *v;
   11800              :                     /* We've computed 'num' above to statically two
   11801              :                        or via constant_multiple_p.  */
   11802         1492 :                     unsigned num
   11803         1492 :                       = (exact_div (tree_to_poly_uint64
   11804         1492 :                                       (TYPE_SIZE_UNIT (vectype)),
   11805              :                                     tree_to_poly_uint64
   11806         1492 :                                       (TYPE_SIZE_UNIT (ltype)))
   11807         1492 :                          .to_constant ());
   11808         1492 :                     vec_alloc (v, num);
   11809         1492 :                     if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
   11810              :                       {
   11811           54 :                         while (--num)
   11812           54 :                           CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
   11813              :                                                   build_zero_cst (ltype));
   11814           21 :                         CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
   11815              :                       }
   11816              :                     else
   11817              :                       {
   11818         1471 :                         CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
   11819         1471 :                         while (--num)
   11820         3384 :                           CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
   11821              :                                                   build_zero_cst (ltype));
   11822              :                       }
   11823         1492 :                     gcc_assert (new_vtype != NULL_TREE);
   11824         1492 :                     if (new_vtype == vectype)
   11825         1460 :                       new_stmt
   11826         1460 :                         = gimple_build_assign (vec_dest,
   11827              :                                                build_constructor (vectype, v));
   11828              :                     else
   11829              :                       {
   11830           32 :                         tree new_vname = make_ssa_name (new_vtype);
   11831           32 :                         new_stmt
   11832           32 :                           = gimple_build_assign (new_vname,
   11833              :                                                  build_constructor (new_vtype,
   11834              :                                                                     v));
   11835           32 :                         vect_finish_stmt_generation (vinfo, stmt_info,
   11836              :                                                      new_stmt, gsi);
   11837           32 :                         new_stmt
   11838           32 :                           = gimple_build_assign (vec_dest,
   11839              :                                                  build1 (VIEW_CONVERT_EXPR,
   11840              :                                                          vectype, new_vname));
   11841              :                       }
   11842              :                   }
   11843              :               }
   11844              :             break;
   11845              :           }
   11846            0 :         case dr_explicit_realign:
   11847            0 :           {
   11848            0 :             if (costing_p)
   11849              :               break;
   11850            0 :             tree ptr, bump;
   11851              : 
   11852            0 :             tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
   11853              : 
   11854            0 :             if (compute_in_loop)
   11855            0 :               msq = vect_setup_realignment (vinfo, first_stmt_info, vectype,
   11856              :                                             gsi, &realignment_token,
   11857              :                                             dr_explicit_realign,
   11858              :                                             dataref_ptr, NULL);
   11859              : 
   11860            0 :             if (TREE_CODE (dataref_ptr) == SSA_NAME)
   11861            0 :               ptr = copy_ssa_name (dataref_ptr);
   11862              :             else
   11863            0 :               ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
   11864              :             // For explicit realign the target alignment should be
   11865              :             // known at compile time.
   11866            0 :             unsigned HOST_WIDE_INT align
   11867            0 :               = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
   11868            0 :             new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr,
   11869              :                                             build_int_cst
   11870            0 :                                               (TREE_TYPE (dataref_ptr),
   11871            0 :                                                -(HOST_WIDE_INT) align));
   11872            0 :             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11873            0 :             data_ref = build2 (MEM_REF, vectype,
   11874              :                                ptr, build_int_cst (ref_type, 0));
   11875            0 :             vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
   11876            0 :             vec_dest = vect_create_destination_var (scalar_dest, vectype);
   11877            0 :             new_stmt = gimple_build_assign (vec_dest, data_ref);
   11878            0 :             new_temp = make_ssa_name (vec_dest, new_stmt);
   11879            0 :             gimple_assign_set_lhs (new_stmt, new_temp);
   11880            0 :             gimple_move_vops (new_stmt, stmt_info->stmt);
   11881            0 :             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11882            0 :             msq = new_temp;
   11883              : 
   11884            0 :             bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
   11885            0 :             bump = size_binop (MINUS_EXPR, bump, size_one_node);
   11886            0 :             ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
   11887            0 :             new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
   11888            0 :                                             build_int_cst (TREE_TYPE (ptr),
   11889            0 :                                                            -(HOST_WIDE_INT) align));
   11890            0 :             if (TREE_CODE (ptr) == SSA_NAME)
   11891            0 :               ptr = copy_ssa_name (ptr, new_stmt);
   11892              :             else
   11893            0 :               ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt);
   11894            0 :             gimple_assign_set_lhs (new_stmt, ptr);
   11895            0 :             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11896            0 :             data_ref = build2 (MEM_REF, vectype,
   11897              :                                ptr, build_int_cst (ref_type, 0));
   11898            0 :             break;
   11899              :           }
   11900            0 :         case dr_explicit_realign_optimized:
   11901            0 :           {
   11902            0 :             if (costing_p)
   11903              :               break;
   11904            0 :             if (TREE_CODE (dataref_ptr) == SSA_NAME)
   11905            0 :               new_temp = copy_ssa_name (dataref_ptr);
   11906              :             else
   11907            0 :               new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
   11908              :             // We should only be doing this if we know the target
   11909              :             // alignment at compile time.
   11910            0 :             unsigned HOST_WIDE_INT align
   11911            0 :               = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
   11912            0 :             new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr,
   11913            0 :                                             build_int_cst (TREE_TYPE (dataref_ptr),
   11914            0 :                                                            -(HOST_WIDE_INT) align));
   11915            0 :             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11916            0 :             data_ref = build2 (MEM_REF, vectype, new_temp,
   11917              :                                build_int_cst (ref_type, 0));
   11918            0 :             break;
   11919              :           }
   11920            0 :         default:
   11921            0 :         gcc_unreachable ();
   11922              :         }
   11923              : 
   11924              :       /* One common place to cost the above vect load for different
   11925              :          alignment support schemes.  */
   11926       952021 :       if (costing_p)
   11927              :         {
   11928              :           /* For the prologue cost for realign,
   11929              :              we only need to count it once for the whole group.  */
   11930       711157 :           bool first_stmt_info_p = first_stmt_info == stmt_info;
   11931       711157 :           bool add_realign_cost = first_stmt_info_p && i == 0;
   11932       711157 :           if (memory_access_type == VMAT_CONTIGUOUS
   11933       711157 :               || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
   11934              :             {
   11935              :               /* Leave realign cases alone to keep them simple.  */
   11936       711157 :               if (alignment_support_scheme == dr_explicit_realign_optimized
   11937              :                   || alignment_support_scheme == dr_explicit_realign)
   11938            0 :                 vect_get_load_cost (vinfo, stmt_info, slp_node, 1,
   11939              :                                     alignment_support_scheme, misalignment,
   11940              :                                     add_realign_cost, &inside_cost,
   11941              :                                     &prologue_cost, cost_vec, cost_vec,
   11942              :                                     true);
   11943              :               else
   11944       711157 :                 n_adjacent_loads++;
   11945              :             }
   11946              :         }
   11947              :       else
   11948              :         {
   11949       240864 :           vec_dest = vect_create_destination_var (scalar_dest, vectype);
   11950              :           /* DATA_REF is null if we've already built the statement.  */
   11951       240864 :           if (data_ref)
   11952              :             {
   11953       238673 :               vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
   11954       238673 :               new_stmt = gimple_build_assign (vec_dest, data_ref);
   11955              :             }
   11956              : 
   11957       481728 :           new_temp = (need_zeroing
   11958       240864 :                       ? make_ssa_name (vectype)
   11959       240864 :                       : make_ssa_name (vec_dest, new_stmt));
   11960       240864 :           gimple_set_lhs (new_stmt, new_temp);
   11961       240864 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11962              : 
   11963              :           /* If we need to explicitly zero inactive elements emit a
   11964              :              VEC_COND_EXPR that does so.  */
   11965       240864 :           if (need_zeroing)
   11966              :             {
   11967            0 :               vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
   11968              :                                                  vectype);
   11969              : 
   11970            0 :               tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
   11971            0 :               new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
   11972              :                                               final_mask, new_temp, vec_els);
   11973            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
   11974              :                                            gsi);
   11975            0 :               new_temp = new_temp2;
   11976              :             }
   11977              :         }
   11978              : 
   11979              :       /* 3. Handle explicit realignment if necessary/supported.
   11980              :          Create in loop:
   11981              :          vec_dest = realign_load (msq, lsq, realignment_token)  */
   11982       952021 :       if (!costing_p
   11983       240864 :           && (alignment_support_scheme == dr_explicit_realign_optimized
   11984              :               || alignment_support_scheme == dr_explicit_realign))
   11985              :         {
   11986            0 :           lsq = gimple_assign_lhs (new_stmt);
   11987            0 :           if (!realignment_token)
   11988            0 :             realignment_token = dataref_ptr;
   11989            0 :           vec_dest = vect_create_destination_var (scalar_dest, vectype);
   11990            0 :           new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq,
   11991              :                                           lsq, realignment_token);
   11992            0 :           new_temp = make_ssa_name (vec_dest, new_stmt);
   11993            0 :           gimple_assign_set_lhs (new_stmt, new_temp);
   11994            0 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   11995              : 
   11996            0 :           if (alignment_support_scheme == dr_explicit_realign_optimized)
   11997              :             {
   11998            0 :               gcc_assert (phi);
   11999            0 :               if (i == vec_num - 1)
   12000            0 :                 add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
   12001              :                              UNKNOWN_LOCATION);
   12002              :               msq = lsq;
   12003              :             }
   12004              :         }
   12005              : 
   12006       952021 :       if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
   12007              :         {
   12008         6089 :           if (costing_p)
   12009         4601 :             inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
   12010              :                                             slp_node, 0, vect_body);
   12011              :           else
   12012              :             {
   12013         1488 :               tree perm_mask = perm_mask_for_reverse (vectype);
   12014         1488 :               new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
   12015              :                                                perm_mask, stmt_info, gsi);
   12016         1488 :               new_stmt = SSA_NAME_DEF_STMT (new_temp);
   12017              :             }
   12018              :         }
   12019              : 
   12020              :       /* Collect vector loads and later create their permutation in
   12021              :          vect_transform_slp_perm_load.  */
   12022       952021 :       if (!costing_p && (grouped_load || ls.slp_perm))
   12023        73735 :         dr_chain.quick_push (new_temp);
   12024              : 
   12025              :       /* Store vector loads in the corresponding SLP_NODE.  */
   12026       240864 :       if (!costing_p && !ls.slp_perm)
   12027       167129 :         slp_node->push_vec_def (new_stmt);
   12028              : 
   12029              :       /* With SLP permutation we load the gaps as well, without
   12030              :          we need to skip the gaps after we manage to fully load
   12031              :          all elements.  group_gap_adj is DR_GROUP_SIZE here.  */
   12032       952021 :       group_elt += nunits;
   12033       952021 :       if (!costing_p
   12034       240864 :           && maybe_ne (group_gap_adj, 0U)
   12035        40410 :           && !ls.slp_perm
   12036       967496 :           && known_eq (group_elt, group_size - group_gap_adj))
   12037              :         {
   12038        12802 :           poly_wide_int bump_val
   12039        12802 :             = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
   12040        12802 :           if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
   12041            0 :             bump_val = -bump_val;
   12042        12802 :           tree bump = wide_int_to_tree (sizetype, bump_val);
   12043        12802 :           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
   12044              :                                          bump);
   12045        12802 :           group_elt = 0;
   12046        12802 :         }
   12047              :     }
   12048              :   /* Bump the vector pointer to account for a gap or for excess
   12049              :      elements loaded for a permuted SLP load.  */
   12050       589638 :   if (!costing_p
   12051       152495 :       && maybe_ne (group_gap_adj, 0U)
   12052       602908 :       && ls.slp_perm)
   12053              :     {
   12054          468 :       poly_wide_int bump_val
   12055          468 :         = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
   12056          468 :       if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
   12057            9 :         bump_val = -bump_val;
   12058          468 :       tree bump = wide_int_to_tree (sizetype, bump_val);
   12059          468 :       dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
   12060          468 :     }
   12061              : 
   12062       589638 :   if (ls.slp_perm)
   12063              :     {
   12064              :       /* For SLP we know we've seen all possible uses of dr_chain so
   12065              :          direct vect_transform_slp_perm_load to DCE the unused parts.
   12066              :          ???  This is a hack to prevent compile-time issues as seen
   12067              :          in PR101120 and friends.  */
   12068        57476 :       if (costing_p)
   12069              :         {
   12070        40576 :           gcc_assert (ls.n_perms != -1U && ls.n_loads != -1U);
   12071        40576 :           if (ls.n_perms != 0)
   12072        40042 :             inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
   12073              :                                             slp_node, 0, vect_body);
   12074        40576 :           if (n_adjacent_loads > 0)
   12075        40576 :             n_adjacent_loads = ls.n_loads;
   12076              :         }
   12077              :       else
   12078              :         {
   12079        16900 :           unsigned n_perms2, n_loads2;
   12080        16900 :           bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
   12081              :                                                   gsi, vf, false, &n_perms2,
   12082              :                                                   &n_loads2, true);
   12083        16900 :           gcc_assert (ok && ls.n_perms == n_perms2 && ls.n_loads == n_loads2);
   12084              :         }
   12085              :     }
   12086              : 
   12087       589638 :   if (costing_p)
   12088              :     {
   12089       437143 :       gcc_assert (memory_access_type == VMAT_CONTIGUOUS
   12090              :                   || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
   12091       437143 :       if (n_adjacent_loads > 0)
   12092       437143 :         vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
   12093              :                             alignment_support_scheme, misalignment, false,
   12094              :                             &inside_cost, &prologue_cost, cost_vec, cost_vec,
   12095              :                             true);
   12096       437143 :       if (dump_enabled_p ())
   12097        25930 :         dump_printf_loc (MSG_NOTE, vect_location,
   12098              :                          "vect_model_load_cost: inside_cost = %u, "
   12099              :                          "prologue_cost = %u .\n",
   12100              :                          inside_cost, prologue_cost);
   12101       437143 :       SLP_TREE_TYPE (slp_node) = load_vec_info_type;
   12102       437143 :       slp_node->data = new vect_load_store_data (std::move (ls));
   12103              :     }
   12104              : 
   12105       589638 :   return true;
   12106      1331606 : }
   12107              : 
   12108              : /* vectorizable_condition.
   12109              : 
   12110              :    Check if STMT_INFO is conditional modify expression that can be vectorized.
   12111              :    If COST_VEC is passed, calculate costs but don't change anything,
   12112              :    otherwise, vectorize STMT_INFO: create a vectorized stmt using
   12113              :    VEC_COND_EXPR to replace it, and insert it at GSI.
   12114              : 
   12115              :    When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
   12116              : 
   12117              :    Return true if STMT_INFO is vectorizable in this way.  */
   12118              : 
   12119              : static bool
   12120       739869 : vectorizable_condition (vec_info *vinfo,
   12121              :                         stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   12122              :                         slp_tree slp_node, stmt_vector_for_cost *cost_vec)
   12123              : {
   12124       739869 :   tree scalar_dest = NULL_TREE;
   12125       739869 :   tree vec_dest = NULL_TREE;
   12126       739869 :   tree then_clause, else_clause;
   12127       739869 :   tree vec_cond_lhs = NULL_TREE;
   12128       739869 :   tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
   12129       739869 :   tree vec_compare;
   12130       739869 :   tree new_temp;
   12131       739869 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   12132       739869 :   enum vect_def_type dts[4]
   12133              :     = {vect_unknown_def_type, vect_unknown_def_type,
   12134              :        vect_unknown_def_type, vect_unknown_def_type};
   12135       739869 :   enum tree_code code;
   12136       739869 :   int i;
   12137       739869 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
   12138       739869 :   vec<tree> vec_oprnds0 = vNULL;
   12139       739869 :   vec<tree> vec_oprnds2 = vNULL;
   12140       739869 :   vec<tree> vec_oprnds3 = vNULL;
   12141       739869 :   tree vec_cmp_type;
   12142              : 
   12143       739869 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
   12144              :     return false;
   12145              : 
   12146              :   /* Is vectorizable conditional operation?  */
   12147       739869 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
   12148       421212 :   if (!stmt)
   12149              :     return false;
   12150              : 
   12151       421212 :   code = gimple_assign_rhs_code (stmt);
   12152       421212 :   if (code != COND_EXPR)
   12153              :     return false;
   12154              : 
   12155        35965 :   int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
   12156        35965 :   vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
   12157        35965 :   bool nested_cycle_p = false;
   12158        35965 :   bool for_reduction = vect_is_reduction (stmt_info);
   12159        35965 :   if (for_reduction)
   12160              :     {
   12161          648 :       if (SLP_TREE_LANES (slp_node) > 1)
   12162              :         return false;
   12163              :       /* ???  With a reduction path we do not get at the reduction info from
   12164              :          every stmt, use the conservative default setting then.  */
   12165          729 :       if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
   12166              :         {
   12167          630 :           vect_reduc_info reduc_info
   12168          630 :             = info_for_reduction (loop_vinfo, slp_node);
   12169          630 :           reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
   12170          630 :           nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
   12171              :                                                   stmt_info);
   12172              :         }
   12173              :     }
   12174              :   else
   12175              :     {
   12176        35317 :       if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
   12177              :         return false;
   12178              :     }
   12179              : 
   12180        35965 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
   12181        35965 :   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
   12182              : 
   12183        35965 :   int vec_num = vect_get_num_copies (vinfo, slp_node);
   12184              : 
   12185        35965 :   slp_tree slp_cond;
   12186        35965 :   tree cond_expr = gimple_assign_rhs1 (stmt);
   12187        35965 :   gcc_assert (! COMPARISON_CLASS_P (cond_expr));
   12188        35965 :   if (TREE_CODE (cond_expr) != SSA_NAME
   12189        35965 :       || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond_expr))
   12190        35965 :       || !vect_is_simple_use (vinfo, slp_node, 0,
   12191              :                               &slp_cond, &dts[0], &vec_cmp_type)
   12192        35965 :       || !vec_cmp_type
   12193        71915 :       || !VECTOR_BOOLEAN_TYPE_P (vec_cmp_type))
   12194              :     return false;
   12195              : 
   12196        35950 :   slp_tree then_slp_node, else_slp_node;
   12197        35950 :   if (!vect_is_simple_use (vinfo, slp_node, 1,
   12198              :                            &then_clause, &then_slp_node, &dts[2], &vectype1))
   12199              :     return false;
   12200        35950 :   if (!vect_is_simple_use (vinfo, slp_node, 2,
   12201              :                            &else_clause, &else_slp_node, &dts[3], &vectype2))
   12202              :     return false;
   12203              : 
   12204        35950 :   if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
   12205              :     return false;
   12206              : 
   12207        35950 :   if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
   12208              :     return false;
   12209              : 
   12210        35938 :   if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
   12211        71876 :                 TYPE_VECTOR_SUBPARTS (vec_cmp_type)))
   12212              :     return false;
   12213              : 
   12214              :   /* For conditional reductions, the "then" value needs to be the candidate
   12215              :      value calculated by this iteration while the "else" value needs to be
   12216              :      the result carried over from previous iterations.  If the COND_EXPR
   12217              :      is the other way around, we need to swap it.  */
   12218        35938 :   bool must_invert_cmp_result = false;
   12219        35938 :   if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
   12220              :     {
   12221       739869 :       must_invert_cmp_result = true;
   12222              :       /* ???  The vectorized operand query below doesn't allow swapping
   12223              :          this way for SLP.  */
   12224              :       return false;
   12225              :       /* std::swap (then_clause, else_clause); */
   12226              :     }
   12227              : 
   12228        35938 :   if (cost_vec)
   12229              :     {
   12230        27199 :       vect_cost_for_stmt kind = vector_stmt;
   12231        27199 :       if (reduction_type == EXTRACT_LAST_REDUCTION)
   12232              :         /* Count one reduction-like operation per vector.  */
   12233              :         kind = vec_to_scalar;
   12234        27199 :       else if (!expand_vec_cond_expr_p (vectype, vec_cmp_type))
   12235              :         return false;
   12236              : 
   12237        27185 :       if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
   12238              :                                              vec_cmp_type)
   12239        27185 :           || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
   12240        54370 :           || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
   12241              :         {
   12242            0 :           if (dump_enabled_p ())
   12243            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12244              :                              "incompatible vector types for invariants\n");
   12245              :           return false;
   12246              :         }
   12247              : 
   12248        27185 :       if (loop_vinfo && for_reduction
   12249          472 :           && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
   12250              :         {
   12251           81 :           if (reduction_type == EXTRACT_LAST_REDUCTION)
   12252              :             {
   12253            0 :               if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
   12254              :                                                   vectype, OPTIMIZE_FOR_SPEED))
   12255            0 :                 vect_record_loop_len (loop_vinfo,
   12256              :                                       &LOOP_VINFO_LENS (loop_vinfo),
   12257              :                                       vec_num, vectype, 1);
   12258              :               else
   12259            0 :                 vect_record_loop_mask (loop_vinfo,
   12260              :                                        &LOOP_VINFO_MASKS (loop_vinfo),
   12261              :                                        vec_num, vectype, NULL);
   12262              :             }
   12263              :           /* Extra inactive lanes should be safe for vect_nested_cycle.  */
   12264           81 :           else if (!nested_cycle_p)
   12265              :             {
   12266           81 :               if (dump_enabled_p ())
   12267            8 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12268              :                                  "conditional reduction prevents the use"
   12269              :                                  " of partial vectors.\n");
   12270           81 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
   12271              :             }
   12272              :         }
   12273              : 
   12274        27185 :       SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
   12275        27185 :       vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
   12276        27185 :       return true;
   12277              :     }
   12278              : 
   12279              :   /* Transform.  */
   12280              : 
   12281              :   /* Handle def.  */
   12282         8739 :   scalar_dest = gimple_assign_lhs (stmt);
   12283         8739 :   if (reduction_type != EXTRACT_LAST_REDUCTION)
   12284         8739 :     vec_dest = vect_create_destination_var (scalar_dest, vectype);
   12285              : 
   12286         8739 :   bool swap_cond_operands = false;
   12287              : 
   12288              :   /* See whether another part of the vectorized code applies a loop
   12289              :      mask to the condition, or to its inverse.  */
   12290              : 
   12291         8739 :   vec_loop_masks *masks = NULL;
   12292         8739 :   vec_loop_lens *lens = NULL;
   12293         8739 :   if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
   12294              :     {
   12295            0 :       if (reduction_type == EXTRACT_LAST_REDUCTION)
   12296            0 :         lens = &LOOP_VINFO_LENS (loop_vinfo);
   12297              :     }
   12298         8739 :   else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
   12299              :     {
   12300            3 :       if (reduction_type == EXTRACT_LAST_REDUCTION)
   12301            0 :         masks = &LOOP_VINFO_MASKS (loop_vinfo);
   12302              :       else
   12303              :         {
   12304            3 :           scalar_cond_masked_key cond (cond_expr, 1);
   12305            3 :           if (loop_vinfo->scalar_cond_masked_set.contains (cond))
   12306            0 :             masks = &LOOP_VINFO_MASKS (loop_vinfo);
   12307              :           else
   12308              :             {
   12309            3 :               bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
   12310            3 :               tree_code orig_code = cond.code;
   12311            3 :               cond.code = invert_tree_comparison (cond.code, honor_nans);
   12312              :               /* Try the inverse of the current mask.  We check if the
   12313              :                  inverse mask is live and if so we generate a negate of
   12314              :                  the current mask such that we still honor NaNs.  */
   12315            3 :               cond.inverted_p = true;
   12316            3 :               cond.code = orig_code;
   12317            3 :               if (loop_vinfo->scalar_cond_masked_set.contains (cond))
   12318              :                 {
   12319            0 :                   masks = &LOOP_VINFO_MASKS (loop_vinfo);
   12320            0 :                   swap_cond_operands = true;
   12321            0 :                   must_invert_cmp_result = true;
   12322              :                 }
   12323              :             }
   12324              :         }
   12325              :     }
   12326              : 
   12327              :   /* Handle cond expr.  */
   12328         8739 :   vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds2,
   12329              :                      reduction_type != EXTRACT_LAST_REDUCTION, &vec_oprnds3);
   12330              : 
   12331         8739 :   if (reduction_type == EXTRACT_LAST_REDUCTION)
   12332            0 :     vec_else_clause = else_clause;
   12333              : 
   12334              :   /* Arguments are ready.  Create the new vector stmt.  */
   12335        20311 :   FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
   12336              :     {
   12337        11572 :       vec_then_clause = vec_oprnds2[i];
   12338        11572 :       if (reduction_type != EXTRACT_LAST_REDUCTION)
   12339        11572 :         vec_else_clause = vec_oprnds3[i];
   12340              : 
   12341        11572 :       if (swap_cond_operands)
   12342            0 :         std::swap (vec_then_clause, vec_else_clause);
   12343              : 
   12344        11572 :       vec_compare = vec_cond_lhs;
   12345              : 
   12346              :       /* If we decided to apply a loop mask to the result of the vector
   12347              :          comparison, AND the comparison with the mask now.  Later passes
   12348              :          should then be able to reuse the AND results between multiple
   12349              :          vector statements.
   12350              : 
   12351              :          For example:
   12352              :          for (int i = 0; i < 100; ++i)
   12353              :          x[i] = y[i] ? z[i] : 10;
   12354              : 
   12355              :          results in following optimized GIMPLE:
   12356              : 
   12357              :          mask__35.8_43 = vect__4.7_41 != { 0, ... };
   12358              :          vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
   12359              :          _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
   12360              :          vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
   12361              :          vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
   12362              :          vect_iftmp.11_47, { 10, ... }>;
   12363              : 
   12364              :          instead of using a masked and unmasked forms of
   12365              :          vec != { 0, ... } (masked in the MASK_LOAD,
   12366              :          unmasked in the VEC_COND_EXPR).  */
   12367              : 
   12368              :       /* Force vec_compare to be an SSA_NAME rather than a comparison,
   12369              :          in cases where that's necessary.  */
   12370              : 
   12371        11572 :       tree len = NULL_TREE, bias = NULL_TREE;
   12372        11572 :       if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
   12373              :         {
   12374            0 :           if (!is_gimple_val (vec_compare))
   12375              :             {
   12376            0 :               tree vec_compare_name = make_ssa_name (vec_cmp_type);
   12377            0 :               gassign *new_stmt = gimple_build_assign (vec_compare_name,
   12378              :                                                        vec_compare);
   12379            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12380            0 :               vec_compare = vec_compare_name;
   12381              :             }
   12382              : 
   12383            0 :           if (must_invert_cmp_result)
   12384              :             {
   12385            0 :               tree vec_compare_name = make_ssa_name (vec_cmp_type);
   12386            0 :               gassign *new_stmt = gimple_build_assign (vec_compare_name,
   12387              :                                                        BIT_NOT_EXPR,
   12388              :                                                        vec_compare);
   12389            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12390            0 :               vec_compare = vec_compare_name;
   12391              :             }
   12392              : 
   12393            0 :           if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
   12394              :                                               vectype, OPTIMIZE_FOR_SPEED))
   12395              :             {
   12396            0 :               if (lens)
   12397              :                 {
   12398              :                   /* ??? Do we really want the adjusted LEN here?  Isn't this
   12399              :                      based on number of elements?  */
   12400            0 :                   len = vect_get_loop_len (loop_vinfo, gsi, lens,
   12401              :                                            vec_num, vectype, i, 1, true);
   12402            0 :                   signed char biasval
   12403            0 :                     = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
   12404            0 :                   bias = build_int_cst (intQI_type_node, biasval);
   12405              :                 }
   12406              :               else
   12407              :                 {
   12408            0 :                   len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
   12409            0 :                   bias = build_int_cst (intQI_type_node, 0);
   12410              :                 }
   12411              :             }
   12412            0 :           if (masks)
   12413              :             {
   12414            0 :               tree loop_mask
   12415            0 :                 = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
   12416              :                                       vectype, i);
   12417            0 :               tree tmp2 = make_ssa_name (vec_cmp_type);
   12418            0 :               gassign *g
   12419            0 :                 = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
   12420              :                                        loop_mask);
   12421            0 :               vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
   12422            0 :               vec_compare = tmp2;
   12423              :             }
   12424              :         }
   12425              : 
   12426            0 :       gimple *new_stmt;
   12427            0 :       if (reduction_type == EXTRACT_LAST_REDUCTION)
   12428              :         {
   12429            0 :           gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
   12430            0 :           tree lhs = gimple_get_lhs (old_stmt);
   12431            0 :           if ((unsigned)i != vec_oprnds0.length () - 1)
   12432            0 :             lhs = copy_ssa_name (lhs);
   12433            0 :           if (len)
   12434            0 :             new_stmt = gimple_build_call_internal
   12435            0 :                 (IFN_LEN_FOLD_EXTRACT_LAST, 5, vec_else_clause, vec_compare,
   12436              :                  vec_then_clause, len, bias);
   12437              :           else
   12438            0 :             new_stmt = gimple_build_call_internal
   12439            0 :                 (IFN_FOLD_EXTRACT_LAST, 3, vec_else_clause, vec_compare,
   12440              :                  vec_then_clause);
   12441            0 :           gimple_call_set_lhs (new_stmt, lhs);
   12442            0 :           SSA_NAME_DEF_STMT (lhs) = new_stmt;
   12443            0 :           if ((unsigned)i != vec_oprnds0.length () - 1)
   12444              :             {
   12445            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12446            0 :               vec_else_clause = lhs;
   12447              :             }
   12448            0 :           else if (old_stmt == gsi_stmt (*gsi))
   12449            0 :             vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
   12450              :           else
   12451              :             {
   12452              :               /* In this case we're moving the definition to later in the
   12453              :                  block.  That doesn't matter because the only uses of the
   12454              :                  lhs are in phi statements.  */
   12455            0 :               gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
   12456            0 :               gsi_remove (&old_gsi, true);
   12457            0 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12458              :             }
   12459              :         }
   12460              :       else
   12461              :         {
   12462        11572 :           new_temp = make_ssa_name (vec_dest);
   12463        11572 :           new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
   12464              :                                           vec_then_clause, vec_else_clause);
   12465        11572 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12466              :         }
   12467        11572 :       slp_node->push_vec_def (new_stmt);
   12468              :     }
   12469              : 
   12470         8739 :   vec_oprnds0.release ();
   12471         8739 :   vec_oprnds2.release ();
   12472         8739 :   vec_oprnds3.release ();
   12473              : 
   12474         8739 :   return true;
   12475              : }
   12476              : 
   12477              : /* Helper of vectorizable_comparison.
   12478              : 
   12479              :    Check if STMT_INFO is comparison expression CODE that can be vectorized.
   12480              :    If COST_VEC is passed, calculate costs but don't change anything,
   12481              :    otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
   12482              :    it at GSI.
   12483              : 
   12484              :    Return true if STMT_INFO is vectorizable in this way.  */
   12485              : 
   12486              : static bool
   12487       393668 : vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
   12488              :                            stmt_vec_info stmt_info, tree_code code,
   12489              :                            gimple_stmt_iterator *gsi,
   12490              :                            slp_tree slp_node, stmt_vector_for_cost *cost_vec)
   12491              : {
   12492       393668 :   tree lhs, rhs1, rhs2;
   12493       393668 :   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
   12494       393668 :   tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
   12495       393668 :   tree new_temp;
   12496       393668 :   enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
   12497       393668 :   poly_uint64 nunits;
   12498       393668 :   enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
   12499       393668 :   int i;
   12500       393668 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
   12501       393668 :   vec<tree> vec_oprnds0 = vNULL;
   12502       393668 :   vec<tree> vec_oprnds1 = vNULL;
   12503       393668 :   tree mask_type;
   12504       393668 :   tree mask = NULL_TREE;
   12505              : 
   12506       393668 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
   12507              :     return false;
   12508              : 
   12509       393668 :   if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
   12510              :     return false;
   12511              : 
   12512       173106 :   mask_type = vectype;
   12513       173106 :   nunits = TYPE_VECTOR_SUBPARTS (vectype);
   12514              : 
   12515       173106 :   if (TREE_CODE_CLASS (code) != tcc_comparison)
   12516              :     return false;
   12517              : 
   12518       171170 :   slp_tree slp_rhs1, slp_rhs2;
   12519       171170 :   if (!vect_is_simple_use (vinfo, slp_node,
   12520              :                            0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
   12521              :     return false;
   12522              : 
   12523       171170 :   if (!vect_is_simple_use (vinfo, slp_node,
   12524              :                            1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
   12525              :     return false;
   12526              : 
   12527       133088 :   if (vectype1 && vectype2
   12528       249405 :       && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
   12529        78235 :                    TYPE_VECTOR_SUBPARTS (vectype2)))
   12530           16 :     return false;
   12531              : 
   12532       171154 :   vectype = vectype1 ? vectype1 : vectype2;
   12533              : 
   12534              :   /* Invariant comparison.  */
   12535       171154 :   if (!vectype)
   12536              :     {
   12537        33494 :       vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), slp_node);
   12538        33494 :       if (!vectype || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
   12539              :         return false;
   12540              :     }
   12541       137660 :   else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
   12542              :     return false;
   12543              : 
   12544              :   /* Can't compare mask and non-mask types.  */
   12545       133072 :   if (vectype1 && vectype2
   12546       405175 :       && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
   12547              :     return false;
   12548              : 
   12549              :   /* We cannot compare non-mode precision _BitInt types.  Unlike bool
   12550              :      or bit-precision INTEGER_TYPE the padding bit values are target
   12551              :      dependent and possibly undefined.  */
   12552       171142 :   if (TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE
   12553       171142 :       && !type_has_mode_precision_p (TREE_TYPE (rhs1)))
   12554              :     return false;
   12555              : 
   12556              :   /* Boolean values may have another representation in vectors
   12557              :      and therefore we prefer bit operations over comparison for
   12558              :      them (which also works for scalar masks).  We store opcodes
   12559              :      to use in bitop1 and bitop2.  Statement is vectorized as
   12560              :        BITOP2 (rhs1 BITOP1 rhs2) or
   12561              :        rhs1 BITOP2 (BITOP1 rhs2)
   12562              :      depending on bitop1 and bitop2 arity.  */
   12563       171076 :   bool swap_p = false;
   12564       171076 :   if (VECTOR_BOOLEAN_TYPE_P (vectype))
   12565              :     {
   12566          765 :       if (code == GT_EXPR)
   12567              :         {
   12568              :           bitop1 = BIT_NOT_EXPR;
   12569              :           bitop2 = BIT_AND_EXPR;
   12570              :         }
   12571              :       else if (code == GE_EXPR)
   12572              :         {
   12573              :           bitop1 = BIT_NOT_EXPR;
   12574              :           bitop2 = BIT_IOR_EXPR;
   12575              :         }
   12576              :       else if (code == LT_EXPR)
   12577              :         {
   12578              :           bitop1 = BIT_NOT_EXPR;
   12579              :           bitop2 = BIT_AND_EXPR;
   12580              :           swap_p = true;
   12581              :         }
   12582              :       else if (code == LE_EXPR)
   12583              :         {
   12584              :           bitop1 = BIT_NOT_EXPR;
   12585              :           bitop2 = BIT_IOR_EXPR;
   12586              :           swap_p = true;
   12587              :         }
   12588              :       else
   12589              :         {
   12590              :           bitop1 = BIT_XOR_EXPR;
   12591              :           if (code == EQ_EXPR)
   12592              :             bitop2 = BIT_NOT_EXPR;
   12593              :         }
   12594              :     }
   12595              : 
   12596       171076 :   if (cost_vec)
   12597              :     {
   12598       158259 :       if (bitop1 == NOP_EXPR)
   12599              :         {
   12600       157643 :           if (!expand_vec_cmp_expr_p (vectype, mask_type, code))
   12601              :             return false;
   12602              :         }
   12603              :       else
   12604              :         {
   12605          616 :           machine_mode mode = TYPE_MODE (vectype);
   12606          616 :           optab optab;
   12607              : 
   12608          616 :           optab = optab_for_tree_code (bitop1, vectype, optab_default);
   12609          616 :           if (!optab || !can_implement_p (optab, mode))
   12610              :             return false;
   12611              : 
   12612          616 :           if (bitop2 != NOP_EXPR)
   12613              :             {
   12614           97 :               optab = optab_for_tree_code (bitop2, vectype, optab_default);
   12615           97 :               if (!optab || !can_implement_p (optab, mode))
   12616              :                 return false;
   12617              :             }
   12618              :         }
   12619              : 
   12620              :       /* Put types on constant and invariant SLP children.  */
   12621       150299 :       if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
   12622       150299 :           || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
   12623              :         {
   12624            2 :           if (dump_enabled_p ())
   12625            2 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12626              :                              "incompatible vector types for invariants\n");
   12627              :           return false;
   12628              :         }
   12629              : 
   12630       150297 :       vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
   12631              :                               slp_node, cost_vec);
   12632       150297 :       return true;
   12633              :     }
   12634              : 
   12635              :   /* Transform.  */
   12636              : 
   12637              :   /* Handle def.  */
   12638        12817 :   lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
   12639        12817 :   if (lhs)
   12640        12817 :     mask = vect_create_destination_var (lhs, mask_type);
   12641              : 
   12642        12817 :   vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds1);
   12643        12817 :   if (swap_p)
   12644           60 :     std::swap (vec_oprnds0, vec_oprnds1);
   12645              : 
   12646              :   /* Arguments are ready.  Create the new vector stmt.  */
   12647        32154 :   FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
   12648              :     {
   12649        19337 :       gimple *new_stmt;
   12650        19337 :       vec_rhs2 = vec_oprnds1[i];
   12651              : 
   12652        19337 :       if (lhs)
   12653        19337 :         new_temp = make_ssa_name (mask);
   12654              :       else
   12655            0 :         new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
   12656        19337 :       if (bitop1 == NOP_EXPR)
   12657              :         {
   12658        19178 :           new_stmt = gimple_build_assign (new_temp, code,
   12659              :                                           vec_rhs1, vec_rhs2);
   12660        19178 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12661              :         }
   12662              :       else
   12663              :         {
   12664          159 :           if (bitop1 == BIT_NOT_EXPR)
   12665           86 :             new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
   12666              :           else
   12667           73 :             new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
   12668              :                                             vec_rhs2);
   12669          159 :           vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12670          159 :           if (bitop2 != NOP_EXPR)
   12671              :             {
   12672           86 :               tree res = make_ssa_name (mask);
   12673           86 :               if (bitop2 == BIT_NOT_EXPR)
   12674            0 :                 new_stmt = gimple_build_assign (res, bitop2, new_temp);
   12675              :               else
   12676           86 :                 new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
   12677              :                                                 new_temp);
   12678           86 :               vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
   12679              :             }
   12680              :         }
   12681        19337 :       slp_node->push_vec_def (new_stmt);
   12682              :     }
   12683              : 
   12684        12817 :   vec_oprnds0.release ();
   12685        12817 :   vec_oprnds1.release ();
   12686              : 
   12687        12817 :   return true;
   12688              : }
   12689              : 
   12690              : /* vectorizable_comparison.
   12691              : 
   12692              :    Check if STMT_INFO is comparison expression that can be vectorized.
   12693              :    If COST_VEC is passed, calculate costs but don't change anything,
   12694              :    otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
   12695              :    it at GSI.
   12696              : 
   12697              :    Return true if STMT_INFO is vectorizable in this way.  */
   12698              : 
   12699              : static bool
   12700       716762 : vectorizable_comparison (vec_info *vinfo,
   12701              :                          stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   12702              :                          slp_tree slp_node, stmt_vector_for_cost *cost_vec)
   12703              : {
   12704       716762 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
   12705              : 
   12706       716762 :   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
   12707              :     return false;
   12708              : 
   12709       716762 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
   12710              :     return false;
   12711              : 
   12712       465191 :   gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
   12713       390907 :   if (!stmt)
   12714              :     return false;
   12715              : 
   12716       390907 :   enum tree_code code = gimple_assign_rhs_code (stmt);
   12717       390907 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
   12718       390907 :   if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
   12719              :                                   slp_node, cost_vec))
   12720              :     return false;
   12721              : 
   12722       160353 :   if (cost_vec)
   12723       147536 :     SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
   12724              : 
   12725              :   return true;
   12726              : }
   12727              : 
   12728              : /* Check to see if the target supports any of the compare and branch optabs for
   12729              :    vectors with MODE as these would be required when expanding.  */
   12730              : static bool
   12731        67203 : supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode)
   12732              : {
   12733        67203 :   bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
   12734        67203 :   bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
   12735              : 
   12736              :   /* The vectorizer only produces vec_cbranch_any_optab directly.  So only
   12737              :      check for support for that or vec_cbranch_any_optab when masked.
   12738              :      We can't produce vcond_cbranch_any directly from the vectorizer as we
   12739              :      want to keep gimple_cond as the GIMPLE representation.  But we'll fold
   12740              :      it in expand.  For that reason we require a backend to support the
   12741              :      unconditional vector cbranch optab if they support the conditional one,
   12742              :      which is just an optimization on the unconditional one.  */
   12743        67203 :   if (masked_loop_p
   12744        67203 :       && direct_optab_handler (cond_vec_cbranch_any_optab, mode)
   12745              :                 != CODE_FOR_nothing)
   12746              :     return true;
   12747        67203 :   else if (len_loop_p
   12748        67203 :            && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode)
   12749              :                 != CODE_FOR_nothing)
   12750              :     return true;
   12751        67203 :   else if (!masked_loop_p && !len_loop_p
   12752       134406 :            && direct_optab_handler (vec_cbranch_any_optab, mode)
   12753              :                 != CODE_FOR_nothing)
   12754              :     return true;
   12755              : 
   12756              :   /* The target can implement cbranch to distinguish between boolean vector
   12757              :      types and data types if they don't have a different mode for both.  */
   12758        67203 :   return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing;
   12759              : }
   12760              : 
   12761              : /* Determine the type to use for early break vectorization's scalar IV.  If
   12762              :    no type is possible return false.  */
   12763              : 
   12764              : static bool
   12765         2761 : vect_compute_type_for_early_break_scalar_iv (loop_vec_info loop_vinfo)
   12766              : {
   12767              :   /* Check if we have a usable scalar IV type for vectorization.  */
   12768         2761 :   tree iters_vf_type = sizetype;
   12769         2761 :   if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
   12770              :     {
   12771              :       /* Find the type with the minimum precision we can use
   12772              :          for the scalar IV.  */
   12773         2536 :       tree cand_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
   12774              : 
   12775              :       /* Work out how many bits we need to represent the limit.  */
   12776         2536 :       unsigned int min_ni_width
   12777         2536 :         = vect_min_prec_for_max_niters (loop_vinfo, 1);
   12778              : 
   12779              :       /* Check if we're using PFA, if so we need a signed IV and an
   12780              :          extra bit for the sign.  */
   12781         2536 :       if (TYPE_UNSIGNED (cand_type)
   12782         2536 :           && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
   12783         4111 :           && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
   12784          159 :         min_ni_width += 1;
   12785              : 
   12786         2536 :       if (TYPE_PRECISION (cand_type) >= min_ni_width)
   12787         2463 :         iters_vf_type = unsigned_type_for (cand_type);
   12788              :       else
   12789              :         {
   12790           73 :           opt_scalar_int_mode cmp_mode_iter;
   12791           73 :           tree iv_type = NULL_TREE;
   12792          357 :           FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
   12793              :             {
   12794          357 :               auto cmp_mode = cmp_mode_iter.require ();
   12795          357 :               unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode);
   12796          357 :               if (cmp_bits >= min_ni_width
   12797          357 :                   && targetm.scalar_mode_supported_p (cmp_mode))
   12798              :                 {
   12799           73 :                   iv_type = build_nonstandard_integer_type (cmp_bits, true);
   12800           73 :                   if (iv_type)
   12801              :                     break;
   12802              :                 }
   12803              :             }
   12804              : 
   12805           73 :           if (!iv_type)
   12806              :             {
   12807            0 :               if (dump_enabled_p ())
   12808            0 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12809              :                        "can't vectorize early exit because the "
   12810              :                        "target doesn't support a scalar type wide "
   12811              :                        "wide enough to hold niters.\n");
   12812            0 :               return false;
   12813              :             }
   12814           73 :           iters_vf_type = iv_type;
   12815              :         }
   12816              :     }
   12817              : 
   12818         2761 :   LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo) = iters_vf_type;
   12819         2761 :   return true;
   12820              : }
   12821              : 
   12822              : /* Check to see if the current early break given in STMT_INFO is valid for
   12823              :    vectorization.  */
   12824              : 
   12825              : bool
   12826       274776 : vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
   12827              :                          gimple_stmt_iterator *gsi,
   12828              :                          slp_tree slp_node, stmt_vector_for_cost *cost_vec)
   12829              : {
   12830       274776 :   if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
   12831              :     return false;
   12832              : 
   12833        68860 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
   12834              :     return false;
   12835              : 
   12836        68860 :   if (!STMT_VINFO_RELEVANT_P (stmt_info))
   12837              :     return false;
   12838              : 
   12839        68860 :   DUMP_VECT_SCOPE ("vectorizable_early_exit");
   12840              : 
   12841        68860 :   auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
   12842              : 
   12843              :   /* For SLP we don't want to use the type of the operands of the SLP node, when
   12844              :      vectorizing using SLP slp_node will be the children of the gcond and we
   12845              :      want to use the type of the direct children which since the gcond is root
   12846              :      will be the current node, rather than a child node as vect_is_simple_use
   12847              :      assumes.  */
   12848        68860 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
   12849        68860 :   if (!vectype)
   12850              :     return false;
   12851              : 
   12852        68860 :   machine_mode mode = TYPE_MODE (vectype);
   12853        68860 :   int vec_num = vect_get_num_copies (loop_vinfo, slp_node);
   12854              : 
   12855        68860 :   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
   12856        68860 :   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
   12857        68860 :   bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
   12858        68860 :   bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
   12859              : 
   12860              :   /* Now build the new conditional.  Pattern gimple_conds get dropped during
   12861              :      codegen so we must replace the original insn.  */
   12862        68860 :   gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info));
   12863        68860 :   gcond *cond_stmt = as_a <gcond *>(orig_stmt);
   12864              : 
   12865        68860 :   tree vectype_out = vectype;
   12866        68860 :   auto bb = gimple_bb (cond_stmt);
   12867        68860 :   edge exit_true_edge = EDGE_SUCC (bb, 0);
   12868        68860 :   if (exit_true_edge->flags & EDGE_FALSE_VALUE)
   12869          867 :     exit_true_edge = EDGE_SUCC (bb, 1);
   12870        68860 :   gcc_assert (exit_true_edge->flags & EDGE_TRUE_VALUE);
   12871              : 
   12872              :   /* When vectorizing we assume that if the branch edge is taken that we're
   12873              :      exiting the loop.  This is not however always the case as the compiler will
   12874              :      rewrite conditions to always be a comparison against 0.  To do this it
   12875              :      sometimes flips the edges.  This is fine for scalar,  but for vector we
   12876              :      then have to negate the result of the test, as we're still assuming that if
   12877              :      you take the branch edge that we found the exit condition.  i.e. we need to
   12878              :      know whether we are generating a `forall` or an `exist` condition.  */
   12879       137720 :   bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
   12880        68860 :                                         exit_true_edge->dest);
   12881              : 
   12882              :   /* See if we support ADDHN and use that for the reduction.  */
   12883        68860 :   internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH;
   12884        68860 :   bool addhn_supported_p
   12885        68860 :     = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH);
   12886        68860 :   tree narrow_type = NULL_TREE;
   12887        68860 :   if (addhn_supported_p)
   12888              :     {
   12889              :       /* Calculate the narrowing type for the result.  */
   12890            0 :       auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2;
   12891            0 :       auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype));
   12892            0 :       tree itype = build_nonstandard_integer_type (halfprec, unsignedp);
   12893            0 :       tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype));
   12894            0 :       narrow_type = truth_type_for (tmp_type);
   12895              : 
   12896            0 :       if (!supports_vector_compare_and_branch (loop_vinfo,
   12897            0 :                                                TYPE_MODE (narrow_type)))
   12898              :         {
   12899            0 :           if (dump_enabled_p ())
   12900            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12901              :                                "can't use ADDHN reduction because cbranch for "
   12902              :                                "the narrowed type is not supported by the "
   12903              :                                "target.\n");
   12904              :           addhn_supported_p = false;
   12905              :         }
   12906              :     }
   12907              : 
   12908              :   /* Analyze only.  */
   12909        68860 :   if (cost_vec)
   12910              :     {
   12911        67203 :       if (!addhn_supported_p
   12912        67203 :           && !supports_vector_compare_and_branch (loop_vinfo, mode))
   12913              :         {
   12914        64442 :           if (dump_enabled_p ())
   12915          793 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   12916              :                                "can't vectorize early exit because the "
   12917              :                                "target doesn't support flag setting vector "
   12918              :                                "comparisons.\n");
   12919              :           return false;
   12920              :         }
   12921              : 
   12922         2761 :       if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi,
   12923              :                                       slp_node, cost_vec))
   12924              :         return false;
   12925              : 
   12926         2761 :       if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
   12927              :         {
   12928         1575 :           if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
   12929              :                                               OPTIMIZE_FOR_SPEED))
   12930            0 :             vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
   12931              :           else
   12932         1575 :             vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
   12933              :         }
   12934              : 
   12935         2761 :       if (!vect_compute_type_for_early_break_scalar_iv (loop_vinfo))
   12936              :         return false;
   12937              : 
   12938              :       return true;
   12939              :     }
   12940              : 
   12941              :   /* Transform.  */
   12942              : 
   12943         1657 :   tree new_temp = NULL_TREE;
   12944         1657 :   gimple *new_stmt = NULL;
   12945              : 
   12946         1657 :   if (dump_enabled_p ())
   12947          463 :     dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
   12948              : 
   12949              :   /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
   12950              :      roots and so by the time we get to them we have already codegened the SLP tree
   12951              :      and so we shouldn't try to do so again.  The arguments have already been
   12952              :      vectorized.  It's not very clean to do this here, But the masking code below is
   12953              :      complex and this keeps it all in one place to ease fixes and backports.  Once we
   12954              :      drop the non-SLP loop vect or split vectorizable_* this can be simplified.  */
   12955              : 
   12956         1657 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
   12957         1657 :   basic_block cond_bb = gimple_bb (stmt);
   12958         1657 :   gimple_stmt_iterator  cond_gsi = gsi_last_bb (cond_bb);
   12959              : 
   12960         1657 :   auto_vec<tree> stmts;
   12961         1657 :   stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
   12962              : 
   12963              :   /* If we're comparing against a previous forall we need to negate the results
   12964              :      before we do the final comparison or reduction.  */
   12965         1657 :   if (flipped)
   12966              :     {
   12967              :       /* Rewrite the if(all(mask)) into if (!all(mask)) which is the same as
   12968              :          if (any(~mask)) by negating the masks and flipping the branches.
   12969              : 
   12970              :         1. For unmasked loops we simply reduce the ~mask.
   12971              :         2. For masked loops we reduce (~mask & loop_mask) which is the same as
   12972              :            doing (mask & loop_mask) ^ loop_mask.  */
   12973          329 :       for (unsigned i = 0; i < stmts.length (); i++)
   12974              :         {
   12975          199 :           tree inv_lhs = make_temp_ssa_name (vectype, NULL, "vexit_inv");
   12976          199 :           auto inv_stmt = gimple_build_assign (inv_lhs, BIT_NOT_EXPR, stmts[i]);
   12977          199 :           vect_finish_stmt_generation (loop_vinfo, stmt_info, inv_stmt,
   12978              :                                        &cond_gsi);
   12979          199 :           stmts[i] = inv_lhs;
   12980              :         }
   12981              : 
   12982          130 :       EDGE_SUCC (bb, 0)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
   12983          130 :       EDGE_SUCC (bb, 1)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
   12984              :     }
   12985              : 
   12986              :   /* Determine if we need to reduce the final value.  */
   12987         1657 :   if (stmts.length () > 1)
   12988              :     {
   12989              :       /* We build the reductions in a way to maintain as much parallelism as
   12990              :          possible.  */
   12991          151 :       auto_vec<tree> workset (stmts.length ());
   12992              : 
   12993              :       /* Mask the statements as we queue them up.  Normally we loop over
   12994              :          vec_num,  but since we inspect the exact results of vectorization
   12995              :          we don't need to and instead can just use the stmts themselves.  */
   12996          151 :       if (masked_loop_p)
   12997            0 :         for (unsigned i = 0; i < stmts.length (); i++)
   12998              :           {
   12999            0 :             tree stmt_mask
   13000            0 :               = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
   13001              :                                     vectype, i);
   13002            0 :             stmt_mask
   13003            0 :               = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
   13004            0 :                                   stmts[i], &cond_gsi);
   13005            0 :             workset.quick_push (stmt_mask);
   13006              :           }
   13007          151 :       else if (len_loop_p)
   13008          151 :         for (unsigned i = 0; i < stmts.length (); i++)
   13009              :           {
   13010            0 :             tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
   13011              :                                                     lens, vec_num,
   13012            0 :                                                     vectype, stmts[i], i, 1);
   13013              : 
   13014            0 :             workset.quick_push (len_mask);
   13015              :           }
   13016              :       else
   13017          151 :         workset.splice (stmts);
   13018              : 
   13019          454 :       while (workset.length () > 1)
   13020              :         {
   13021          303 :           tree arg0 = workset.pop ();
   13022          303 :           tree arg1 = workset.pop ();
   13023          303 :           if (addhn_supported_p && workset.length () == 0)
   13024              :             {
   13025            0 :               new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1);
   13026            0 :               vectype_out = narrow_type;
   13027            0 :               new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
   13028            0 :               gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp);
   13029            0 :               gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true);
   13030              :             }
   13031              :           else
   13032              :             {
   13033          303 :               new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
   13034          303 :               new_stmt
   13035          303 :                 = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
   13036              :             }
   13037          303 :           vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
   13038              :                                        &cond_gsi);
   13039          303 :           workset.quick_insert (0, new_temp);
   13040              :         }
   13041          151 :     }
   13042              :   else
   13043              :     {
   13044         1506 :       new_temp = stmts[0];
   13045         1506 :       if (masked_loop_p)
   13046              :         {
   13047            2 :           tree mask
   13048            2 :             = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
   13049            2 :           new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
   13050              :                                        new_temp, &cond_gsi);
   13051              :         }
   13052         1504 :       else if (len_loop_p)
   13053            0 :         new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
   13054              :                                            1, vectype, new_temp, 0, 1);
   13055              :     }
   13056              : 
   13057         1657 :   gcc_assert (new_temp);
   13058              : 
   13059         1657 :   tree cst = build_zero_cst (vectype_out);
   13060         1657 :   gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
   13061         1657 :   update_stmt (orig_stmt);
   13062              : 
   13063              :   /* ??? */
   13064         1657 :   SLP_TREE_VEC_DEFS (slp_node).truncate (0);
   13065              : 
   13066         1657 :   return true;
   13067         1657 : }
   13068              : 
   13069              : /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
   13070              :    can handle all live statements in the node.  Otherwise return true
   13071              :    if STMT_INFO is not live or if vectorizable_live_operation can handle it.
   13072              :    VEC_STMT_P is as for vectorizable_live_operation.  */
   13073              : 
   13074              : static bool
   13075      1339089 : can_vectorize_live_stmts (vec_info *vinfo,
   13076              :                           slp_tree slp_node, slp_instance slp_node_instance,
   13077              :                           bool vec_stmt_p,
   13078              :                           stmt_vector_for_cost *cost_vec)
   13079              : {
   13080      1339089 :   stmt_vec_info slp_stmt_info;
   13081      1339089 :   unsigned int i;
   13082      2815002 :   FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
   13083              :     {
   13084      1475913 :       if (slp_stmt_info
   13085      1459355 :           && STMT_VINFO_LIVE_P (slp_stmt_info)
   13086      1614410 :           && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
   13087              :                                            slp_node_instance, i,
   13088              :                                            vec_stmt_p, cost_vec))
   13089              :         return false;
   13090              :     }
   13091              : 
   13092              :   return true;
   13093              : }
   13094              : 
   13095              : /* Make sure the statement is vectorizable.  */
   13096              : 
   13097              : opt_result
   13098      2673960 : vect_analyze_stmt (vec_info *vinfo,
   13099              :                    slp_tree node, slp_instance node_instance,
   13100              :                    stmt_vector_for_cost *cost_vec)
   13101              : {
   13102      2673960 :   stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
   13103      2673960 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
   13104      2673960 :   enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
   13105      2673960 :   bool ok;
   13106              : 
   13107      2673960 :   if (dump_enabled_p ())
   13108       107900 :     dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
   13109              :                      stmt_info->stmt);
   13110              : 
   13111      5042820 :   if (gimple_has_volatile_ops (stmt_info->stmt))
   13112              :     {
   13113              :       /* ???  This shouldn't really happen, volatile stmts should
   13114              :          not end up in the SLP graph.  */
   13115            0 :       return opt_result::failure_at (stmt_info->stmt,
   13116              :                                      "not vectorized:"
   13117              :                                      " stmt has volatile operands: %G\n",
   13118              :                                      stmt_info->stmt);
   13119              :     }
   13120              : 
   13121              :   /* Skip stmts that do not need to be vectorized.  */
   13122      2673960 :   if (!STMT_VINFO_RELEVANT_P (stmt_info)
   13123            0 :       && !STMT_VINFO_LIVE_P (stmt_info))
   13124              :     {
   13125            0 :       if (dump_enabled_p ())
   13126            0 :         dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
   13127              : 
   13128              :       /* ???  This shouldn't really happen, irrelevant stmts should
   13129              :          not end up in the SLP graph.  */
   13130            0 :       return opt_result::failure_at (stmt_info->stmt,
   13131              :                                      "not vectorized:"
   13132              :                                      " irrelevant stmt as SLP node %p "
   13133              :                                      "representative.\n",
   13134              :                                      (void *)node);
   13135              :     }
   13136              : 
   13137      2673960 :   switch (STMT_VINFO_DEF_TYPE (stmt_info))
   13138              :     {
   13139              :       case vect_internal_def:
   13140              :       case vect_condition_def:
   13141              :         break;
   13142              : 
   13143        82404 :       case vect_reduction_def:
   13144        82404 :       case vect_nested_cycle:
   13145        82404 :          gcc_assert (!bb_vinfo
   13146              :                      && (relevance == vect_used_in_outer
   13147              :                          || relevance == vect_used_in_outer_by_reduction
   13148              :                          || relevance == vect_used_by_reduction
   13149              :                          || relevance == vect_unused_in_scope
   13150              :                          || relevance == vect_used_only_live));
   13151              :          break;
   13152              : 
   13153          314 :       case vect_double_reduction_def:
   13154          314 :         gcc_assert (!bb_vinfo && node);
   13155              :         break;
   13156              : 
   13157       168847 :       case vect_induction_def:
   13158       168847 :       case vect_first_order_recurrence:
   13159       168847 :         gcc_assert (!bb_vinfo);
   13160              :         break;
   13161              : 
   13162            0 :       case vect_constant_def:
   13163            0 :       case vect_external_def:
   13164            0 :       case vect_unknown_def_type:
   13165            0 :       default:
   13166            0 :         gcc_unreachable ();
   13167              :     }
   13168              : 
   13169      2673960 :   tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
   13170      2673960 :   STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
   13171              : 
   13172      2673960 :   if (STMT_VINFO_RELEVANT_P (stmt_info))
   13173              :     {
   13174      2673960 :       gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
   13175      2673960 :       gcc_assert (SLP_TREE_VECTYPE (node)
   13176              :                   || gimple_code (stmt_info->stmt) == GIMPLE_COND
   13177              :                   || (call && gimple_call_lhs (call) == NULL_TREE));
   13178              :     }
   13179              : 
   13180      2673960 :   ok = true;
   13181      2673960 :   if (bb_vinfo
   13182      1545005 :       || (STMT_VINFO_RELEVANT_P (stmt_info)
   13183            0 :           || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
   13184              :     /* Prefer vectorizable_call over vectorizable_simd_clone_call so
   13185              :        -mveclibabi= takes preference over library functions with
   13186              :        the simd attribute.  */
   13187      2673960 :     ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
   13188      2665858 :           || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
   13189              :                                            cost_vec)
   13190      2665357 :           || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec)
   13191      2577806 :           || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec)
   13192      2154398 :           || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec)
   13193      2082483 :           || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
   13194      1617121 :           || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
   13195       780427 :           || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
   13196       731130 :           || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
   13197       703945 :           || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
   13198       556409 :           || (bb_vinfo
   13199       141182 :               && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
   13200      3169308 :           || (is_a <loop_vec_info> (vinfo)
   13201       415227 :               && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
   13202              :                                               stmt_info, node, cost_vec)
   13203       414487 :                   || vectorizable_reduction (as_a <loop_vec_info> (vinfo),
   13204              :                                              stmt_info,
   13205              :                                              node, node_instance, cost_vec)
   13206       334721 :                   || vectorizable_induction (as_a <loop_vec_info> (vinfo),
   13207              :                                              stmt_info, node, cost_vec)
   13208       206990 :                   || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
   13209              :                                           stmt_info, node)
   13210       206185 :                   || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
   13211              :                                           stmt_info, node, cost_vec)
   13212       205916 :                   || vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
   13213              :                                               stmt_info, NULL, node,
   13214              :                                               cost_vec))));
   13215              : 
   13216      2673960 :   STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
   13217              : 
   13218      2387923 :   if (!ok)
   13219       286037 :     return opt_result::failure_at (stmt_info->stmt,
   13220              :                                    "not vectorized:"
   13221              :                                    " relevant stmt not supported: %G",
   13222              :                                    stmt_info->stmt);
   13223              : 
   13224              :   /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
   13225              :       need extra handling, except for vectorizable reductions.  */
   13226      2387923 :   if (!bb_vinfo
   13227      1339089 :       && (SLP_TREE_TYPE (node) != lc_phi_info_type
   13228          805 :           || SLP_TREE_DEF_TYPE (node) == vect_internal_def)
   13229      1339089 :       && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node))
   13230      3727012 :       && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
   13231              :                                     node, node_instance,
   13232              :                                     false, cost_vec))
   13233            0 :     return opt_result::failure_at (stmt_info->stmt,
   13234              :                                    "not vectorized:"
   13235              :                                    " live stmt not supported: %G",
   13236              :                                    stmt_info->stmt);
   13237              : 
   13238      2387923 :   return opt_result::success ();
   13239              : }
   13240              : 
   13241              : 
   13242              : /* Function vect_transform_stmt.
   13243              : 
   13244              :    Create a vectorized stmt to replace STMT_INFO, and insert it at GSI.  */
   13245              : 
   13246              : void
   13247       976994 : vect_transform_stmt (vec_info *vinfo,
   13248              :                      stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   13249              :                      slp_tree slp_node, slp_instance slp_node_instance)
   13250              : {
   13251       976994 :   bool done;
   13252              : 
   13253       976994 :   gcc_assert (slp_node);
   13254              : 
   13255       976994 :   if (stmt_info)
   13256       960086 :     STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
   13257              : 
   13258       976994 :   switch (SLP_TREE_TYPE (slp_node))
   13259              :     {
   13260        23709 :     case type_demotion_vec_info_type:
   13261        23709 :     case type_promotion_vec_info_type:
   13262        23709 :     case type_conversion_vec_info_type:
   13263        23709 :       done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
   13264        23709 :       gcc_assert (done);
   13265              :       break;
   13266              : 
   13267        17011 :     case induc_vec_info_type:
   13268        17011 :       done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
   13269              :                                      stmt_info, slp_node, NULL);
   13270        17011 :       gcc_assert (done);
   13271              :       break;
   13272              : 
   13273         9232 :     case shift_vec_info_type:
   13274         9232 :       done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
   13275         9232 :       gcc_assert (done);
   13276              :       break;
   13277              : 
   13278       119230 :     case op_vec_info_type:
   13279       119230 :       done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
   13280       119230 :       gcc_assert (done);
   13281              :       break;
   13282              : 
   13283        16563 :     case assignment_vec_info_type:
   13284        16563 :       done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
   13285        16563 :       gcc_assert (done);
   13286              :       break;
   13287              : 
   13288       157666 :     case load_vec_info_type:
   13289       157666 :       done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
   13290       157666 :       gcc_assert (done);
   13291              :       break;
   13292              : 
   13293       550512 :     case store_vec_info_type:
   13294       550512 :       done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
   13295       550512 :       gcc_assert (done);
   13296              :       break;
   13297              : 
   13298         8739 :     case condition_vec_info_type:
   13299         8739 :       done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
   13300         8739 :       gcc_assert (done);
   13301              :       break;
   13302              : 
   13303        12817 :     case comparison_vec_info_type:
   13304        12817 :       done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
   13305        12817 :       gcc_assert (done);
   13306              :       break;
   13307              : 
   13308         4611 :     case call_vec_info_type:
   13309         4611 :       done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
   13310         4611 :       break;
   13311              : 
   13312          382 :     case call_simd_clone_vec_info_type:
   13313          382 :       done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
   13314              :                                            slp_node, NULL);
   13315          382 :       break;
   13316              : 
   13317         2692 :     case reduc_vec_info_type:
   13318         2692 :       done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
   13319              :                                        gsi, slp_node);
   13320         2692 :       gcc_assert (done);
   13321              :       break;
   13322              : 
   13323        23672 :     case cycle_phi_info_type:
   13324        23672 :       done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
   13325              :                                        slp_node, slp_node_instance);
   13326        23672 :       gcc_assert (done);
   13327              :       break;
   13328              : 
   13329          489 :     case lc_phi_info_type:
   13330          489 :       done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo),
   13331              :                                     stmt_info, slp_node);
   13332          489 :       gcc_assert (done);
   13333              :       break;
   13334              : 
   13335           43 :     case recurr_info_type:
   13336           43 :       done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
   13337              :                                   stmt_info, slp_node, NULL);
   13338           43 :       gcc_assert (done);
   13339              :       break;
   13340              : 
   13341        12718 :     case phi_info_type:
   13342        12718 :       done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
   13343              :                                stmt_info, slp_node, NULL);
   13344        12718 :       gcc_assert (done);
   13345              :       break;
   13346              : 
   13347            0 :     case loop_exit_ctrl_vec_info_type:
   13348            0 :       done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
   13349              :                                       stmt_info, gsi, slp_node, NULL);
   13350            0 :       gcc_assert (done);
   13351              :       break;
   13352              : 
   13353        16908 :     case permute_info_type:
   13354        16908 :       done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL);
   13355        16908 :       gcc_assert (done);
   13356              :       break;
   13357              : 
   13358            0 :     default:
   13359            0 :       if (!STMT_VINFO_LIVE_P (stmt_info))
   13360              :         {
   13361            0 :           if (dump_enabled_p ())
   13362            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   13363              :                              "stmt not supported.\n");
   13364            0 :           gcc_unreachable ();
   13365              :         }
   13366       976994 :       done = true;
   13367              :     }
   13368              : 
   13369       976994 :   if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
   13370       426482 :       && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node)))
   13371              :     {
   13372              :       /* Handle stmts whose DEF is used outside the loop-nest that is
   13373              :          being vectorized.  */
   13374       536038 :       for (unsigned lane : SLP_TREE_LIVE_LANES (slp_node))
   13375              :         {
   13376        40382 :           stmt_vec_info slp_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[lane];
   13377        40382 :           done = vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
   13378              :                                               slp_node_instance, lane,
   13379              :                                               true, NULL);
   13380        40382 :           gcc_assert (done);
   13381              :         }
   13382              :     }
   13383       976994 : }
   13384              : 
   13385              : 
   13386              : /* Remove a group of stores (for SLP or interleaving), free their
   13387              :    stmt_vec_info.  */
   13388              : 
   13389              : void
   13390            0 : vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
   13391              : {
   13392            0 :   stmt_vec_info next_stmt_info = first_stmt_info;
   13393              : 
   13394            0 :   while (next_stmt_info)
   13395              :     {
   13396            0 :       stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
   13397            0 :       next_stmt_info = vect_orig_stmt (next_stmt_info);
   13398              :       /* Free the attached stmt_vec_info and remove the stmt.  */
   13399            0 :       vinfo->remove_stmt (next_stmt_info);
   13400            0 :       next_stmt_info = tmp;
   13401              :     }
   13402            0 : }
   13403              : 
   13404              : /* If NUNITS is nonzero, return a vector type that contains NUNITS
   13405              :    elements of type SCALAR_TYPE, or null if the target doesn't support
   13406              :    such a type.
   13407              : 
   13408              :    If NUNITS is zero, return a vector type that contains elements of
   13409              :    type SCALAR_TYPE, choosing whichever vector size the target prefers.
   13410              : 
   13411              :    If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
   13412              :    for this vectorization region and want to "autodetect" the best choice.
   13413              :    Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
   13414              :    and we want the new type to be interoperable with it.   PREVAILING_MODE
   13415              :    in this case can be a scalar integer mode or a vector mode; when it
   13416              :    is a vector mode, the function acts like a tree-level version of
   13417              :    related_vector_mode.  */
   13418              : 
   13419              : tree
   13420     30945187 : get_related_vectype_for_scalar_type (machine_mode prevailing_mode,
   13421              :                                      tree scalar_type, poly_uint64 nunits)
   13422              : {
   13423     30945187 :   tree orig_scalar_type = scalar_type;
   13424     30945187 :   scalar_mode inner_mode;
   13425     30945187 :   machine_mode simd_mode;
   13426     30945187 :   tree vectype;
   13427              : 
   13428     30945187 :   if ((!INTEGRAL_TYPE_P (scalar_type)
   13429     10177717 :        && !POINTER_TYPE_P (scalar_type)
   13430      2162166 :        && !SCALAR_FLOAT_TYPE_P (scalar_type))
   13431     40566816 :       || (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
   13432      1606231 :           && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode)))
   13433              :     return NULL_TREE;
   13434              : 
   13435     30385722 :   unsigned int nbytes = GET_MODE_SIZE (inner_mode);
   13436              : 
   13437              :   /* Interoperability between modes requires one to be a constant multiple
   13438              :      of the other, so that the number of vectors required for each operation
   13439              :      is a compile-time constant.  */
   13440     30385722 :   if (prevailing_mode != VOIDmode
   13441     29215109 :       && !constant_multiple_p (nunits * nbytes,
   13442     29215109 :                                GET_MODE_SIZE (prevailing_mode))
   13443     31630753 :       && !constant_multiple_p (GET_MODE_SIZE (prevailing_mode),
   13444      1245031 :                                nunits * nbytes))
   13445              :     return NULL_TREE;
   13446              : 
   13447              :   /* For vector types of elements whose mode precision doesn't
   13448              :      match their types precision we use a element type of mode
   13449              :      precision.  The vectorization routines will have to make sure
   13450              :      they support the proper result truncation/extension.
   13451              :      We also make sure to build vector types with INTEGER_TYPE
   13452              :      component type only.  */
   13453     30385722 :   if (INTEGRAL_TYPE_P (scalar_type)
   13454     51153039 :       && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
   13455     19546089 :           || TREE_CODE (scalar_type) != INTEGER_TYPE))
   13456      1437181 :     scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
   13457      1437181 :                                                   TYPE_UNSIGNED (scalar_type));
   13458              : 
   13459              :   /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
   13460              :      When the component mode passes the above test simply use a type
   13461              :      corresponding to that mode.  The theory is that any use that
   13462              :      would cause problems with this will disable vectorization anyway.  */
   13463     28948541 :   else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
   13464              :            && !INTEGRAL_TYPE_P (scalar_type))
   13465      8015551 :     scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
   13466              : 
   13467              :   /* We can't build a vector type of elements with alignment bigger than
   13468              :      their size.  */
   13469     20932990 :   else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
   13470       421268 :     scalar_type = lang_hooks.types.type_for_mode (inner_mode,
   13471       210634 :                                                   TYPE_UNSIGNED (scalar_type));
   13472              : 
   13473              :   /* If we felt back to using the mode fail if there was
   13474              :      no scalar type for it.  */
   13475     30385722 :   if (scalar_type == NULL_TREE)
   13476              :     return NULL_TREE;
   13477              : 
   13478              :   /* If no prevailing mode was supplied, use the mode the target prefers.
   13479              :      Otherwise lookup a vector mode based on the prevailing mode.  */
   13480     30385722 :   if (prevailing_mode == VOIDmode)
   13481              :     {
   13482      1170613 :       gcc_assert (known_eq (nunits, 0U));
   13483      1170613 :       simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
   13484      1170613 :       if (SCALAR_INT_MODE_P (simd_mode))
   13485              :         {
   13486              :           /* Traditional behavior is not to take the integer mode
   13487              :              literally, but simply to use it as a way of determining
   13488              :              the vector size.  It is up to mode_for_vector to decide
   13489              :              what the TYPE_MODE should be.
   13490              : 
   13491              :              Note that nunits == 1 is allowed in order to support single
   13492              :              element vector types.  */
   13493        66490 :           if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits)
   13494        33245 :               || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
   13495              :             return NULL_TREE;
   13496              :         }
   13497              :     }
   13498     29215109 :   else if (SCALAR_INT_MODE_P (prevailing_mode)
   13499     29215109 :            || !related_vector_mode (prevailing_mode,
   13500     29214452 :                                     inner_mode, nunits).exists (&simd_mode))
   13501              :     {
   13502              :       /* Fall back to using mode_for_vector, mostly in the hope of being
   13503              :          able to use an integer mode.  */
   13504      2386307 :       if (known_eq (nunits, 0U)
   13505      4767004 :           && !multiple_p (GET_MODE_SIZE (prevailing_mode), nbytes, &nunits))
   13506              :         return NULL_TREE;
   13507              : 
   13508       242143 :       if (!mode_for_vector (inner_mode, nunits).exists (&simd_mode))
   13509       226753 :         return NULL_TREE;
   13510              :     }
   13511              : 
   13512     27982261 :   vectype = build_vector_type_for_mode (scalar_type, simd_mode);
   13513              : 
   13514              :   /* In cases where the mode was chosen by mode_for_vector, check that
   13515              :      the target actually supports the chosen mode, or that it at least
   13516              :      allows the vector mode to be replaced by a like-sized integer.  */
   13517     55964522 :   if (!VECTOR_MODE_P (TYPE_MODE (vectype))
   13518     27997960 :       && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
   13519              :     return NULL_TREE;
   13520              : 
   13521              :   /* Re-attach the address-space qualifier if we canonicalized the scalar
   13522              :      type.  */
   13523     27974048 :   if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
   13524            3 :     return build_qualified_type
   13525            3 :       (vectype, qualifier_set {TYPE_UNQUALIFIED,
   13526            3 :                                TYPE_ADDR_SPACE (orig_scalar_type)});
   13527              : 
   13528              :   return vectype;
   13529              : }
   13530              : 
   13531              : /* Function get_vectype_for_scalar_type.
   13532              : 
   13533              :    Returns the vector type corresponding to SCALAR_TYPE as supported
   13534              :    by the target.  If GROUP_SIZE is nonzero and we're performing BB
   13535              :    vectorization, make sure that the number of elements in the vector
   13536              :    is no bigger than GROUP_SIZE.  */
   13537              : 
   13538              : tree
   13539     26987820 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type,
   13540              :                              unsigned int group_size)
   13541              : {
   13542              :   /* For BB vectorization, we should always have a group size once we've
   13543              :      constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
   13544              :      are tentative requests during things like early data reference
   13545              :      analysis and pattern recognition.  */
   13546     26987820 :   if (is_a <bb_vec_info> (vinfo))
   13547     23804536 :     gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
   13548              :   else
   13549              :     group_size = 0;
   13550              : 
   13551     26987820 :   tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
   13552              :                                                       scalar_type);
   13553     26987820 :   if (vectype && vinfo->vector_mode == VOIDmode)
   13554      1085863 :     vinfo->vector_mode = TYPE_MODE (vectype);
   13555              : 
   13556              :   /* Register the natural choice of vector type, before the group size
   13557              :      has been applied.  */
   13558            0 :   if (vectype)
   13559     24191415 :     vinfo->used_vector_modes.add (TYPE_MODE (vectype));
   13560              : 
   13561              :   /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
   13562              :      try again with an explicit number of elements.  */
   13563     24191415 :   if (vectype
   13564     24191415 :       && group_size
   13565     26987820 :       && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
   13566              :     {
   13567              :       /* Start with the biggest number of units that fits within
   13568              :          GROUP_SIZE and halve it until we find a valid vector type.
   13569              :          Usually either the first attempt will succeed or all will
   13570              :          fail (in the latter case because GROUP_SIZE is too small
   13571              :          for the target), but it's possible that a target could have
   13572              :          a hole between supported vector types.
   13573              : 
   13574              :          If GROUP_SIZE is not a power of 2, this has the effect of
   13575              :          trying the largest power of 2 that fits within the group,
   13576              :          even though the group is not a multiple of that vector size.
   13577              :          The BB vectorizer will then try to carve up the group into
   13578              :          smaller pieces.  */
   13579      2590231 :       unsigned int nunits = 1 << floor_log2 (group_size);
   13580      2590231 :       do
   13581              :         {
   13582      2590231 :           vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
   13583      2590231 :                                                          scalar_type, nunits);
   13584      2590231 :           nunits /= 2;
   13585              :         }
   13586      2590231 :       while (nunits > 1 && !vectype);
   13587              :     }
   13588              : 
   13589     26987820 :   return vectype;
   13590              : }
   13591              : 
   13592              : /* Return the vector type corresponding to SCALAR_TYPE as supported
   13593              :    by the target.  NODE, if nonnull, is the SLP tree node that will
   13594              :    use the returned vector type.  */
   13595              : 
   13596              : tree
   13597       184274 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node)
   13598              : {
   13599       184274 :   unsigned int group_size = 0;
   13600       184274 :   if (node)
   13601       184274 :     group_size = SLP_TREE_LANES (node);
   13602       184274 :   return get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
   13603              : }
   13604              : 
   13605              : /* Function get_mask_type_for_scalar_type.
   13606              : 
   13607              :    Returns the mask type corresponding to a result of comparison
   13608              :    of vectors of specified SCALAR_TYPE as supported by target.
   13609              :    If GROUP_SIZE is nonzero and we're performing BB vectorization,
   13610              :    make sure that the number of elements in the vector is no bigger
   13611              :    than GROUP_SIZE.  */
   13612              : 
   13613              : tree
   13614      1158751 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
   13615              :                                unsigned int group_size)
   13616              : {
   13617      1158751 :   tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
   13618              : 
   13619      1158751 :   if (!vectype)
   13620              :     return NULL;
   13621              : 
   13622      1136932 :   return truth_type_for (vectype);
   13623              : }
   13624              : 
   13625              : /* Function get_mask_type_for_scalar_type.
   13626              : 
   13627              :    Returns the mask type corresponding to a result of comparison
   13628              :    of vectors of specified SCALAR_TYPE as supported by target.
   13629              :    NODE, if nonnull, is the SLP tree node that will use the returned
   13630              :    vector type.  */
   13631              : 
   13632              : tree
   13633           19 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
   13634              :                                slp_tree node)
   13635              : {
   13636           19 :   tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
   13637              : 
   13638           19 :   if (!vectype)
   13639              :     return NULL;
   13640              : 
   13641           19 :   return truth_type_for (vectype);
   13642              : }
   13643              : 
   13644              : /* Function get_same_sized_vectype
   13645              : 
   13646              :    Returns a vector type corresponding to SCALAR_TYPE of size
   13647              :    VECTOR_TYPE if supported by the target.  */
   13648              : 
   13649              : tree
   13650         1110 : get_same_sized_vectype (tree scalar_type, tree vector_type)
   13651              : {
   13652         1110 :   if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
   13653            0 :     return truth_type_for (vector_type);
   13654              : 
   13655         1110 :   poly_uint64 nunits;
   13656         2220 :   if (!multiple_p (GET_MODE_SIZE (TYPE_MODE (vector_type)),
   13657         2220 :                    GET_MODE_SIZE (TYPE_MODE (scalar_type)), &nunits))
   13658              :     return NULL_TREE;
   13659              : 
   13660         1110 :   return get_related_vectype_for_scalar_type (TYPE_MODE (vector_type),
   13661         1110 :                                               scalar_type, nunits);
   13662              : }
   13663              : 
   13664              : /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
   13665              :    would not change the chosen vector modes.  */
   13666              : 
   13667              : bool
   13668      1495663 : vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode)
   13669              : {
   13670      1495663 :   for (vec_info::mode_set::iterator i = vinfo->used_vector_modes.begin ();
   13671      2580205 :        i != vinfo->used_vector_modes.end (); ++i)
   13672      1985655 :     if (!VECTOR_MODE_P (*i)
   13673      5956695 :         || related_vector_mode (vector_mode, GET_MODE_INNER (*i), 0) != *i)
   13674       901113 :       return false;
   13675       594550 :   return true;
   13676              : }
   13677              : 
   13678              : /* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not
   13679              :    change the chosen vector modes for analysis of a loop.  */
   13680              : 
   13681              : bool
   13682       404081 : vect_chooses_same_modes_p (machine_mode vector_mode,
   13683              :                            machine_mode alt_vector_mode)
   13684              : {
   13685        66091 :   return (VECTOR_MODE_P (vector_mode)
   13686       404081 :           && VECTOR_MODE_P (alt_vector_mode)
   13687       808162 :           && (related_vector_mode (vector_mode,
   13688              :                                    GET_MODE_INNER (alt_vector_mode))
   13689       404081 :               == alt_vector_mode)
   13690       431543 :           && (related_vector_mode (alt_vector_mode,
   13691              :                                    GET_MODE_INNER (vector_mode))
   13692        13731 :               == vector_mode));
   13693              : }
   13694              : 
   13695              : /* Function vect_is_simple_use.
   13696              : 
   13697              :    Input:
   13698              :    VINFO - the vect info of the loop or basic block that is being vectorized.
   13699              :    OPERAND - operand in the loop or bb.
   13700              :    Output:
   13701              :    DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
   13702              :      case OPERAND is an SSA_NAME that is defined in the vectorizable region
   13703              :    DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
   13704              :      the definition could be anywhere in the function
   13705              :    DT - the type of definition
   13706              : 
   13707              :    Returns whether a stmt with OPERAND can be vectorized.
   13708              :    For loops, supportable operands are constants, loop invariants, and operands
   13709              :    that are defined by the current iteration of the loop.  Unsupportable
   13710              :    operands are those that are defined by a previous iteration of the loop (as
   13711              :    is the case in reduction/induction computations).
   13712              :    For basic blocks, supportable operands are constants and bb invariants.
   13713              :    For now, operands defined outside the basic block are not supported.  */
   13714              : 
   13715              : bool
   13716     44459959 : vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
   13717              :                     stmt_vec_info *def_stmt_info_out, gimple **def_stmt_out)
   13718              : {
   13719     44459959 :   if (def_stmt_info_out)
   13720     42410225 :     *def_stmt_info_out = NULL;
   13721     44459959 :   if (def_stmt_out)
   13722     10353972 :     *def_stmt_out = NULL;
   13723     44459959 :   *dt = vect_unknown_def_type;
   13724              : 
   13725     44459959 :   if (dump_enabled_p ())
   13726              :     {
   13727       818656 :       dump_printf_loc (MSG_NOTE, vect_location,
   13728              :                        "vect_is_simple_use: operand ");
   13729       818656 :       if (TREE_CODE (operand) == SSA_NAME
   13730       818656 :           && !SSA_NAME_IS_DEFAULT_DEF (operand))
   13731       748091 :         dump_gimple_expr (MSG_NOTE, TDF_SLIM, SSA_NAME_DEF_STMT (operand), 0);
   13732              :       else
   13733        70565 :         dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
   13734              :     }
   13735              : 
   13736     44459959 :   if (CONSTANT_CLASS_P (operand))
   13737      3402669 :     *dt = vect_constant_def;
   13738     41057290 :   else if (is_gimple_min_invariant (operand))
   13739       400074 :     *dt = vect_external_def;
   13740     40657216 :   else if (TREE_CODE (operand) != SSA_NAME)
   13741         1064 :     *dt = vect_unknown_def_type;
   13742     40656152 :   else if (SSA_NAME_IS_DEFAULT_DEF (operand))
   13743       727021 :     *dt = vect_external_def;
   13744              :   else
   13745              :     {
   13746     39929131 :       gimple *def_stmt = SSA_NAME_DEF_STMT (operand);
   13747     39929131 :       stmt_vec_info stmt_vinfo = vinfo->lookup_def (operand);
   13748     39929131 :       if (!stmt_vinfo)
   13749       880101 :         *dt = vect_external_def;
   13750              :       else
   13751              :         {
   13752     39049030 :           stmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
   13753     39049030 :           def_stmt = stmt_vinfo->stmt;
   13754     39049030 :           *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
   13755     39049030 :           if (def_stmt_info_out)
   13756     37008242 :             *def_stmt_info_out = stmt_vinfo;
   13757              :         }
   13758     39929131 :       if (def_stmt_out)
   13759     10124715 :         *def_stmt_out = def_stmt;
   13760              :     }
   13761              : 
   13762     44459959 :   if (dump_enabled_p ())
   13763              :     {
   13764       818656 :       dump_printf (MSG_NOTE, ", type of def: ");
   13765       818656 :       switch (*dt)
   13766              :         {
   13767            0 :         case vect_uninitialized_def:
   13768            0 :           dump_printf (MSG_NOTE, "uninitialized\n");
   13769            0 :           break;
   13770        58854 :         case vect_constant_def:
   13771        58854 :           dump_printf (MSG_NOTE, "constant\n");
   13772        58854 :           break;
   13773        28045 :         case vect_external_def:
   13774        28045 :           dump_printf (MSG_NOTE, "external\n");
   13775        28045 :           break;
   13776       583347 :         case vect_internal_def:
   13777       583347 :           dump_printf (MSG_NOTE, "internal\n");
   13778       583347 :           break;
   13779       115946 :         case vect_induction_def:
   13780       115946 :           dump_printf (MSG_NOTE, "induction\n");
   13781       115946 :           break;
   13782        29050 :         case vect_reduction_def:
   13783        29050 :           dump_printf (MSG_NOTE, "reduction\n");
   13784        29050 :           break;
   13785          482 :         case vect_double_reduction_def:
   13786          482 :           dump_printf (MSG_NOTE, "double reduction\n");
   13787          482 :           break;
   13788         2193 :         case vect_nested_cycle:
   13789         2193 :           dump_printf (MSG_NOTE, "nested cycle\n");
   13790         2193 :           break;
   13791          277 :         case vect_first_order_recurrence:
   13792          277 :           dump_printf (MSG_NOTE, "first order recurrence\n");
   13793          277 :           break;
   13794            0 :         case vect_condition_def:
   13795            0 :           dump_printf (MSG_NOTE, "control flow\n");
   13796            0 :           break;
   13797          462 :         case vect_unknown_def_type:
   13798          462 :           dump_printf (MSG_NOTE, "unknown\n");
   13799          462 :           break;
   13800              :         }
   13801              :     }
   13802              : 
   13803     44459959 :   if (*dt == vect_unknown_def_type)
   13804              :     {
   13805        60483 :       if (dump_enabled_p ())
   13806          462 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   13807              :                          "Unsupported pattern.\n");
   13808              :       return false;
   13809              :     }
   13810              : 
   13811              :   return true;
   13812              : }
   13813              : 
   13814              : /* Function vect_is_simple_use.
   13815              : 
   13816              :    Same as vect_is_simple_use but determines the operand by operand
   13817              :    position OPERAND from either STMT or SLP_NODE, filling in *OP
   13818              :    and *SLP_DEF (when SLP_NODE is not NULL).  */
   13819              : 
   13820              : bool
   13821      3729113 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
   13822              :                     unsigned operand, tree *op, slp_tree *slp_def,
   13823              :                     enum vect_def_type *dt,
   13824              :                     tree *vectype, stmt_vec_info *def_stmt_info_out)
   13825              : {
   13826      3729113 :   slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
   13827      3729113 :   *slp_def = child;
   13828      3729113 :   *vectype = SLP_TREE_VECTYPE (child);
   13829      3729113 :   if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
   13830              :     {
   13831              :       /* ???  VEC_PERM nodes might be intermediate and their lane value
   13832              :          have no representative (nor do we build a VEC_PERM stmt for
   13833              :          the actual operation).  Note for two-operator nodes we set
   13834              :          a representative but leave scalar stmts empty as we'd only
   13835              :          have one for a subset of lanes.  Ideally no caller would
   13836              :          require *op for internal defs.  */
   13837      2025085 :       if (SLP_TREE_REPRESENTATIVE (child))
   13838              :         {
   13839      1967651 :           *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
   13840      1967651 :           return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
   13841              :         }
   13842              :       else
   13843              :         {
   13844        57434 :           gcc_assert (SLP_TREE_PERMUTE_P (child));
   13845        57434 :           *op = error_mark_node;
   13846        57434 :           *dt = vect_internal_def;
   13847        57434 :           if (def_stmt_info_out)
   13848          155 :             *def_stmt_info_out = NULL;
   13849              :           return true;
   13850              :         }
   13851              :     }
   13852              :   else
   13853              :     {
   13854      1704028 :       if (def_stmt_info_out)
   13855        55907 :         *def_stmt_info_out = NULL;
   13856      1704028 :       *op = SLP_TREE_SCALAR_OPS (child)[0];
   13857      1704028 :       *dt = SLP_TREE_DEF_TYPE (child);
   13858      1704028 :       return true;
   13859              :     }
   13860              : }
   13861              : 
   13862              : /* Wrapper around vect_is_simple_use that elides the scalar op output.  */
   13863              : 
   13864              : bool
   13865       285565 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
   13866              :                     unsigned operand, slp_tree *slp_def,
   13867              :                     enum vect_def_type *dt, tree *vectype)
   13868              : {
   13869       285565 :   tree op;
   13870       285565 :   return vect_is_simple_use (vinfo, slp_node, operand, &op, slp_def, dt,
   13871       285565 :                              vectype);
   13872              : }
   13873              : 
   13874              : /* If OP is not NULL and is external or constant update its vector
   13875              :    type with VECTYPE.  Returns true if successful or false if not,
   13876              :    for example when conflicting vector types are present.  */
   13877              : 
   13878              : bool
   13879      3253881 : vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
   13880              : {
   13881      3253881 :   if (!op || SLP_TREE_DEF_TYPE (op) == vect_internal_def)
   13882              :     return true;
   13883      1123942 :   if (SLP_TREE_VECTYPE (op))
   13884        23916 :     return types_compatible_p (SLP_TREE_VECTYPE (op), vectype);
   13885              :   /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
   13886              :      should be handled by patters.  Allow vect_constant_def for now
   13887              :      as well as the trivial single-lane uniform vect_external_def case
   13888              :      both of which we code-generate reasonably.  */
   13889      1100026 :   if (VECTOR_BOOLEAN_TYPE_P (vectype)
   13890         1713 :       && SLP_TREE_DEF_TYPE (op) == vect_external_def
   13891      1101272 :       && SLP_TREE_LANES (op) > 1)
   13892              :     return false;
   13893      1099836 :   SLP_TREE_VECTYPE (op) = vectype;
   13894      1099836 :   return true;
   13895              : }
   13896              : 
   13897              : /* Function supportable_widening_operation
   13898              : 
   13899              :    Check whether an operation represented by the code CODE is a
   13900              :    widening operation that is supported by the target platform in
   13901              :    vector form (i.e., when operating on arguments of type VECTYPE_IN
   13902              :    producing a result of type VECTYPE_OUT).
   13903              : 
   13904              :    Widening operations we currently support are NOP (CONVERT), FLOAT,
   13905              :    FIX_TRUNC and WIDEN_MULT.  This function checks if these operations
   13906              :    are supported by the target platform either directly (via vector
   13907              :    tree-codes), or via target builtins.
   13908              : 
   13909              :    When EVENODD_OK then also lane-swizzling operations are considered.
   13910              : 
   13911              :    Output:
   13912              :    - CODE1 and CODE2 are codes of vector operations to be used when
   13913              :    vectorizing the operation, if available.
   13914              :    - MULTI_STEP_CVT determines the number of required intermediate steps in
   13915              :    case of multi-step conversion (like char->short->int - in that case
   13916              :    MULTI_STEP_CVT will be 1).
   13917              :    - INTERM_TYPES contains the intermediate type required to perform the
   13918              :    widening operation (short in the above example).  */
   13919              : 
   13920              : bool
   13921       556888 : supportable_widening_operation (code_helper code,
   13922              :                                 tree vectype_out, tree vectype_in,
   13923              :                                 bool evenodd_ok,
   13924              :                                 code_helper *code1,
   13925              :                                 code_helper *code2,
   13926              :                                 int *multi_step_cvt,
   13927              :                                 vec<tree> *interm_types)
   13928              : {
   13929       556888 :   machine_mode vec_mode;
   13930       556888 :   enum insn_code icode1, icode2;
   13931       556888 :   optab optab1 = unknown_optab, optab2 = unknown_optab;
   13932       556888 :   tree vectype = vectype_in;
   13933       556888 :   tree wide_vectype = vectype_out;
   13934       556888 :   tree_code c1 = MAX_TREE_CODES, c2 = MAX_TREE_CODES;
   13935       556888 :   int i;
   13936       556888 :   tree prev_type, intermediate_type;
   13937       556888 :   machine_mode intermediate_mode, prev_mode;
   13938       556888 :   optab optab3, optab4;
   13939              : 
   13940       556888 :   *multi_step_cvt = 0;
   13941              : 
   13942       556888 :   switch (code.safe_as_tree_code ())
   13943              :     {
   13944              :     case MAX_TREE_CODES:
   13945              :       /* Don't set c1 and c2 if code is not a tree_code.  */
   13946              :       break;
   13947              : 
   13948       211589 :     case WIDEN_MULT_EXPR:
   13949              :       /* The result of a vectorized widening operation usually requires
   13950              :          two vectors (because the widened results do not fit into one vector).
   13951              :          The generated vector results would normally be expected to be
   13952              :          generated in the same order as in the original scalar computation,
   13953              :          i.e. if 8 results are generated in each vector iteration, they are
   13954              :          to be organized as follows:
   13955              :                 vect1: [res1,res2,res3,res4],
   13956              :                 vect2: [res5,res6,res7,res8].
   13957              : 
   13958              :          However, in the special case that the result of the widening
   13959              :          operation is used in a reduction computation only, the order doesn't
   13960              :          matter (because when vectorizing a reduction we change the order of
   13961              :          the computation).  Some targets can take advantage of this and
   13962              :          generate more efficient code.  For example, targets like Altivec,
   13963              :          that support widen_mult using a sequence of {mult_even,mult_odd}
   13964              :          generate the following vectors:
   13965              :                 vect1: [res1,res3,res5,res7],
   13966              :                 vect2: [res2,res4,res6,res8].
   13967              : 
   13968              :          When vectorizing outer-loops, we execute the inner-loop sequentially
   13969              :          (each vectorized inner-loop iteration contributes to VF outer-loop
   13970              :          iterations in parallel).  We therefore don't allow to change the
   13971              :          order of the computation in the inner-loop during outer-loop
   13972              :          vectorization.  */
   13973              :       /* TODO: Another case in which order doesn't *really* matter is when we
   13974              :          widen and then contract again, e.g. (short)((int)x * y >> 8).
   13975              :          Normally, pack_trunc performs an even/odd permute, whereas the
   13976              :          repack from an even/odd expansion would be an interleave, which
   13977              :          would be significantly simpler for e.g. AVX2.  */
   13978              :       /* In any case, in order to avoid duplicating the code below, recurse
   13979              :          on VEC_WIDEN_MULT_EVEN_EXPR.  If it succeeds, all the return values
   13980              :          are properly set up for the caller.  If we fail, we'll continue with
   13981              :          a VEC_WIDEN_MULT_LO/HI_EXPR check.  */
   13982       211589 :       if (evenodd_ok
   13983       211589 :           && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
   13984              :                                              vectype_out, vectype_in,
   13985              :                                              evenodd_ok, code1,
   13986              :                                              code2, multi_step_cvt,
   13987              :                                              interm_types))
   13988       108456 :         return true;
   13989              :       c1 = VEC_WIDEN_MULT_LO_EXPR;
   13990              :       c2 = VEC_WIDEN_MULT_HI_EXPR;
   13991              :       break;
   13992              : 
   13993              :     case DOT_PROD_EXPR:
   13994       448432 :       c1 = DOT_PROD_EXPR;
   13995       448432 :       c2 = DOT_PROD_EXPR;
   13996              :       break;
   13997              : 
   13998            0 :     case SAD_EXPR:
   13999            0 :       c1 = SAD_EXPR;
   14000            0 :       c2 = SAD_EXPR;
   14001            0 :       break;
   14002              : 
   14003       209447 :     case VEC_WIDEN_MULT_EVEN_EXPR:
   14004              :       /* Support the recursion induced just above.  */
   14005       209447 :       c1 = VEC_WIDEN_MULT_EVEN_EXPR;
   14006       209447 :       c2 = VEC_WIDEN_MULT_ODD_EXPR;
   14007       209447 :       break;
   14008              : 
   14009         9813 :     case WIDEN_LSHIFT_EXPR:
   14010         9813 :       c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
   14011         9813 :       c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
   14012         9813 :       break;
   14013              : 
   14014        45515 :     CASE_CONVERT:
   14015        45515 :       c1 = VEC_UNPACK_LO_EXPR;
   14016        45515 :       c2 = VEC_UNPACK_HI_EXPR;
   14017        45515 :       break;
   14018              : 
   14019         9281 :     case FLOAT_EXPR:
   14020         9281 :       c1 = VEC_UNPACK_FLOAT_LO_EXPR;
   14021         9281 :       c2 = VEC_UNPACK_FLOAT_HI_EXPR;
   14022         9281 :       break;
   14023              : 
   14024          127 :     case FIX_TRUNC_EXPR:
   14025          127 :       c1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
   14026          127 :       c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
   14027          127 :       break;
   14028              : 
   14029            0 :     default:
   14030            0 :       gcc_unreachable ();
   14031              :     }
   14032              : 
   14033       448432 :   if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
   14034              :     std::swap (c1, c2);
   14035              : 
   14036       448432 :   if (code == FIX_TRUNC_EXPR)
   14037              :     {
   14038              :       /* The signedness is determined from output operand.  */
   14039          127 :       optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
   14040          127 :       optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
   14041              :     }
   14042       788620 :   else if (CONVERT_EXPR_CODE_P (code.safe_as_tree_code ())
   14043        45515 :            && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
   14044         8257 :            && VECTOR_BOOLEAN_TYPE_P (vectype)
   14045         8257 :            && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
   14046       377653 :            && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
   14047              :     {
   14048              :       /* If the input and result modes are the same, a different optab
   14049              :          is needed where we pass in the number of units in vectype.  */
   14050              :       optab1 = vec_unpacks_sbool_lo_optab;
   14051              :       optab2 = vec_unpacks_sbool_hi_optab;
   14052              :     }
   14053              : 
   14054       448432 :   vec_mode = TYPE_MODE (vectype);
   14055       448432 :   if (widening_fn_p (code))
   14056              :      {
   14057              :        /* If this is an internal fn then we must check whether the target
   14058              :           supports either a low-high split or an even-odd split.  */
   14059        71116 :       internal_fn ifn = as_internal_fn ((combined_fn) code);
   14060              : 
   14061        71116 :       internal_fn lo, hi, even, odd;
   14062        71116 :       lookup_hilo_internal_fn (ifn, &lo, &hi);
   14063        71116 :       if (BYTES_BIG_ENDIAN)
   14064              :         std::swap (lo, hi);
   14065        71116 :       *code1 = as_combined_fn (lo);
   14066        71116 :       *code2 = as_combined_fn (hi);
   14067        71116 :       optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
   14068        71116 :       optab2 = direct_internal_fn_optab (hi, {vectype, vectype});
   14069              : 
   14070              :       /* If we don't support low-high, then check for even-odd.  */
   14071        71116 :       if (!optab1
   14072        71116 :           || (icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
   14073            0 :           || !optab2
   14074        71116 :           || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
   14075              :         {
   14076        71116 :           lookup_evenodd_internal_fn (ifn, &even, &odd);
   14077        71116 :           *code1 = as_combined_fn (even);
   14078        71116 :           *code2 = as_combined_fn (odd);
   14079        71116 :           optab1 = direct_internal_fn_optab (even, {vectype, vectype});
   14080        71116 :           optab2 = direct_internal_fn_optab (odd, {vectype, vectype});
   14081              :         }
   14082              :     }
   14083       377316 :   else if (code.is_tree_code ())
   14084              :     {
   14085       377316 :       if (code == FIX_TRUNC_EXPR)
   14086              :         {
   14087              :           /* The signedness is determined from output operand.  */
   14088          127 :           optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
   14089          127 :           optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
   14090              :         }
   14091       377189 :       else if (CONVERT_EXPR_CODE_P ((tree_code) code.safe_as_tree_code ())
   14092        45515 :                && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
   14093         8257 :                && VECTOR_BOOLEAN_TYPE_P (vectype)
   14094         8257 :                && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
   14095       377653 :                && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
   14096              :         {
   14097              :           /* If the input and result modes are the same, a different optab
   14098              :              is needed where we pass in the number of units in vectype.  */
   14099              :           optab1 = vec_unpacks_sbool_lo_optab;
   14100              :           optab2 = vec_unpacks_sbool_hi_optab;
   14101              :         }
   14102              :       else
   14103              :         {
   14104       376725 :           optab1 = optab_for_tree_code (c1, vectype, optab_default);
   14105       376725 :           optab2 = optab_for_tree_code (c2, vectype, optab_default);
   14106              :         }
   14107       377316 :       *code1 = c1;
   14108       377316 :       *code2 = c2;
   14109              :     }
   14110              : 
   14111       448432 :   if (!optab1 || !optab2)
   14112              :     return false;
   14113              : 
   14114       448432 :   if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
   14115       448432 :        || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
   14116              :     return false;
   14117              : 
   14118              : 
   14119       168632 :   if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
   14120       168632 :       && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
   14121              :     {
   14122       156885 :       if (!VECTOR_BOOLEAN_TYPE_P (vectype))
   14123              :         return true;
   14124              :       /* For scalar masks we may have different boolean
   14125              :          vector types having the same QImode.  Thus we
   14126              :          add additional check for elements number.  */
   14127         4459 :       if (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
   14128              :                     TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
   14129              :         return true;
   14130              :     }
   14131              : 
   14132              :   /* Check if it's a multi-step conversion that can be done using intermediate
   14133              :      types.  */
   14134              : 
   14135        11976 :   prev_type = vectype;
   14136        11976 :   prev_mode = vec_mode;
   14137              : 
   14138        11976 :   if (!CONVERT_EXPR_CODE_P (code.safe_as_tree_code ()))
   14139              :     return false;
   14140              : 
   14141              :   /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
   14142              :      intermediate steps in promotion sequence.  We try
   14143              :      MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
   14144              :      not.  */
   14145        11924 :   interm_types->create (MAX_INTERM_CVT_STEPS);
   14146        25170 :   for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
   14147              :     {
   14148        13246 :       intermediate_mode = insn_data[icode1].operand[0].mode;
   14149        13246 :       if (VECTOR_BOOLEAN_TYPE_P (prev_type))
   14150         4699 :         intermediate_type
   14151         4699 :           = vect_halve_mask_nunits (prev_type, intermediate_mode);
   14152         8547 :       else if (VECTOR_MODE_P (intermediate_mode))
   14153              :         {
   14154         8547 :           tree intermediate_element_type
   14155         8547 :             = lang_hooks.types.type_for_mode (GET_MODE_INNER (intermediate_mode),
   14156         8547 :                                               TYPE_UNSIGNED (prev_type));
   14157         8547 :           intermediate_type
   14158         8547 :             = build_vector_type_for_mode (intermediate_element_type,
   14159              :                                           intermediate_mode);
   14160         8547 :         }
   14161              :       else
   14162            0 :         intermediate_type
   14163            0 :           = lang_hooks.types.type_for_mode (intermediate_mode,
   14164            0 :                                             TYPE_UNSIGNED (prev_type));
   14165              : 
   14166        13246 :       if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
   14167         4699 :           && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
   14168         4699 :           && intermediate_mode == TYPE_MODE (wide_vectype)
   14169        13557 :           && SCALAR_INT_MODE_P (intermediate_mode))
   14170              :         {
   14171              :           /* If the input and result modes are the same, a different optab
   14172              :              is needed where we pass in the number of units in vectype.  */
   14173              :           optab3 = vec_unpacks_sbool_lo_optab;
   14174              :           optab4 = vec_unpacks_sbool_hi_optab;
   14175              :         }
   14176              :       else
   14177              :         {
   14178        12935 :           optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
   14179        12935 :           optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
   14180              :         }
   14181              : 
   14182        13246 :       if (!optab3 || !optab4
   14183        13246 :           || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
   14184        13206 :           || insn_data[icode1].operand[0].mode != intermediate_mode
   14185        13206 :           || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
   14186        13206 :           || insn_data[icode2].operand[0].mode != intermediate_mode
   14187        13206 :           || ((icode1 = optab_handler (optab3, intermediate_mode))
   14188              :               == CODE_FOR_nothing)
   14189        26068 :           || ((icode2 = optab_handler (optab4, intermediate_mode))
   14190              :               == CODE_FOR_nothing))
   14191              :         break;
   14192              : 
   14193        12822 :       interm_types->quick_push (intermediate_type);
   14194        12822 :       (*multi_step_cvt)++;
   14195              : 
   14196        12822 :       if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
   14197        12822 :           && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
   14198              :         {
   14199        11582 :           if (!VECTOR_BOOLEAN_TYPE_P (vectype))
   14200              :             return true;
   14201         3774 :           if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
   14202              :                         TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
   14203              :             return true;
   14204              :         }
   14205              : 
   14206         1322 :       prev_type = intermediate_type;
   14207         1322 :       prev_mode = intermediate_mode;
   14208              :     }
   14209              : 
   14210          424 :   interm_types->release ();
   14211          424 :   return false;
   14212              : }
   14213              : 
   14214              : 
   14215              : /* Function supportable_narrowing_operation
   14216              : 
   14217              :    Check whether an operation represented by the code CODE is a
   14218              :    narrowing operation that is supported by the target platform in
   14219              :    vector form (i.e., when operating on arguments of type VECTYPE_IN
   14220              :    and producing a result of type VECTYPE_OUT).
   14221              : 
   14222              :    Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
   14223              :    and FLOAT.  This function checks if these operations are supported by
   14224              :    the target platform directly via vector tree-codes.
   14225              : 
   14226              :    Output:
   14227              :    - CODE1 is the code of a vector operation to be used when
   14228              :    vectorizing the operation, if available.
   14229              :    - MULTI_STEP_CVT determines the number of required intermediate steps in
   14230              :    case of multi-step conversion (like int->short->char - in that case
   14231              :    MULTI_STEP_CVT will be 1).
   14232              :    - INTERM_TYPES contains the intermediate type required to perform the
   14233              :    narrowing operation (short in the above example).   */
   14234              : 
   14235              : bool
   14236        45351 : supportable_narrowing_operation (code_helper code,
   14237              :                                  tree vectype_out, tree vectype_in,
   14238              :                                  code_helper *code1, int *multi_step_cvt,
   14239              :                                  vec<tree> *interm_types)
   14240              : {
   14241        45351 :   machine_mode vec_mode;
   14242        45351 :   enum insn_code icode1;
   14243        45351 :   optab optab1, interm_optab;
   14244        45351 :   tree vectype = vectype_in;
   14245        45351 :   tree narrow_vectype = vectype_out;
   14246        45351 :   enum tree_code c1;
   14247        45351 :   tree intermediate_type, prev_type;
   14248        45351 :   machine_mode intermediate_mode, prev_mode;
   14249        45351 :   int i;
   14250        45351 :   unsigned HOST_WIDE_INT n_elts;
   14251        45351 :   bool uns;
   14252              : 
   14253        45351 :   if (!code.is_tree_code ())
   14254              :     return false;
   14255              : 
   14256        45351 :   *multi_step_cvt = 0;
   14257        45351 :   switch ((tree_code) code)
   14258              :     {
   14259        43841 :     CASE_CONVERT:
   14260        43841 :       c1 = VEC_PACK_TRUNC_EXPR;
   14261        43841 :       if (VECTOR_BOOLEAN_TYPE_P (narrow_vectype)
   14262        11630 :           && VECTOR_BOOLEAN_TYPE_P (vectype)
   14263        11630 :           && SCALAR_INT_MODE_P (TYPE_MODE (vectype))
   14264         5271 :           && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&n_elts)
   14265        49112 :           && n_elts < BITS_PER_UNIT)
   14266              :         optab1 = vec_pack_sbool_trunc_optab;
   14267              :       else
   14268        41354 :         optab1 = optab_for_tree_code (c1, vectype, optab_default);
   14269              :       break;
   14270              : 
   14271          572 :     case FIX_TRUNC_EXPR:
   14272          572 :       c1 = VEC_PACK_FIX_TRUNC_EXPR;
   14273              :       /* The signedness is determined from output operand.  */
   14274          572 :       optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
   14275          572 :       break;
   14276              : 
   14277          938 :     case FLOAT_EXPR:
   14278          938 :       c1 = VEC_PACK_FLOAT_EXPR;
   14279          938 :       optab1 = optab_for_tree_code (c1, vectype, optab_default);
   14280          938 :       break;
   14281              : 
   14282            0 :     default:
   14283            0 :       gcc_unreachable ();
   14284              :     }
   14285              : 
   14286        45351 :   if (!optab1)
   14287              :     return false;
   14288              : 
   14289        45351 :   vec_mode = TYPE_MODE (vectype);
   14290        45351 :   if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
   14291              :     return false;
   14292              : 
   14293        39787 :   *code1 = c1;
   14294              : 
   14295        39787 :   if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
   14296              :     {
   14297        25126 :       if (!VECTOR_BOOLEAN_TYPE_P (vectype))
   14298              :         return true;
   14299              :       /* For scalar masks we may have different boolean
   14300              :          vector types having the same QImode.  Thus we
   14301              :          add additional check for elements number.  */
   14302         5833 :       if (known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
   14303              :                     TYPE_VECTOR_SUBPARTS (narrow_vectype)))
   14304              :         return true;
   14305              :     }
   14306              : 
   14307        14812 :   if (code == FLOAT_EXPR)
   14308              :     return false;
   14309              : 
   14310              :   /* Check if it's a multi-step conversion that can be done using intermediate
   14311              :      types.  */
   14312        14812 :   prev_mode = vec_mode;
   14313        14812 :   prev_type = vectype;
   14314        14812 :   if (code == FIX_TRUNC_EXPR)
   14315           94 :     uns = TYPE_UNSIGNED (vectype_out);
   14316              :   else
   14317        14718 :     uns = TYPE_UNSIGNED (vectype);
   14318              : 
   14319              :   /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
   14320              :      conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
   14321              :      costly than signed.  */
   14322        14812 :   if (code == FIX_TRUNC_EXPR && uns)
   14323              :     {
   14324           28 :       enum insn_code icode2;
   14325              : 
   14326           28 :       intermediate_type
   14327           28 :         = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
   14328           28 :       interm_optab
   14329           28 :         = optab_for_tree_code (c1, intermediate_type, optab_default);
   14330           28 :       if (interm_optab != unknown_optab
   14331           28 :           && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
   14332           28 :           && insn_data[icode1].operand[0].mode
   14333           28 :              == insn_data[icode2].operand[0].mode)
   14334              :         {
   14335              :           uns = false;
   14336              :           optab1 = interm_optab;
   14337              :           icode1 = icode2;
   14338              :         }
   14339              :     }
   14340              : 
   14341              :   /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
   14342              :      intermediate steps in promotion sequence.  We try
   14343              :      MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not.  */
   14344        14812 :   interm_types->create (MAX_INTERM_CVT_STEPS);
   14345        31708 :   for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
   14346              :     {
   14347        16896 :       intermediate_mode = insn_data[icode1].operand[0].mode;
   14348        16896 :       if (VECTOR_BOOLEAN_TYPE_P (prev_type))
   14349         7028 :         intermediate_type
   14350         7028 :           = vect_double_mask_nunits (prev_type, intermediate_mode);
   14351              :       else
   14352         9868 :         intermediate_type
   14353         9868 :           = lang_hooks.types.type_for_mode (intermediate_mode, uns);
   14354        16896 :       if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
   14355         7028 :           && VECTOR_BOOLEAN_TYPE_P (prev_type)
   14356         7028 :           && SCALAR_INT_MODE_P (prev_mode)
   14357         3144 :           && TYPE_VECTOR_SUBPARTS (intermediate_type).is_constant (&n_elts)
   14358        20040 :           && n_elts < BITS_PER_UNIT)
   14359              :         interm_optab = vec_pack_sbool_trunc_optab;
   14360              :       else
   14361        16544 :         interm_optab
   14362        16544 :           = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
   14363              :                                  optab_default);
   14364          352 :       if (!interm_optab
   14365        16896 :           || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
   14366        16896 :           || insn_data[icode1].operand[0].mode != intermediate_mode
   14367        33440 :           || ((icode1 = optab_handler (interm_optab, intermediate_mode))
   14368              :               == CODE_FOR_nothing))
   14369              :         break;
   14370              : 
   14371        15965 :       interm_types->quick_push (intermediate_type);
   14372        15965 :       (*multi_step_cvt)++;
   14373              : 
   14374        15965 :       if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
   14375              :         {
   14376        13881 :           if (!VECTOR_BOOLEAN_TYPE_P (vectype))
   14377              :             return true;
   14378         4962 :           if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
   14379              :                         TYPE_VECTOR_SUBPARTS (narrow_vectype)))
   14380              :             return true;
   14381              :         }
   14382              : 
   14383         2084 :       prev_mode = intermediate_mode;
   14384         2084 :       prev_type = intermediate_type;
   14385         2084 :       optab1 = interm_optab;
   14386              :     }
   14387              : 
   14388          931 :   interm_types->release ();
   14389          931 :   return false;
   14390              : }
   14391              : 
   14392              : /* Function supportable_indirect_convert_operation
   14393              : 
   14394              :    Check whether an operation represented by the code CODE is single or multi
   14395              :    operations that are supported by the target platform in
   14396              :    vector form (i.e., when operating on arguments of type VECTYPE_IN
   14397              :    producing a result of type VECTYPE_OUT).
   14398              : 
   14399              :    Convert operations we currently support directly are FIX_TRUNC and FLOAT.
   14400              :    This function checks if these operations are supported
   14401              :    by the target platform directly (via vector tree-codes).
   14402              : 
   14403              :    Output:
   14404              :    - converts contains some pairs to perform the convert operation,
   14405              :    the pair's first is the intermediate type, and its second is the code of
   14406              :    a vector operation to be used when converting the operation from the
   14407              :    previous type to the intermediate type. */
   14408              : bool
   14409        94811 : supportable_indirect_convert_operation (code_helper code,
   14410              :                                         tree vectype_out,
   14411              :                                         tree vectype_in,
   14412              :                                         vec<std::pair<tree, tree_code> > &converts,
   14413              :                                         slp_tree slp_op0)
   14414              : {
   14415        94811 :   bool found_mode = false;
   14416        94811 :   scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out));
   14417        94811 :   scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in));
   14418        94811 :   tree_code code1, code2;
   14419              : 
   14420        94811 :   tree cvt_type = NULL_TREE;
   14421        94811 :   poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (vectype_in);
   14422              : 
   14423        94811 :   if (supportable_convert_operation ((tree_code) code,
   14424              :                                      vectype_out,
   14425              :                                      vectype_in))
   14426              :     {
   14427        87240 :       converts.safe_push (std::make_pair (vectype_out, (tree_code) code));
   14428        87240 :       return true;
   14429              :     }
   14430              : 
   14431              :   /* For conversions between mask types where the destination has
   14432              :      a data mode attempt a vcond_mask conversion.  */
   14433         7571 :   if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
   14434          145 :       && VECTOR_BOOLEAN_TYPE_P (vectype_out)
   14435         7716 :       && GET_MODE_CLASS (TYPE_MODE (vectype_out)) == MODE_VECTOR_INT)
   14436              :     {
   14437          144 :       tree scalar_datatype
   14438          144 :         = build_nonstandard_integer_type (element_precision (vectype_out), 0);
   14439          144 :       tree datatype_out = build_vector_type_for_mode (scalar_datatype,
   14440          144 :                                                       TYPE_MODE (vectype_out));
   14441          144 :       if (expand_vec_cond_expr_p (datatype_out, vectype_in))
   14442              :         {
   14443            2 :           converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
   14444            2 :           return true;
   14445              :         }
   14446              :     }
   14447              : 
   14448              :   /* For conversions between float and integer types try whether
   14449              :      we can use intermediate signed integer types to support the
   14450              :      conversion.  */
   14451        15138 :   if (GET_MODE_SIZE (lhs_mode) != GET_MODE_SIZE (rhs_mode)
   14452         7569 :       && (code == FLOAT_EXPR
   14453         4933 :           || (code == FIX_TRUNC_EXPR && !flag_trapping_math)))
   14454              :     {
   14455         1592 :       bool demotion = GET_MODE_SIZE (rhs_mode) > GET_MODE_SIZE (lhs_mode);
   14456          796 :       bool float_expr_p = code == FLOAT_EXPR;
   14457          796 :       unsigned short target_size;
   14458          796 :       scalar_mode intermediate_mode;
   14459          796 :       if (demotion)
   14460              :         {
   14461           99 :           intermediate_mode = lhs_mode;
   14462           99 :           target_size = GET_MODE_SIZE (rhs_mode);
   14463              :         }
   14464              :       else
   14465              :         {
   14466          697 :           target_size = GET_MODE_SIZE (lhs_mode);
   14467          697 :           if (!int_mode_for_size
   14468          697 :               (GET_MODE_BITSIZE (rhs_mode), 0).exists (&intermediate_mode))
   14469          142 :             return false;
   14470              :         }
   14471          796 :       code1 = float_expr_p ? (tree_code) code : NOP_EXPR;
   14472          729 :       code2 = float_expr_p ? NOP_EXPR : (tree_code) code;
   14473          796 :       opt_scalar_mode mode_iter;
   14474         2483 :       FOR_EACH_2XWIDER_MODE (mode_iter, intermediate_mode)
   14475              :         {
   14476         2483 :           intermediate_mode = mode_iter.require ();
   14477              : 
   14478         4966 :           if (GET_MODE_SIZE (intermediate_mode) > target_size)
   14479              :             break;
   14480              : 
   14481         1882 :           scalar_mode cvt_mode;
   14482         1882 :           if (!int_mode_for_size
   14483         1882 :               (GET_MODE_BITSIZE (intermediate_mode), 0).exists (&cvt_mode))
   14484              :             break;
   14485              : 
   14486         1852 :           cvt_type = build_nonstandard_integer_type
   14487         1852 :             (GET_MODE_BITSIZE (cvt_mode), 0);
   14488              : 
   14489              :           /* Check if the intermediate type can hold OP0's range.
   14490              :              When converting from float to integer this is not necessary
   14491              :              because values that do not fit the (smaller) target type are
   14492              :              unspecified anyway.  */
   14493         1852 :           if (demotion && float_expr_p)
   14494              :             {
   14495           23 :               wide_int op_min_value, op_max_value;
   14496              :               /* For vector form, it looks like slp_op0 doesn't have RANGE_INFO.
   14497              :                  In the future, if it is supported, changes may need to be made
   14498              :                  to this part, such as checking the RANGE of each element
   14499              :                  in the vector.  */
   14500           23 :               if (slp_op0)
   14501              :                 {
   14502           13 :                   tree def;
   14503              :                   /* ???  Merge ranges in case of more than one lane.  */
   14504           13 :                   if (SLP_TREE_LANES (slp_op0) != 1
   14505            0 :                       || !(def = vect_get_slp_scalar_def (slp_op0, 0))
   14506           13 :                       || !vect_get_range_info (def,
   14507              :                                                &op_min_value, &op_max_value))
   14508              :                     break;
   14509              :                 }
   14510              :               else
   14511              :                 break;
   14512              : 
   14513            0 :               if (cvt_type == NULL_TREE
   14514            0 :                   || (wi::min_precision (op_max_value, SIGNED)
   14515            0 :                       > TYPE_PRECISION (cvt_type))
   14516            0 :                   || (wi::min_precision (op_min_value, SIGNED)
   14517            0 :                       > TYPE_PRECISION (cvt_type)))
   14518            0 :                 continue;
   14519           23 :             }
   14520              : 
   14521         1829 :           cvt_type = get_related_vectype_for_scalar_type (TYPE_MODE (vectype_in),
   14522              :                                                           cvt_type,
   14523              :                                                           nelts);
   14524              :           /* This should only happened for SLP as long as loop vectorizer
   14525              :              only supports same-sized vector.  */
   14526         3516 :           if (cvt_type == NULL_TREE
   14527         3658 :               || maybe_ne (TYPE_VECTOR_SUBPARTS (cvt_type), nelts)
   14528         1829 :               || !supportable_convert_operation ((tree_code) code1,
   14529              :                                                  vectype_out, cvt_type)
   14530         2564 :               || !supportable_convert_operation ((tree_code) code2,
   14531              :                                                  cvt_type, vectype_in))
   14532         1687 :             continue;
   14533              : 
   14534              :           found_mode = true;
   14535              :           break;
   14536              :         }
   14537              : 
   14538          796 :       if (found_mode)
   14539              :         {
   14540          142 :           converts.safe_push (std::make_pair (cvt_type, code2));
   14541          142 :           if (TYPE_MODE (cvt_type) != TYPE_MODE (vectype_out))
   14542          142 :             converts.safe_push (std::make_pair (vectype_out, code1));
   14543              :           return true;
   14544              :         }
   14545              :     }
   14546              :   return false;
   14547              : }
   14548              : 
   14549              : /* Generate and return a vector mask of MASK_TYPE such that
   14550              :    mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
   14551              :    Add the statements to SEQ.  */
   14552              : 
   14553              : tree
   14554            0 : vect_gen_while (gimple_seq *seq, tree mask_type, tree start_index,
   14555              :                 tree end_index, const char *name)
   14556              : {
   14557            0 :   tree cmp_type = TREE_TYPE (start_index);
   14558            0 :   gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
   14559              :                                                        cmp_type, mask_type,
   14560              :                                                        OPTIMIZE_FOR_SPEED));
   14561            0 :   gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
   14562              :                                             start_index, end_index,
   14563              :                                             build_zero_cst (mask_type));
   14564            0 :   tree tmp;
   14565            0 :   if (name)
   14566            0 :     tmp = make_temp_ssa_name (mask_type, NULL, name);
   14567              :   else
   14568            0 :     tmp = make_ssa_name (mask_type);
   14569            0 :   gimple_call_set_lhs (call, tmp);
   14570            0 :   gimple_seq_add_stmt (seq, call);
   14571            0 :   return tmp;
   14572              : }
   14573              : 
   14574              : /* Generate a vector mask of type MASK_TYPE for which index I is false iff
   14575              :    J + START_INDEX < END_INDEX for all J <= I.  Add the statements to SEQ.  */
   14576              : 
   14577              : tree
   14578            0 : vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
   14579              :                     tree end_index)
   14580              : {
   14581            0 :   tree tmp = vect_gen_while (seq, mask_type, start_index, end_index);
   14582            0 :   return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
   14583              : }
   14584              : 
   14585              : /* Try to compute the vector types required to vectorize STMT_INFO,
   14586              :    returning true on success and false if vectorization isn't possible.
   14587              :    If GROUP_SIZE is nonzero and we're performing BB vectorization,
   14588              :    take sure that the number of elements in the vectors is no bigger
   14589              :    than GROUP_SIZE.
   14590              : 
   14591              :    On success:
   14592              : 
   14593              :    - Set *STMT_VECTYPE_OUT to:
   14594              :      - NULL_TREE if the statement doesn't need to be vectorized;
   14595              :      - the equivalent of STMT_VINFO_VECTYPE otherwise.  */
   14596              : 
   14597              : opt_result
   14598      6252099 : vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
   14599              :                                 tree *stmt_vectype_out,
   14600              :                                 unsigned int group_size)
   14601              : {
   14602      6252099 :   gimple *stmt = stmt_info->stmt;
   14603              : 
   14604              :   /* For BB vectorization, we should always have a group size once we've
   14605              :      constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
   14606              :      are tentative requests during things like early data reference
   14607              :      analysis and pattern recognition.  */
   14608      6252099 :   if (is_a <bb_vec_info> (vinfo))
   14609      4967593 :     gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
   14610              :   else
   14611              :     group_size = 0;
   14612              : 
   14613      6252099 :   *stmt_vectype_out = NULL_TREE;
   14614              : 
   14615      6252099 :   if (gimple_get_lhs (stmt) == NULL_TREE
   14616              :       /* Allow vector conditionals through here.  */
   14617         2657 :       && !is_a <gcond *> (stmt)
   14618              :       /* MASK_STORE and friends have no lhs, but are ok.  */
   14619      6257383 :       && !(is_gimple_call (stmt)
   14620         2657 :            && gimple_call_internal_p (stmt)
   14621         2627 :            && internal_store_fn_p (gimple_call_internal_fn (stmt))))
   14622              :     {
   14623           30 :       if (is_a <gcall *> (stmt))
   14624              :         {
   14625              :           /* Ignore calls with no lhs.  These must be calls to
   14626              :              #pragma omp simd functions, and what vectorization factor
   14627              :              it really needs can't be determined until
   14628              :              vectorizable_simd_clone_call.  */
   14629           30 :           if (dump_enabled_p ())
   14630           28 :             dump_printf_loc (MSG_NOTE, vect_location,
   14631              :                              "defer to SIMD clone analysis.\n");
   14632           30 :           return opt_result::success ();
   14633              :         }
   14634              : 
   14635            0 :       return opt_result::failure_at (stmt,
   14636              :                                      "not vectorized: irregular stmt: %G", stmt);
   14637              :     }
   14638              : 
   14639      6252069 :   tree vectype;
   14640      6252069 :   tree scalar_type = NULL_TREE;
   14641      6252069 :   if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
   14642              :     {
   14643      1708990 :       vectype = STMT_VINFO_VECTYPE (stmt_info);
   14644      1708990 :       if (dump_enabled_p ())
   14645        85433 :         dump_printf_loc (MSG_NOTE, vect_location,
   14646              :                          "precomputed vectype: %T\n", vectype);
   14647              :     }
   14648      4543079 :   else if (vect_use_mask_type_p (stmt_info))
   14649              :     {
   14650       229022 :       unsigned int precision = stmt_info->mask_precision;
   14651       229022 :       scalar_type = build_nonstandard_integer_type (precision, 1);
   14652       229022 :       vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
   14653       229022 :       if (!vectype)
   14654            0 :         return opt_result::failure_at (stmt, "not vectorized: unsupported"
   14655              :                                        " data-type %T\n", scalar_type);
   14656       229022 :       if (dump_enabled_p ())
   14657         5102 :         dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
   14658              :     }
   14659              :   else
   14660              :     {
   14661              :       /* If we got here with a gcond it means that the target had no available vector
   14662              :          mode for the scalar type.  We can't vectorize so abort.  */
   14663      4314057 :       if (is_a <gcond *> (stmt))
   14664            0 :         return opt_result::failure_at (stmt,
   14665              :                                        "not vectorized:"
   14666              :                                        " unsupported data-type for gcond %T\n",
   14667              :                                        scalar_type);
   14668              : 
   14669      4314057 :       if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
   14670      1532760 :         scalar_type = TREE_TYPE (DR_REF (dr));
   14671              :       else
   14672      2781297 :         scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
   14673              : 
   14674      4314057 :       if (dump_enabled_p ())
   14675              :         {
   14676        69626 :           if (group_size)
   14677         9298 :             dump_printf_loc (MSG_NOTE, vect_location,
   14678              :                              "get vectype for scalar type (group size %d):"
   14679              :                              " %T\n", group_size, scalar_type);
   14680              :           else
   14681        60328 :             dump_printf_loc (MSG_NOTE, vect_location,
   14682              :                              "get vectype for scalar type: %T\n", scalar_type);
   14683              :         }
   14684      4314057 :       vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
   14685      4314057 :       if (!vectype)
   14686       257784 :         return opt_result::failure_at (stmt,
   14687              :                                        "not vectorized:"
   14688              :                                        " unsupported data-type %T\n",
   14689              :                                        scalar_type);
   14690              : 
   14691      4056273 :       if (dump_enabled_p ())
   14692        69355 :         dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
   14693              :     }
   14694              : 
   14695      4370728 :   if (scalar_type && VECTOR_MODE_P (TYPE_MODE (scalar_type)))
   14696            0 :     return opt_result::failure_at (stmt,
   14697              :                                    "not vectorized: vector stmt in loop:%G",
   14698              :                                    stmt);
   14699              : 
   14700      5994285 :   *stmt_vectype_out = vectype;
   14701      5994285 :   return opt_result::success ();
   14702              : }
   14703              : 
   14704              : /* Generate and return statement sequence that sets vector length LEN that is:
   14705              : 
   14706              :    min_of_start_and_end = min (START_INDEX, END_INDEX);
   14707              :    left_len = END_INDEX - min_of_start_and_end;
   14708              :    rhs = min (left_len, LEN_LIMIT);
   14709              :    LEN = rhs;
   14710              : 
   14711              :    Note: the cost of the code generated by this function is modeled
   14712              :    by vect_estimate_min_profitable_iters, so changes here may need
   14713              :    corresponding changes there.  */
   14714              : 
   14715              : gimple_seq
   14716            0 : vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
   14717              : {
   14718            0 :   gimple_seq stmts = NULL;
   14719            0 :   tree len_type = TREE_TYPE (len);
   14720            0 :   gcc_assert (TREE_TYPE (start_index) == len_type);
   14721              : 
   14722            0 :   tree min = gimple_build (&stmts, MIN_EXPR, len_type, start_index, end_index);
   14723            0 :   tree left_len = gimple_build (&stmts, MINUS_EXPR, len_type, end_index, min);
   14724            0 :   tree rhs = gimple_build (&stmts, MIN_EXPR, len_type, left_len, len_limit);
   14725            0 :   gimple* stmt = gimple_build_assign (len, rhs);
   14726            0 :   gimple_seq_add_stmt (&stmts, stmt);
   14727              : 
   14728            0 :   return stmts;
   14729              : }
   14730              : 
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.